diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs index 3a3381bbd8..e5bf80b787 100644 --- a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -1,7 +1,7 @@ //! The Python shape names of the natively hosted compiler shapes. /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 491] = [ ( "sqlbuild._native:AlignedInterval", "sqlbuild.cursor_algebra.models:AlignedInterval", @@ -446,6 +446,10 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [ "sqlbuild._native:DeclarationVisibilityIndex", "sqlbuild.compiler.scopes.models:DeclarationVisibilityIndex", ), + ( + "sqlbuild._native:CompiledResourceType", + "sqlbuild.compiler.compile.types:CompiledResourceType", + ), ( "sqlbuild._native:DefaultsConfig", "sqlbuild.spec.contracts.models:DefaultsConfig", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 1ddd6ac196..9b40f131af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -77,7 +77,7 @@ impl<'py> PythonSqlOwner<'py> { /// Return P005, P007 and P008 errors and unmatched Python-SQL relations; raise P001. #[pyfunction] #[pyo3(signature = (*, project, discovered_inputs, enforce_explicit))] -fn explicit_reference_report<'py>( +pub(crate) fn explicit_reference_report<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 14310b945c..0239b771e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -20,6 +20,9 @@ use crate::bindings::_helpers::analysis_session::session::{ infer_expression_source_shapes, prove_dynamic_column_contracts, }; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::stage_work::{ + EXPRESSION_SHAPE_BATCHES, EXPRESSION_SHAPES_INFERRED, count_stage, +}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -105,9 +108,16 @@ fn infer_native_expression_source_shapes<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("infer_native_expression_source_shapes", request)?; - let py: Python<'py> = request.py(); - let expressions: Bound<'py, PyAny> = request.get("expressions")?; - let profile: Bound<'py, PyAny> = request.get("profile")?; + expression_source_shapes(&request.get("expressions")?, &request.get("profile")?) +} + +/// One shape per expression, cached on `profile`'s binding catalog; a failure raises. +pub(crate) fn expression_source_shapes<'py>( + expressions: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = profile.py(); + let profile: Bound<'py, PyAny> = profile.clone(); let dialect: String = aliased_dialect(py, profile_dialect(&profile)?)?; let catalog: Bound<'py, PyAny> = profile_catalog(&profile, &dialect)?; let cache: Bound<'py, PyAny> = native_field(&catalog, "expression_shapes")?; @@ -119,6 +129,8 @@ fn infer_native_expression_source_shapes<'py>( } } if !pending.is_empty() { + count_stage(py, EXPRESSION_SHAPE_BATCHES, 1)?; + count_stage(py, EXPRESSION_SHAPES_INFERRED, pending.len() as u64)?; let (rules, adapter_rules) = nullability_rules(&profile)?; let native: PyRef<'py, ProjectCatalog> = native_field(&catalog, "native")?.extract()?; let pending_names: Vec = pending @@ -169,10 +181,20 @@ fn prove_native_dynamic_contracts<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("prove_native_dynamic_contracts", request)?; - let py: Python<'py> = request.py(); - let session: Bound<'py, PyAny> = request.get("session")?; - let tables: Bound<'py, PyAny> = request.get("tables")?; - let models: Bound<'py, PyAny> = request.get("models")?; + dynamic_contract_proofs( + &request.get("session")?, + &request.get("tables")?, + &request.get("models")?, + ) +} + +/// The proof row per `(sql, families)` model from `session`, or from `tables` without one. +pub(crate) fn dynamic_contract_proofs<'py>( + session: &Bound<'py, PyAny>, + tables: &Bound<'py, PyAny>, + models: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = models.py(); if !models.is_truthy()? { return Ok(PyTuple::empty(py)); } @@ -182,7 +204,7 @@ fn prove_native_dynamic_contracts<'py>( rows.push((sql, family_rows(&families)?)); } let contracts: Vec = if session.is_none() { - let by_table: Bound<'py, PyAny> = native_field(&tables, "dynamic_families_by_table")?; + let by_table: Bound<'py, PyAny> = native_field(tables, "dynamic_families_by_table")?; let mut families: Vec<(String, Vec)> = Vec::new(); for item in by_table.call_method0("items")?.try_iter()? { let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; @@ -191,13 +213,13 @@ fn prove_native_dynamic_contracts<'py>( prove_dynamic_column_contracts( py, ( - dialect_or_generic(&native_field(&tables, "dialect")?)?, - shape_rows(&native_field(&tables, "column_types_by_table")?)?, + dialect_or_generic(&native_field(tables, "dialect")?)?, + shape_rows(&native_field(tables, "column_types_by_table")?)?, shape_rows(&native_field( - &tables, + tables, "authoritative_column_types_by_table", )?)?, - shape_rows(&native_field(&tables, "column_nullability_by_table")?)?, + shape_rows(&native_field(tables, "column_nullability_by_table")?)?, families, rows, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs index a4914d0697..147b8f8056 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs @@ -13,6 +13,7 @@ use sqlbuild_core::build_command::errors::BuildCommandError; use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::policies::build_command_error; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -586,30 +587,32 @@ pub(crate) fn run_defer_clone_boundary_prephase( } let adapter: Bound<'_, PyAny> = native_field(invocation, "adapter")?; let project_dir: Bound<'_, PyAny> = native_field(invocation, "effective_project_dir")?; - let project: Bound<'_, PyAny> = native( + let project: Bound<'_, PyAny> = build_compiled_project( py, - NativeAttr::BuildCompiledProject, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("selected_target", native_field(request, "selected_target")?), - ( - "no_sql_validation", - native_field(request, "no_sql_validation")?, - ), - ( - "analysis_selection", - call_with( - ®istry::class(py, RecordId::CompileAnalysisSelection)?, - &[("no_cache", native_field(request, "no_cache")?)], - )?, - ), - ("cli_vars", native_field(request, "cli_vars")?), - ( - "external_sql_reference_resolver", - external_resolver(&project_dir, &discovered)?, - ), - ], + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", native_field(request, "selected_target")?), + ( + "no_sql_validation", + native_field(request, "no_sql_validation")?, + ), + ( + "analysis_selection", + call_with( + ®istry::class(py, RecordId::CompileAnalysisSelection)?, + &[("no_cache", native_field(request, "no_cache")?)], + )?, + ), + ("cli_vars", native_field(request, "cli_vars")?), + ( + "external_sql_reference_resolver", + external_resolver(&project_dir, &discovered)?, + ), + ], + )?), )?; let scope: Bound<'_, PyAny> = native( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 89a6712f1f..d7756831c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -10,6 +10,8 @@ use sqlbuild_core::build_command::models::DurationLimit; use crate::bindings::_helpers::build_command::policies::build_command_error; use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::fields::field as native_field; @@ -105,12 +107,15 @@ pub(crate) fn resolve_build_context<'py>( let project_dir: Bound<'py, PyAny> = or_none(native_field(request, "project_dir")?, || { imported(py, "pathlib", "Path")?.call_method0("cwd") })?; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("extract_output_column_locations", boolean(py, false)), - ], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", boolean(py, false)), + ], + )?), )?; let selected_target: Bound<'py, PyAny> = native_field(request, "selected_target")?; let target_name: Bound<'py, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index b86d3fed28..5637fc9624 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -22,7 +22,6 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const SUCCESS_STATUS: &str = "success"; /// The cursor window and output settings the Python lifecycle runs with. @@ -323,7 +322,7 @@ fn lifecycle_plan<'py>( &[("selection", selection), ("python_graph", graph.clone())], )?; let source_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let mut keys: Vec> = Vec::new(); for name in native_field(&plan, "ingress_loader_names")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index 5597092321..51707dada5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -36,7 +36,7 @@ use crate::bindings::_helpers::check::check_results::{ check_result_text, results_json, write_results, }; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::compile_outputs::runtime_target::write_python_check_runtime_target; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::graph::python_selection::python_node_names; @@ -44,10 +44,10 @@ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::result_output::event_writer::ExecutionEventWriter; -const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const GRAPH_OPERATOR_CODE: &str = "C681"; const GRAPH_OPERATOR_MESSAGE: &str = "sqb check selectors do not support graph operators; \ select checks directly without '+', '~', or path-between expansion"; @@ -201,7 +201,7 @@ fn compile_pipeline<'py>( let loaders: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, loaders)?; let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let source_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; let mut keys: Vec> = Vec::new(); for reference in required_refs.try_iter()? { keys.push(sql_ref_key(py, &reference?)?); @@ -572,7 +572,7 @@ fn execute<'py>( let connection: Bound<'py, PyAny> = adapter.call_method1("connect", (native_field(invocation, "connection_config")?,))?; let event_writer: Bound<'py, PyAny> = - registry::native(py, NativeAttr::ExecutionEventWriter)?.call0()?; + Bound::new(py, ExecutionEventWriter::new(py, None)?)?.into_any(); let executed: PyResult>> = (|| { let projector: Bound<'py, PyAny> = current_native_progress_projector(py)?; if !projector.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs index 3e519738d2..ac240e891f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs @@ -48,7 +48,7 @@ impl ConnectionProgressReporter { impl ConnectionProgressReporter { #[new] #[pyo3(signature = (**options))] - fn new(py: Python<'_>, options: Option>) -> PyResult { + pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { let arguments: KeywordArguments<'_> = KeywordArguments::read( py, "ConnectionProgressReporter", @@ -133,7 +133,11 @@ impl PlanningProgressReporter { impl PlanningProgressReporter { #[new] #[pyo3(signature = (*, stream, use_color = false))] - fn new(py: Python<'_>, stream: &Bound<'_, PyAny>, use_color: bool) -> PyResult { + pub(crate) fn new( + py: Python<'_>, + stream: &Bound<'_, PyAny>, + use_color: bool, + ) -> PyResult { let status: Bound<'_, PyAny> = status_reporter(stream, use_color)?; Ok(Self { status: status.unbind(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs index 1ddcfdbf89..0861ce3202 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs @@ -24,6 +24,8 @@ use crate::bindings::_helpers::clone::clone_execution::{ use crate::bindings::_helpers::clone::clone_pipeline::{ClonePipeline, prepared_clone_pipeline}; use crate::bindings::_helpers::clone::clone_progress::{clone_item_text, write_line}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -132,9 +134,9 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = py.import("sys")?.getattr("stdout")?; let progress: Bound<'py, PyAny> = progress_reporter(&stream, style)?; progress.call_method1("start", ("Discovering project...",))?; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; progress.call_method1("complete", ("Discovered project.",))?; call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs index 96c7645755..5b3bb5f04e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs @@ -12,14 +12,13 @@ use crate::bindings::_helpers::clone::clone_command::Invocation; use crate::bindings::_helpers::clone::clone_progress::{ CloneFingerprintReporter, CloneItemPrinter, }; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const DESTINATION_ENTRY_GROUPS: [&str; 4] = [ "destination_source_entries", "destination_model_entries", @@ -125,7 +124,7 @@ fn dependency_locations<'py>( let location_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledRelationLocation)?; let source_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; for pair in source_map.call_method0("items")?.try_iter()? { let (name, source): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; let table: Bound<'py, PyAny> = native_field(&source, "table")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs index 4f2c067f60..9522e6af79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs @@ -5,7 +5,9 @@ use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -39,30 +41,33 @@ fn compiled_environment<'py>( ) -> PyResult> { let py: Python<'py> = target_name.py(); let options: &Bound<'py, PyAny> = &pipeline.options; - call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", pipeline.discovered.clone()), - ("adapter", pipeline.adapter.clone()), - ("selected_target", target_name.clone()), - ( - "resolved_connection", - native_field(&pipeline.connection, "config")?, - ), - ( - "no_sql_validation", - native_field(options, "no_sql_validation")?, - ), - ( - "analysis_selection", - call_with( - ®istry::class(py, RecordId::CompileAnalysisSelection)?, - &[("no_cache", native_field(options, "no_cache")?)], - )?, - ), - ("cli_vars", native_field(options, "cli_vars")?), - ("external_sql_reference_resolver", pipeline.resolver.clone()), - ], + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", pipeline.discovered.clone()), + ("adapter", pipeline.adapter.clone()), + ("selected_target", target_name.clone()), + ( + "resolved_connection", + native_field(&pipeline.connection, "config")?, + ), + ( + "no_sql_validation", + native_field(options, "no_sql_validation")?, + ), + ( + "analysis_selection", + call_with( + ®istry::class(py, RecordId::CompileAnalysisSelection)?, + &[("no_cache", native_field(options, "no_cache")?)], + )?, + ), + ("cli_vars", native_field(options, "cli_vars")?), + ("external_sql_reference_resolver", pipeline.resolver.clone()), + ], + )?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index e8b1dee75f..a43dcb0451 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -4,6 +4,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyBool; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::fields::field as native_field; @@ -73,9 +75,9 @@ fn invocation_context<'py>( } else { requested }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let adapter_context: Bound<'py, PyAny> = resolve_adapter_connection_context( &discovered, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs index ac40f36b3c..227025419c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -5,6 +5,7 @@ use pyo3::pyfunction; use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::stage_work::{DECLARATION_SCOPES, count_stage}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -50,6 +51,7 @@ pub(crate) fn build_declaration_scope<'py>( compile_cache_dir: Option>, ) -> PyResult> { let py: Python<'py> = discovered_inputs.py(); + count_stage(py, DECLARATION_SCOPES, 1)?; let scan_cache: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::SqlTestScanCache)?, &[( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs index 1b98edf359..f8c3245c49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -7,9 +7,10 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::compile_attachment::storage_policies::inherited_storage; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::compiled_project::loop_classes::table_backed; +use crate::bindings::_helpers::model_config::model_config_builder::NativeModelConfigBuilder; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// Capture the project inputs every model's native config build and validation read. @@ -39,27 +40,24 @@ pub(crate) fn native_model_config_session<'py>( .map(|class| registry::class(py, *class)) .collect::>>()?, )?; - let target_namespace: Bound<'py, PyAny> = if target.is_none() { - target.clone() + let target_namespace: Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = if target.is_none() { + None } else { - PyTuple::new( - py, - [ - native_field(&target, "database")?, - native_field(&target, "schema")?, - ], - )? - .into_any() + Some(( + native_field(&target, "database")?, + native_field(&target, "schema")?, + )) }; - let builder: Bound<'py, PyAny> = registry::native(py, NativeAttr::NativeModelConfigBuilder)? - .call1(( + let builder: Bound<'py, PyAny> = Bound::new( + py, + NativeModelConfigBuilder::new( ( native_field(&project_config, "defaults")?, - native_field(&project_config, "path_defaults")?, + native_field(&project_config, "path_defaults")?.downcast_into()?, hook_entries, ), ( - native_field(inputs, "effective_vars")?, + native_field(inputs, "effective_vars")?.downcast_into()?, unicode_environment(py)?, ), ( @@ -69,7 +67,9 @@ pub(crate) fn native_model_config_session<'py>( ), target_namespace, ), - ))?; + )?, + )? + .into_any(); let inherited: Bound<'py, PyDict> = PyDict::new(py); let mut materializations: Vec> = vec![py.None().into_bound(py)]; materializations.extend(table_backed(py)?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 7c4ed3d9c3..0788d4989d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -6,6 +6,8 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods}; use crate::bindings::_helpers::compile_attachment::declaration_scope::rebind_declaration_scope; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_vars; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; @@ -156,15 +158,18 @@ pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult discovered, - None => call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", sources.project_dir.clone()), - ( - "sql_analysis_enabled_override", - PyBool::new(py, false).to_owned().into_any(), - ), - ], + None => discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", sources.project_dir.clone()), + ( + "sql_analysis_enabled_override", + PyBool::new(py, false).to_owned().into_any(), + ), + ], + )?), )?, }; let project_config: Bound<'_, PyAny> = native_field(&discovered, "project_config")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index cb6fbaf5bf..bafd15f163 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -7,16 +7,28 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMetho use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_project_graph_with_analysis_selection; use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::compile_driver::stage_work::{ + COLUMN_LINEAGE, MODEL_CONTRACTS, count_stage, +}; +use crate::bindings::_helpers::contracts::contract_diagnostics::evaluate_model_contracts; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::lineage::project_lineage::build_project_column_lineage; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_selection::selectors::select_project_graph; +use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::RULE_FIX_AVAILABLE_NOTE; use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::rules_progress; +use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; use crate::bindings::_helpers::rules::rules_config::load_rules_config; const RICH_LINEAGE: &str = "rich"; @@ -46,11 +58,16 @@ fn compile_lineage<'py>( } else { return Ok(py.None().into_bound(py)); }; - registry::native(py, NativeAttr::BuildProjectColumnLineage)?.call1(( - graph.getattr("project")?, - dialect, - column_lineage_mode(py, member)?, - py.None(), + count_stage(py, COLUMN_LINEAGE, 1)?; + Ok(build_project_column_lineage( + &native_field(graph, "project")?, + Some(dialect), + &column_lineage_mode(py, member)?, + None, + )? + .map_or_else( + || py.None().into_bound(py), + |lineage| lineage.into_bound(py), )) } @@ -105,7 +122,7 @@ fn seconds(milliseconds: u64) -> String { /// Whether any of `diagnostics` is an error. fn any_error(diagnostics: &Bound<'_, PyAny>) -> PyResult { for diagnostic in diagnostics.try_iter()? { - if diagnostic?.getattr("is_error")?.is_truthy()? { + if native_field(&diagnostic?, "is_error")?.is_truthy()? { return Ok(true); } } @@ -129,8 +146,8 @@ pub(crate) fn analyze<'py>( ) -> PyResult> { let options: AnalysisOptions<'py> = AnalysisOptions::read(py, options)?; let selection: &Bound<'py, PyAny> = &options.analysis_selection; - let enabled: bool = !(selection.getattr("select")?.is_truthy()? - || selection.getattr("exclude")?.is_truthy()?); + let enabled: bool = !(native_field(selection, "select")?.is_truthy()? + || native_field(selection, "exclude")?.is_truthy()?); let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; let analysis: PyResult> = analyzed(py, &options, &early_lint); let _ = early_lint.borrow().stop(py); @@ -165,8 +182,8 @@ fn analyzed<'py>( options, status: Status(options.status.clone()), early_lint, - select: options.analysis_selection.getattr("select")?, - exclude: options.analysis_selection.getattr("exclude")?, + select: native_field(&options.analysis_selection, "select")?, + exclude: native_field(&options.analysis_selection, "exclude")?, }; let (discovered, discover_ms): (Bound<'py, PyAny>, u64) = driver.discover()?; let adapter: Bound<'py, PyAny> = driver.adapter(&discovered)?; @@ -178,28 +195,22 @@ fn analyzed<'py>( }; let (graph, graph_ms): (Bound<'py, PyAny>, u64) = driver.graph((&discovered, &adapter), &lint_dialect)?; - let project: Bound<'py, PyAny> = graph.getattr("project")?; - let graph_failed: bool = any_error(&project.getattr("diagnostics")?)?; + let project: Bound<'py, PyAny> = native_field(&graph, "project")?; + let graph_failed: bool = any_error(&native_field(&project, "diagnostics")?)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, &dialect)?; - let selected_keys: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::SelectProjectGraph)?, - &[ - ("graph", graph.clone()), - ("select", driver.select.clone()), - ("exclude", driver.exclude.clone()), - ], - )?; + let selected_keys: Bound<'py, PyAny> = + select_project_graph(&graph, &driver.select, &driver.exclude)?.into_any(); let core_diagnostics: Bound<'py, PyTuple> = PyTuple::new( py, project .getattr("diagnostics")? .try_iter()? - .chain(contracts.result.getattr("diagnostics")?.try_iter()?) + .chain(native_field(&contracts.result, "diagnostics")?.try_iter()?) .collect::>>()?, )?; let rules_result: Bound<'py, PyAny> = if graph_failed { @@ -235,18 +246,27 @@ fn analyzed<'py>( "contract_cpu_ms", contracts.cpu_ms.into_pyobject(py)?.into_any(), ), - ("built_in_rules_ms", rules_result.getattr("built_in_ms")?), - ("custom_rules_ms", rules_result.getattr("custom_ms")?), + ( + "built_in_rules_ms", + native_field(&rules_result, "built_in_ms")?, + ), + ("custom_rules_ms", native_field(&rules_result, "custom_ms")?), ( "custom_rules_cpu_ms", - rules_result.getattr("custom_cpu_ms")?, + native_field(&rules_result, "custom_cpu_ms")?, ), ("early_lint_wait_ms", early_lint_wait_ms), - ("rule_cache_hits", rules_result.getattr("cache_hits")?), - ("rule_cache_misses", rules_result.getattr("cache_misses")?), + ( + "rule_cache_hits", + native_field(&rules_result, "cache_hits")?, + ), + ( + "rule_cache_misses", + native_field(&rules_result, "cache_misses")?, + ), ( "skipped_type_proof_rules", - rules_result.getattr("skipped_type_proof_rules")?, + native_field(&rules_result, "skipped_type_proof_rules")?, ), ], ) @@ -283,22 +303,18 @@ impl<'py> Driver<'_, 'py> { } else { py.None().into_bound(py) }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", self.options.project_dir.clone()), - ("extract_output_column_locations", false_value), - ("sql_analysis_enabled_override", override_analysis), - ], - )?; - call_with( - ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, - &[ - ("select", self.select.clone()), - ("exclude", self.exclude.clone()), - ("discovered_inputs", discovered.clone()), - ], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", self.options.project_dir.clone()), + ("extract_output_column_locations", false_value), + ("sql_analysis_enabled_override", override_analysis), + ], + )?), )?; + reject_unit_test_selectors(&self.select, &self.exclude, &discovered)?; let discover_ms: u64 = elapsed_ms(started); self.status .complete(&format!("Discovered project. ({}s)", seconds(discover_ms)))?; @@ -307,12 +323,9 @@ impl<'py> Driver<'_, 'py> { /// The adapter the project and local configuration select. fn adapter(&self, discovered: &Bound<'py, PyAny>) -> PyResult> { - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(self.py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ], + let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, )?; resolve_cli_adapter(&adapter_name, &self.options.project_dir) } @@ -354,21 +367,24 @@ impl<'py> Driver<'_, 'py> { }, )?; let graph: Bound<'py, PyAny> = within(&lifecycle, |_| { - call_with( - ®istry::native(py, NativeAttr::BuildProjectGraphWithAnalysisSelection)?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("selected_target", options.selected_target.clone()), - ("no_sql_validation", options.no_sql_validation.clone()), - ( - "skip_column_inference", - options.profile_flags.getattr("skip_column_inference")?, - ), - ("column_lineage_mode", lineage_mode), - ("cli_vars", options.cli_vars.clone()), - ("analysis_selection", analysis_selection), - ], + build_project_graph_with_analysis_selection( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("selected_target", options.selected_target.clone()), + ("no_sql_validation", options.no_sql_validation.clone()), + ( + "skip_column_inference", + native_field(&options.profile_flags, "skip_column_inference")?, + ), + ("column_lineage_mode", lineage_mode), + ("cli_vars", options.cli_vars.clone()), + ("analysis_selection", analysis_selection), + ], + )?), ) })?; let graph_ms: u64 = elapsed_ms(started); @@ -417,10 +433,8 @@ impl<'py> Driver<'_, 'py> { )? } else { self.status.start("Validating model contracts...")?; - call_with( - ®istry::native(py, NativeAttr::EvaluateModelContracts)?, - &[("project", project.clone()), ("dialect", dialect.clone())], - )? + count_stage(py, MODEL_CONTRACTS, 1)?; + evaluate_model_contracts(py, project, Some(dialect))? }; let wall_ms: u64 = elapsed_ms(started); let cpu_ended: f64 = thread_time.call0()?.extract()?; @@ -453,11 +467,11 @@ impl<'py> Driver<'_, 'py> { if !options.prepared_artifacts.is_none() && !any_error(core_diagnostics.as_any())? { let target_dir: Bound<'py, PyAny> = project_dir.call_method1("__truediv__", (TARGET_DIRECTORY,))?; - let stage_artifacts: bool = rules_config.getattr("select")?.is_truthy()?; + let stage_artifacts: bool = native_field(&rules_config, "select")?.is_truthy()?; call_with( - &options.prepared_artifacts.getattr("start")?, + &native_field(&options.prepared_artifacts, "start")?, &[ - ("project", graph.getattr("project")?), + ("project", native_field(graph, "project")?), ("adapter", adapter.clone()), ("target_dir", target_dir), ( @@ -476,18 +490,21 @@ impl<'py> Driver<'_, 'py> { } else { py.None().into_bound(py) }; - let result: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::EvaluateRules)?, - &[ - ("graph", graph.clone()), - ("discovered_inputs", discovered.clone()), - ("config", rules_config), - ("project_dir", project_dir.clone()), - ("dialect", lint_dialect.clone()), - ("selected_keys", selected), - ("early_lint", self.early_lint.clone().into_any()), - ("no_sql_analysis", options.no_sql_validation.clone()), - ], + let result: Bound<'py, PyAny> = evaluate_rules( + py, + Some(keywords( + py, + &[ + ("graph", graph.clone()), + ("discovered_inputs", discovered.clone()), + ("config", rules_config), + ("project_dir", project_dir.clone()), + ("dialect", lint_dialect.clone()), + ("selected_keys", selected), + ("early_lint", self.early_lint.clone().into_any()), + ("no_sql_analysis", options.no_sql_validation.clone()), + ], + )?), )?; let ended: f64 = monotonic.call0()?.extract()?; let progress: String = rules_progress(ended - started, &result, false)?; @@ -509,9 +526,9 @@ fn with_rule_diagnostics<'py>( registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?; let fix_note: Bound<'py, PyAny> = PyString::new(py, RULE_FIX_AVAILABLE_NOTE).into_any(); let mut diagnostics: Vec> = core.iter().collect(); - for fault in rules_result.getattr("findings")?.try_iter()? { + for fault in native_field(rules_result, "findings")?.try_iter()? { let fault: Bound<'py, PyAny> = fault?; - let notes: Bound<'py, PyTuple> = if fault.getattr("fixable")?.is_truthy()? { + let notes: Bound<'py, PyTuple> = if native_field(&fault, "fixable")?.is_truthy()? { PyTuple::new(py, [fix_note.clone()])? } else { PyTuple::empty(py) @@ -521,14 +538,14 @@ fn with_rule_diagnostics<'py>( &[ ("phase", phase.clone()), ("severity", severity.clone()), - ("code", fault.getattr("code")?), - ("message", fault.getattr("message")?), - ("path", fault.getattr("path")?), - ("line", fault.getattr("line")?), - ("column", fault.getattr("column")?), - ("help", fault.getattr("remediation")?), + ("code", native_field(&fault, "code")?), + ("message", native_field(&fault, "message")?), + ("path", native_field(&fault, "path")?), + ("line", native_field(&fault, "line")?), + ("column", native_field(&fault, "column")?), + ("help", native_field(&fault, "remediation")?), ("notes", notes.into_any()), - ("affected_rules", fault.getattr("affected_rules")?), + ("affected_rules", native_field(&fault, "affected_rules")?), ], )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 2bd86cf8e0..fa3ba20617 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -1,10 +1,15 @@ //! Planner-ready project assembly: diagnostics collection, the macro bridge and explicit references. +use std::sync::{Arc, Mutex, PoisonError}; + +use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::analysis::explicit_references::explicit_reference_report; use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; +use crate::bindings::_helpers::compile_driver::diagnostic_collector::collect_compile_diagnostics; use crate::bindings::_helpers::compile_driver::project::lineage_mode; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ @@ -14,8 +19,12 @@ use crate::bindings::_helpers::compiled_project::records::project_with; use crate::bindings::_helpers::compiled_project::records::record_model_analyses; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::fields::field as native_field; + +const STAGE_REACHED: &str = "assembly stopped before the requested stage"; +const EXPRESSION_SHAPES_STAGE: &str = "expression_shapes"; +const MODEL_FACTS_STAGE: &str = "model_facts"; +const SEMANTIC_COMPLETION_STAGE: &str = "semantic_completion"; const OPTIONAL: [&str; 5] = [ "inference_profile", @@ -33,6 +42,53 @@ pub(crate) struct AssemblyRequest { pub(crate) column_lineage_mode: Py, pub(crate) analysis_cache_dir: Py, pub(crate) analysis_model_names: Py, + /// The stage to stop before, keeping its inputs, when assembly runs for stage tests. + pub(crate) stop: Option, +} + +/// An assembly stage whose inputs tests can take instead of running it. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum AssemblyStage { + ExpressionShapes, + ModelFacts, + SemanticCompletion, +} + +/// Where an assembly stopped for a stage test keeps that stage's inputs. +pub(crate) struct StageStop { + stage: AssemblyStage, + inputs: Arc>>>, +} + +impl AssemblyRequest { + /// Stop with `inputs` before `stage` when the request stops there; else continue. + pub(crate) fn before_stage<'py>( + &self, + stage: AssemblyStage, + inputs: impl FnOnce() -> PyResult>, + ) -> PyResult<()> { + let Some(stop) = self.stop.as_ref().filter(|stop| stop.stage == stage) else { + return Ok(()); + }; + *stop.inputs.lock().unwrap_or_else(PoisonError::into_inner) = Some(inputs()?.unbind()); + Err(PyRuntimeError::new_err(STAGE_REACHED)) + } +} + +/// The assembly request of `arguments`, stopping at `stop` when given. +fn assembly_request( + arguments: &KeywordArguments<'_>, + stop: Option, +) -> PyResult { + Ok(AssemblyRequest { + inputs: arguments.value("inputs")?.unbind(), + inference_profile: arguments.value("inference_profile")?.unbind(), + skip_column_inference: arguments.value_or_false("skip_column_inference")?.unbind(), + column_lineage_mode: lineage_mode(arguments)?.unbind(), + analysis_cache_dir: arguments.value("analysis_cache_dir")?.unbind(), + analysis_model_names: arguments.value("analysis_model_names")?.unbind(), + stop, + }) } /// Convert compile inputs into the planner-ready project view. @@ -44,25 +100,64 @@ pub(crate) fn assemble_project<'py>( ) -> PyResult> { let arguments: KeywordArguments<'py> = KeywordArguments::read(py, "assemble_project", options, (&["inputs"], &OPTIONAL))?; - let request: AssemblyRequest = AssemblyRequest { - inputs: arguments.value("inputs")?.unbind(), - inference_profile: arguments.value("inference_profile")?.unbind(), - skip_column_inference: arguments.value_or_false("skip_column_inference")?.unbind(), - column_lineage_mode: lineage_mode(&arguments)?.unbind(), - analysis_cache_dir: arguments.value("analysis_cache_dir")?.unbind(), - analysis_model_names: arguments.value("analysis_model_names")?.unbind(), + collected_project(py, assembly_request(&arguments, None)?) +} + +/// Test support: assemble like `assemble_project` but stop before `stage`, returning its inputs or None. +#[pyfunction] +#[pyo3(signature = (stage, **options))] +pub(crate) fn assembly_stage_inputs<'py>( + py: Python<'py>, + stage: &str, + options: Option>, +) -> PyResult>> { + let stage: AssemblyStage = match stage { + EXPRESSION_SHAPES_STAGE => AssemblyStage::ExpressionShapes, + MODEL_FACTS_STAGE => AssemblyStage::ModelFacts, + SEMANTIC_COMPLETION_STAGE => AssemblyStage::SemanticCompletion, + _ => { + return Err(PyValueError::new_err(format!( + "unknown assembly stage '{stage}'" + ))); + } }; - let collector: Bound<'py, PyAny> = - registry::native(py, NativeAttr::CollectCompileDiagnostics)?.call0()?; + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "assembly_stage_inputs", + options, + (&["inputs"], &OPTIONAL), + )?; + let inputs: Arc>>> = Arc::new(Mutex::new(None)); + let stop: StageStop = StageStop { + stage, + inputs: Arc::clone(&inputs), + }; + let assembled: PyResult> = + collected_project(py, assembly_request(&arguments, Some(stop))?); + let captured: Option> = inputs.lock().unwrap_or_else(PoisonError::into_inner).take(); + match (captured, assembled) { + (Some(captured), _) => Ok(Some(captured.into_bound(py))), + (None, Ok(_)) => Ok(None), + (None, Err(error)) => Err(error), + } +} + +/// `request`'s project, with diagnostics collected while assembling appended. +fn collected_project<'py>( + py: Python<'py>, + request: AssemblyRequest, +) -> PyResult> { + let collector: Bound<'py, PyAny> = collect_compile_diagnostics(py)?.into_any(); let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = within(&collector, |violations| { Ok((referenced_project(py, request)?, violations.clone())) })?; - let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; + let collected: Bound<'py, PyAny> = native_field(&violations, "diagnostics")?; if !collected.is_truthy()? { return Ok(built); } - let diagnostics: Bound<'py, PyTuple> = chained(&built.getattr("diagnostics")?, &collected)?; + let diagnostics: Bound<'py, PyTuple> = + chained(&native_field(&built, "diagnostics")?, &collected)?; project_with( &built, Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), @@ -106,28 +201,26 @@ fn referenced_project(py: Python<'_>, request: AssemblyRequest) -> PyResult = - with_native_project(&project, &inputs.getattr("native_project")?)?; - let references: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ExplicitReferenceReport)?, - &[ - ("project", project.clone()), - ("discovered_inputs", inputs.getattr("discovered_inputs")?), - ( - "enforce_explicit", - inputs - .getattr("project_config")? - .getattr("references")? - .getattr("enforce_explicit")?, - ), - ], + with_native_project(&project, &native_field(&inputs, "native_project")?)?; + let references: Bound<'_, PyAny> = explicit_reference_report( + py, + &project, + &native_field(&inputs, "discovered_inputs")?, + inputs + .getattr("project_config")? + .getattr("references")? + .getattr("enforce_explicit")? + .extract()?, )?; - let reference_diagnostics: Bound<'_, PyAny> = references.getattr("diagnostics")?; - let unmatched: Bound<'_, PyAny> = references.getattr("unmatched")?; + let reference_diagnostics: Bound<'_, PyAny> = native_field(&references, "diagnostics")?; + let unmatched: Bound<'_, PyAny> = native_field(&references, "unmatched")?; if !reference_diagnostics.is_truthy()? && !unmatched.is_truthy()? { return Ok(project); } - let diagnostics: Bound<'_, PyTuple> = - chained(&project.getattr("diagnostics")?, &reference_diagnostics)?; + let diagnostics: Bound<'_, PyTuple> = chained( + &native_field(&project, "diagnostics")?, + &reference_diagnostics, + )?; project_with( &project, Some(keywords( @@ -149,8 +242,8 @@ fn with_native_project<'py>( if native_project.is_none() { return Ok(project.clone()); } - record_model_analyses(native_project.cast()?, &project.getattr("models")?)?; - if project.getattr("native_project")?.is(native_project) { + record_model_analyses(native_project.cast()?, &native_field(project, "models")?)?; + if native_field(project, "native_project")?.is(native_project) { return Ok(project.clone()); } project_with( @@ -160,5 +253,6 @@ fn with_native_project<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(assemble_project, module)?) + module.add_function(wrap_pyfunction!(assemble_project, module)?)?; + module.add_function(wrap_pyfunction!(assembly_stage_inputs, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index f9b1c6463f..29ba31d849 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -1,7 +1,7 @@ //! Project assembly's inference profile, binding catalog, source shapes and model analysis. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; @@ -9,17 +9,26 @@ use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignor use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; +use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; -use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; +use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; use crate::bindings::_helpers::compile_driver::compile_timings::{ record_compile_cpu_timing, record_compile_timing, }; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compile_driver::stage_work::{ + EXPRESSION_SHAPES, MODEL_ANALYSIS, count_stage, +}; use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::project_assembly::declared_relations::{ + DeclaredRelations, declared_relations, +}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::semantic_shapes::declarations::{declared_types, function_names}; const GENERIC_DIALECT: &str = "generic"; const UDF_PREFIX: &str = "__sqlbuild_udf_"; @@ -78,11 +87,17 @@ pub(crate) fn analysis_context<'py>( .getattr("sql_analysis")?, request.skip_column_inference.bind(py), )?; - let relations: Bound<'py, PyAny> = - registry::native(py, NativeAttr::DeclaredRelations)?.call1((inputs,))?; - let column_types: Bound<'py, PyAny> = relations.get_item(0)?; - let nullability: Bound<'py, PyAny> = relations.get_item(1)?; - let complete_schemas: Bound<'py, PyAny> = relations.get_item(2)?; + let (column_types, nullability, complete_schemas): DeclaredRelations<'py> = + declared_relations(inputs.clone())?; + let (column_types, nullability, complete_schemas): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = ( + column_types.into_any(), + nullability.into_any(), + complete_schemas.into_any(), + ); let dynamic_families: Bound<'py, PyAny> = dynamic_families_by_table(inputs)?.into_any(); let profile: Bound<'py, PyAny> = profile_with_catalog(inputs, request.inference_profile.bind(py), &column_types)?; @@ -96,9 +111,9 @@ pub(crate) fn analysis_context<'py>( analyses: None, }; if context.sql_analysis_enabled.is_truthy()? { - record_expression_shapes(inputs, &context)?; + record_expression_shapes(request, &context)?; } - let catalog: Bound<'py, PyAny> = context.profile.getattr("binding_catalog")?; + let catalog: Bound<'py, PyAny> = native_field(&context.profile, "binding_catalog")?; catalog .getattr("native")? .call_method1("update_relations", (&context.complete_schemas,))?; @@ -115,7 +130,7 @@ fn dynamic_families_by_table<'py>(inputs: &Bound<'py, PyAny>) -> PyResult = registry::class(py, RecordId::ContractPolicy)?.getattr("ENFORCED")?; let families: Bound<'py, PyDict> = PyDict::new(py); - for model_input in inputs.getattr("model_inputs")?.try_iter()? { + for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; let contract: Bound<'py, PyAny> = model_input .getattr("config")? @@ -140,16 +155,10 @@ pub(crate) fn declared_names<'py>( column_types: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let py: Python<'py> = inputs.py(); - let functions: Bound<'py, PyAny> = inputs.getattr("sql_function_inputs")?; - let names: Bound<'py, PyAny> = - registry::native(py, NativeAttr::KnownFunctionNames)?.call1((functions.clone(),))?; - let types: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::KnownDeclaredTypes)?, - &[ - ("functions", functions), - ("column_types", column_types.clone()), - ], - )?; + let functions: Bound<'py, PyAny> = native_field(inputs, "sql_function_inputs")?; + let names: Bound<'py, PyAny> = PyTuple::new(py, function_names(&functions)?)?.into_any(); + let types: Bound<'py, PyAny> = + PyTuple::new(py, declared_types(&functions, column_types)?)?.into_any(); Ok((names, types)) } @@ -168,10 +177,10 @@ fn profile_with_catalog<'py>( }; let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = declared_names(inputs, column_types)?; - let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let dialect: Bound<'py, PyAny> = native_field(&profile, "sql_analysis_dialect")?; let ignore_case: Bound<'py, PyAny> = PyBool::new( py, - ignores_quoted_case(&inputs.getattr("effective_connection")?, &dialect)?, + ignores_quoted_case(&native_field(inputs, "effective_connection")?, &dialect)?, ) .to_owned() .into_any(); @@ -200,7 +209,7 @@ fn profile_with_catalog<'py>( ], )?), )?; - let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + let dialect: Bound<'py, PyAny> = native_field(&profile, "sql_analysis_dialect")?; let dialect: String = if dialect.is_truthy()? { dialect.extract()? } else { @@ -215,8 +224,8 @@ fn profile_with_catalog<'py>( profile .getattr("quoted_identifiers_ignore_case")? .extract()?, - profile.getattr("semantic_known_functions")?.extract()?, - profile.getattr("semantic_known_types")?.extract()?, + native_field(&profile, "semantic_known_functions")?.extract()?, + native_field(&profile, "semantic_known_types")?.extract()?, ), PyDict::new(py).as_any(), )?, @@ -235,19 +244,19 @@ fn function_return_types<'py>( ) -> PyResult> { let py: Python<'py> = inputs.py(); let types: Bound<'py, PyDict> = PyDict::new(py); - types.call_method1("update", (profile.getattr("function_return_types")?,))?; + types.call_method1("update", (native_field(profile, "function_return_types")?,))?; let scalar: Bound<'py, PyDict> = PyDict::new(py); - for function in inputs.getattr("sql_function_inputs")?.try_iter()? { + for function in native_field(inputs, "sql_function_inputs")?.try_iter()? { let function: Bound<'py, PyAny> = function?; - if function.getattr("return_columns")?.is_truthy()? { + if native_field(&function, "return_columns")?.is_truthy()? { continue; } - let name: Bound<'py, PyAny> = function.getattr("name")?.str()?.into_any(); + let name: Bound<'py, PyAny> = native_field(&function, "name")?.str()?.into_any(); let key: Bound<'py, PyAny> = PyString::new(py, UDF_PREFIX) .into_any() .add(name)? .call_method0("upper")?; - scalar.set_item(key, function.getattr("returns")?)?; + scalar.set_item(key, native_field(&function, "returns")?)?; } types.update(scalar.as_mapping())?; Ok(types) @@ -255,13 +264,14 @@ fn function_return_types<'py>( /// Record each expression source's analysed shape as its declared relation. fn record_expression_shapes<'py>( - inputs: &Bound<'py, PyAny>, + request: &AssemblyRequest, context: &AnalysisContext<'py>, ) -> PyResult<()> { - let py: Python<'py> = inputs.py(); + let py: Python<'py> = context.profile.py(); + let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); let mut sources: Vec> = Vec::new(); let mut expressions: Vec> = Vec::new(); - for source_input in inputs.getattr("source_inputs")?.try_iter()? { + for source_input in native_field(inputs, "source_inputs")?.try_iter()? { let source_input: Bound<'py, PyAny> = source_input?; let expression: Bound<'py, PyAny> = source_input .getattr("source_entry")? @@ -271,15 +281,20 @@ fn record_expression_shapes<'py>( expressions.push(expression); } } - let shapes: Vec> = call_with( - ®istry::native(py, NativeAttr::InferNativeExpressionSourceShapes)?, - &[ - ("expressions", PyTuple::new(py, expressions)?.into_any()), - ("profile", context.profile.clone()), - ], - )? - .try_iter()? - .collect::>()?; + let expressions: Bound<'py, PyAny> = PyTuple::new(py, expressions)?.into_any(); + request.before_stage(AssemblyStage::ExpressionShapes, || { + keywords( + py, + &[ + ("expressions", expressions.clone()), + ("profile", context.profile.clone()), + ], + ) + })?; + count_stage(py, EXPRESSION_SHAPES, 1)?; + let shapes: Vec> = expression_source_shapes(&expressions, &context.profile)? + .iter() + .collect(); if shapes.len() != sources.len() { let relation: &str = if shapes.len() < sources.len() { "shorter" @@ -296,7 +311,8 @@ fn record_expression_shapes<'py>( if shape.is_none() { continue; } - let name: Bound<'py, PyAny> = source_input.getattr("source_entry")?.getattr("name")?; + let name: Bound<'py, PyAny> = + native_field(source_input, "source_entry")?.getattr("name")?; let declared_types: Bound<'py, PyAny> = context .column_types .call_method1("get", (&name, PyDict::new(py)))?; @@ -336,8 +352,8 @@ fn model_analyses<'py>( let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); let mode: &Bound<'py, PyAny> = request.column_lineage_mode.bind(py); let lineage_mode: Bound<'py, PyAny> = registry::class(py, RecordId::ColumnLineageMode)?; - let rich: bool = mode.eq(lineage_mode.getattr("RICH")?)?; - let allow_compact: bool = mode.eq(lineage_mode.getattr("FAST")?)? || rich; + let rich: bool = mode.eq(native_field(&lineage_mode, "RICH")?)?; + let allow_compact: bool = mode.eq(native_field(&lineage_mode, "FAST")?)? || rich; let flags: (Bound<'py, PyAny>, Bound<'py, PyAny>) = ( PyBool::new(py, allow_compact).to_owned().into_any(), PyBool::new(py, rich).to_owned().into_any(), @@ -350,14 +366,15 @@ fn model_analyses<'py>( record_compile_timing("model_analysis_ms", || { record_compile_cpu_timing(py, "model_analysis_cpu_ms", || { let mut model_inputs: Vec> = Vec::new(); - for model_input in inputs.getattr("model_inputs")?.try_iter()? { + for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; - if model_input.getattr("sql_validation_enabled")?.is_truthy()? + if native_field(&model_input, "sql_validation_enabled")?.is_truthy()? && selected_for_analysis(&model_input, names)? { model_inputs.push(model_input); } } + count_stage(py, MODEL_ANALYSIS, 1)?; analyze_models(&ModelAnalysisRequest { model_inputs, profile: context.profile.clone(), @@ -388,8 +405,8 @@ fn analysis_cache<'py>( &[ ("known_functions", known_functions), ("known_types", known_types), - ("target", inputs.getattr("effective_target_name")?), - ("vars", inputs.getattr("effective_vars")?), + ("target", native_field(inputs, "effective_target_name")?), + ("vars", native_field(inputs, "effective_vars")?), ], )?; Ok(analysis_cache_context( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 8e9a68e439..621e9178cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -6,11 +6,13 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; -use crate::bindings::_helpers::native_host::facade::FacadeHost; -use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; +use crate::bindings::_helpers::render::macro_expansion_result::{ + ExpansionRequest, expansion_result, +}; use pyo3::IntoPyObjectExt; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. @@ -36,14 +38,14 @@ pub(crate) fn test_model_query_overrides<'py>( ) -> PyResult> { let py: Python<'py> = inputs.py(); let overrides: Bound<'py, PyDict> = PyDict::new(py); - let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; + let payload: Bound<'py, PyAny> = native_field(test_input, "payload")?; if !payload.is_instance(®istry::class( py, RecordId::CompileModelSqlTestInputPayload, )?)? { return Ok(overrides); } - let macro_mocks: Bound<'py, PyAny> = payload.getattr("macro_mocks")?; + let macro_mocks: Bound<'py, PyAny> = native_field(&payload, "macro_mocks")?; if !macro_mocks.is_truthy()? { return Ok(overrides); } @@ -57,48 +59,47 @@ pub(crate) fn test_model_query_overrides<'py>( )], )?), )?; - let declaration_scope: Bound<'py, PyAny> = inputs.getattr("declaration_scope")?; + let declaration_scope: Bound<'py, PyAny> = native_field(inputs, "declaration_scope")?; let lookup: Bound<'py, PyAny> = if declaration_scope.is_none() { py.None().into_bound(py) } else { - declaration_scope.getattr("resolver")?.getattr("lookup")? + native_field(&declaration_scope, "resolver")?.getattr("lookup")? }; - let loaded_macros: Bound<'py, PyAny> = inputs.getattr("loaded_macros")?; + let loaded_macros: Bound<'py, PyAny> = native_field(inputs, "loaded_macros")?; let declaration_resolver: Bound<'py, PyAny> = declaration_scope_resolver( - &inputs.getattr("discovered_inputs")?, - &inputs.getattr("scope_index")?, + &native_field(inputs, "discovered_inputs")?, + &native_field(inputs, "scope_index")?, (&loaded_macros, &lookup), )?; - let expand: Bound<'py, PyAny> = registry::native(py, NativeAttr::ExpandSqlMacros)?; - for model_input in inputs.getattr("model_inputs")?.try_iter()? { + for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; let file_path: Bound<'py, PyAny> = - model_input.getattr("model_file")?.getattr("file_path")?; - let model_name: Bound<'py, PyAny> = file_path.getattr("stem")?; - let macro_source_sql: Bound<'py, PyAny> = model_input.getattr("macro_source_sql")?; + native_field(&model_input, "model_file")?.getattr("file_path")?; + let model_name: Bound<'py, PyAny> = native_field(&file_path, "stem")?; + let macro_source_sql: Bound<'py, PyAny> = native_field(&model_input, "macro_source_sql")?; let macro_source_sql: Bound<'py, PyAny> = if macro_source_sql.is_truthy()? { macro_source_sql } else { - model_input.getattr("query_sql")? + native_field(&model_input, "query_sql")? }; - let expanded: Bound<'py, PyAny> = call_with( - &expand, - &[ - ("sql", macro_source_sql), - ("file_path", file_path), - ("loaded_macros", loaded_macros.clone()), - ("macro_overrides", macro_mocks.clone()), - ("macro_context", macro_context.clone()), - ("declaration_resolver", declaration_resolver.clone()), - ], - )? + let expanded: Bound<'py, PyAny> = expansion_result(ExpansionRequest { + sql: macro_source_sql.downcast_into()?, + file_path, + loaded_macros: loaded_macros.downcast()?.clone(), + macro_overrides: Some(macro_mocks.downcast()?.clone()), + macro_context: macro_context.clone(), + declaration_resolver: (!declaration_resolver.is_none()) + .then(|| declaration_resolver.clone()), + declarations: None, + consumer: None, + })? .getattr("sql")?; let query: String = validated_model_cursor_sql( py, &expanded.extract::()?, ( &model_name.extract::()?, - &model_input.getattr("config")?.getattr("values")?, + &native_field(&model_input, "config")?.getattr("values")?, ), )?; overrides.set_item(model_name, query)?; @@ -109,13 +110,13 @@ pub(crate) fn test_model_query_overrides<'py>( /// The compile's macro context, or one built from the effective adapter, settings and target. fn model_macro_context<'py>(inputs: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = inputs.py(); - let macro_context: Bound<'py, PyAny> = inputs.getattr("macro_context")?; + let macro_context: Bound<'py, PyAny> = native_field(inputs, "macro_context")?; if macro_context.is_truthy()? { return Ok(macro_context); } let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( - &inputs.getattr("project_config")?, - &inputs.getattr("local_config")?, + &native_field(inputs, "project_config")?, + &native_field(inputs, "local_config")?, ) .and_then(|value| value.into_bound_py_any(py))?; call_with( @@ -128,8 +129,11 @@ fn model_macro_context<'py>(inputs: &Bound<'py, PyAny>) -> PyResult( let scenario_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledSqlScenario)?; let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let resource_type: Bound<'py, PyAny> = - host::entry(py, FacadeHost::CompiledResourceType)?.getattr("SQL_SCENARIO")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SQL_SCENARIO")?; let mut scenarios: Vec> = Vec::new(); - for scenario_input in inputs.getattr("scenario_inputs")?.try_iter()? { + for scenario_input in native_field(inputs, "scenario_inputs")?.try_iter()? { let scenario_input: Bound<'py, PyAny> = scenario_input?; - let scenario_file: Bound<'py, PyAny> = scenario_input.getattr("scenario_file")?; - let name: Bound<'py, PyAny> = scenario_file.getattr("name")?; + let scenario_file: Bound<'py, PyAny> = native_field(&scenario_input, "scenario_file")?; + let name: Bound<'py, PyAny> = native_field(&scenario_file, "name")?; let key: Bound<'py, PyAny> = call_with( &key_class, &[ @@ -166,8 +170,14 @@ pub(crate) fn compiled_sql_scenarios<'py>( for field in SCENARIO_FIELDS { fields.set_item(field, scenario_input.getattr(field)?)?; } - fields.set_item("source_path", scenario_file.getattr("relative_path")?)?; - fields.set_item("ownership_root", scenario_file.getattr("ownership_root")?)?; + fields.set_item( + "source_path", + native_field(&scenario_file, "relative_path")?, + )?; + fields.set_item( + "ownership_root", + native_field(&scenario_file, "ownership_root")?, + )?; scenarios.push(scenario_class.call((), Some(&fields))?); } PyTuple::new(py, scenarios) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index 7f00d931ba..098a62cf58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -2,15 +2,20 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple, PyTupleMethods}; +use crate::bindings::_helpers::analysis_session::stage_functions::dynamic_contract_proofs; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::stage_work::{ + PIVOT_PROOFS, SESSION_PIVOT_PROOFS, count_stage, +}; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_models; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -22,8 +27,8 @@ pub(crate) fn native_project_facts<'py>( session: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let model_inputs: Bound<'py, PyAny> = inputs.getattr("model_inputs")?; - let dialect: Bound<'py, PyAny> = context.profile.getattr("sql_analysis_dialect")?; + let model_inputs: Bound<'py, PyAny> = native_field(inputs, "model_inputs")?; + let dialect: Bound<'py, PyAny> = native_field(&context.profile, "sql_analysis_dialect")?; let (succeeded, positions): (Bound<'py, PyAny>, Bound<'py, PyAny>) = match &context.analyses { Some(analyses) => ( analyses.succeeded.clone().into_any(), @@ -50,8 +55,8 @@ pub(crate) fn native_project_facts<'py>( context, session, )?; - let deps: Bound<'py, PyAny> = resources.getattr("model_deps")?; - let syntax_valid: Bound<'py, PyAny> = resources.getattr("model_syntax_valid")?; + let deps: Bound<'py, PyAny> = native_field(&resources, "model_deps")?; + let syntax_valid: Bound<'py, PyAny> = native_field(&resources, "model_syntax_valid")?; let mut models: Vec> = Vec::new(); for index in 0..model_inputs.len()? { let proof: Bound<'py, PyAny> = match proofs.get_item(index)? { @@ -80,17 +85,16 @@ pub(crate) fn native_project_facts<'py>( .into_any()) } -/// Each standalone pivot model's dynamic contract proof, by model position. -fn pivot_proofs<'py>( - pivots: &Bound<'py, PyDict>, - context: &AnalysisContext<'py>, - session: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = pivots.py(); - let tables: Bound<'py, PyAny> = call_with( +/// The declared relation facts standalone pivot proofs read. +pub(crate) fn pivot_tables<'py>(context: &AnalysisContext<'py>) -> PyResult> { + let py: Python<'py> = context.profile.py(); + call_with( ®istry::class(py, RecordId::NativePivotTables)?, &[ - ("dialect", context.profile.getattr("sql_analysis_dialect")?), + ( + "dialect", + native_field(&context.profile, "sql_analysis_dialect")?, + ), ("column_types_by_table", context.column_types.clone()), ( "authoritative_column_types_by_table", @@ -102,17 +106,29 @@ fn pivot_proofs<'py>( context.dynamic_families.clone(), ), ], - )?; - let proven: Vec> = call_with( - ®istry::native(py, NativeAttr::ProveNativeDynamicContracts)?, - &[ - ("session", session.clone()), - ("tables", tables), - ("models", PyTuple::new(py, pivots.values())?.into_any()), - ], + ) +} + +/// Each standalone pivot model's dynamic contract proof, by model position. +fn pivot_proofs<'py>( + pivots: &Bound<'py, PyDict>, + context: &AnalysisContext<'py>, + session: &Bound<'py, PyAny>, +) -> PyResult> { + let tables: Bound<'py, PyAny> = pivot_tables(context)?; + let py: Python<'py> = pivots.py(); + let proven: Vec> = dynamic_contract_proofs( + session, + &tables, + &PyTuple::new(py, pivots.values())?.into_any(), )? - .try_iter()? - .collect::>()?; + .iter() + .collect(); + count_stage(py, PIVOT_PROOFS, proven.len() as u64)?; + if !session.is_none() { + let session_proofs: usize = proven.iter().filter(|proof| !proof.is_none()).count(); + count_stage(py, SESSION_PIVOT_PROOFS, session_proofs as u64)?; + } if proven.len() != pivots.len() { return Err(PyValueError::new_err( "native pivot proofs do not match the standalone pivot models", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index c89ed3ab49..51f6e19b4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -1,32 +1,33 @@ //! Each compiled model of an assembled project, with its syntax checks and analysis fallback. use pyo3::IntoPyObject; +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSetMethods, PyList, PyString, PyTuple}; use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; +use crate::bindings::_helpers::analysis_session::query_columns::ColumnsRow; +use crate::bindings::_helpers::analysis_session::stage_functions::query_column_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, model_stem, selected_for_analysis, }; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; use crate::bindings::_helpers::hand_built::compiled_model::compiled_model as hand_built_model; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::project_assembly::syntax_checks::model_placeholders; +use crate::bindings::_helpers::project_assembly::syntax_validation::{ + ParseOptions, validate_hook_sql_syntax, validate_sql_syntax, +}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::macro_calls::top_level_macro_call_names; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; -/// The Python stages each model's assembly calls, looked up once per project. -struct ModelStages<'py> { - placeholders: Bound<'py, PyAny>, - validate_hooks: Bound<'py, PyAny>, - validate_sql: Bound<'py, PyAny>, - native_input: Bound<'py, PyAny>, -} - /// One model's assembly inputs: the attached model, its native facts and its two gates. struct ModelRequest<'a, 'py> { model_input: &'a Bound<'py, PyAny>, @@ -42,18 +43,15 @@ pub(crate) fn compiled_models<'py>( (native_models, names): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = inputs.py(); - let stages: ModelStages<'py> = ModelStages { - placeholders: registry::native(py, NativeAttr::ModelPlaceholders)?, - validate_hooks: registry::native(py, NativeAttr::ValidateHookSqlSyntax)?, - validate_sql: registry::native(py, NativeAttr::ValidateSqlSyntax)?, - native_input: registry::native(py, NativeAttr::NativeModelInput)?, - }; let analysis_enabled: bool = context.sql_analysis_enabled.is_truthy()?; let mut models: Vec> = Vec::new(); - for (index, model_input) in inputs.getattr("model_inputs")?.try_iter()?.enumerate() { + for (index, model_input) in native_field(inputs, "model_inputs")? + .try_iter()? + .enumerate() + { let model_input: Bound<'py, PyAny> = model_input?; let sql_analysis_enabled: bool = analysis_enabled - && model_input.getattr("sql_validation_enabled")?.is_truthy()? + && native_field(&model_input, "sql_validation_enabled")?.is_truthy()? && selected_for_analysis(&model_input, names)?; let request: ModelRequest<'_, 'py> = ModelRequest { model_input: &model_input, @@ -61,56 +59,70 @@ pub(crate) fn compiled_models<'py>( sql_validation_enabled: selected_for_analysis(&model_input, names)?, native_model: native_models.get_item(index)?, }; - models.push(compiled_model(&stages, context, &request)?); + models.push(compiled_model(context, &request)?); } PyTuple::new(py, models) } /// `_assemble_compiled_model`. fn compiled_model<'py>( - stages: &ModelStages<'py>, context: &AnalysisContext<'py>, request: &ModelRequest<'_, 'py>, ) -> PyResult> { let model_input: &Bound<'py, PyAny> = request.model_input; let py: Python<'py> = model_input.py(); let model_name: Bound<'py, PyAny> = model_stem(model_input)?; - let syntax_validated: bool = request.native_model.getattr("syntax_valid")?.is_truthy()?; - let config_values: Bound<'py, PyAny> = model_input.getattr("config")?.getattr("values")?; + let syntax_validated: bool = + native_field(&request.native_model, "syntax_valid")?.is_truthy()?; + let config_values: Bound<'py, PyAny> = + native_field(model_input, "config")?.getattr("values")?; let analysis_query_sql: Bound<'py, PyAny> = PyString::new( py, &analysis_cursor_sql( py, - &model_input.getattr("query_sql")?.extract::()?, + &native_field(model_input, "query_sql")?.extract::()?, &config_values.call_method1("get", ("cursor_type",))?, )?, ) .into_any(); - let placeholders: Bound<'py, PyAny> = stages.placeholders.call1((model_input,))?; - let input_validated: bool = model_input.getattr("sql_validation_enabled")?.is_truthy()?; - let location: Bound<'py, PyTuple> = PyTuple::new( - py, - [ - model_name.clone(), - model_input.getattr("model_file")?.getattr("file_path")?, - ], - )?; - let syntax: Bound<'py, PyTuple> = PyTuple::new( - py, - [ - placeholders.clone(), - context.profile.getattr("sql_analysis_dialect")?, - ], - )?; + let placeholder_values: Option> = model_placeholders(model_input)?; + let placeholders: Bound<'py, PyAny> = match &placeholder_values { + Some(values) => values.clone().into_any(), + None => py.None().into_bound(py), + }; + let input_validated: bool = native_field(model_input, "sql_validation_enabled")?.is_truthy()?; + let location = || -> PyResult<(String, Bound<'py, PyAny>)> { + Ok(( + model_name.extract()?, + native_field(model_input, "model_file")?.getattr("file_path")?, + )) + }; + let syntax = || -> PyResult { + Ok(( + placeholder_values + .as_ref() + .map(|values| values.extract()) + .transpose()?, + native_field(&context.profile, "sql_analysis_dialect")?.extract()?, + )) + }; + let validate_sql = || -> PyResult<()> { + validate_sql_syntax( + py, + &analysis_query_sql.extract::()?, + location()?, + syntax()?, + ) + }; let validates: bool = input_validated && !syntax_validated; if request.sql_validation_enabled && validates { for hook_name in HOOK_NAMES { - stages.validate_hooks.call1(( + validate_hook_sql_syntax( config_values.call_method1("get", (hook_name,))?, hook_name, - &location, - &syntax, - ))?; + location()?, + syntax()?, + )?; } } let mut session_analysis: Bound<'py, PyAny> = py.None().into_bound(py); @@ -137,18 +149,14 @@ fn compiled_model<'py>( session_analysis = PyTuple::new(py, [session, position])?.into_any(); if !analyses.succeeded.contains(&model_name)? { if validates { - stages - .validate_sql - .call1((&analysis_query_sql, &location, &syntax))?; + validate_sql()?; } fallback = fallback_columns(context, &analysis_query_sql, &placeholders)?; } } else if request.sql_validation_enabled && validates { - stages - .validate_sql - .call1((&analysis_query_sql, &location, &syntax))?; + validate_sql()?; } - let declared_macro_deps: Bound<'py, PyAny> = model_input.getattr("macro_deps")?; + let declared_macro_deps: Bound<'py, PyAny> = native_field(model_input, "macro_deps")?; let macro_deps: Bound<'py, PyAny> = if declared_macro_deps.is_truthy()? { declared_macro_deps } else { @@ -161,16 +169,16 @@ fn compiled_model<'py>( .into_any() }; let native_model: &Bound<'py, PyAny> = &request.native_model; - if model_input.is_instance(&stages.native_input)? { + if model_input.is_instance_of::() { let macro_list: Bound<'py, PyList> = PyList::new(py, macro_deps.try_iter()?.collect::>>()?)?; return model_input.call_method1( "compiled_model", (( - native_model.getattr("deps")?, + native_field(native_model, "deps")?, session_analysis, fallback, - native_model.getattr("dynamic_contract")?, + native_field(native_model, "dynamic_contract")?, macro_list, ),), ); @@ -182,13 +190,13 @@ fn compiled_model<'py>( ("fallback", fallback), ( "standalone_proof", - native_model.getattr("dynamic_contract")?, + native_field(native_model, "dynamic_contract")?, ), ], )?; hand_built_model( model_input, - &native_model.getattr("deps")?, + &native_field(native_model, "deps")?, &analysis, ¯o_deps, ) @@ -209,16 +217,21 @@ fn fallback_columns<'py>( ("placeholders", placeholders.clone()), ], )?; - call_with( - ®istry::native(py, NativeAttr::InferNativeQueryColumnRows)?, - &[ - ("queries", PyTuple::new(py, [query])?.into_any()), - ("profile", context.profile.clone()), - ("column_nullability_by_table", context.nullability.clone()), - ], - )? - .get_item(0)? - .get_item(1) + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let nullability: Bound<'py, PyAny> = if context.nullability.is_truthy()? { + context.nullability.clone() + } else { + empty.clone() + }; + let rows: Vec = query_column_rows( + &PyTuple::new(py, [query])?.into_any(), + &context.profile, + (&empty, &nullability), + )?; + rows.into_iter() + .next() + .map(|(_, columns, _)| columns) + .into_bound_py_any(py) } /// `_project_binding_diagnostics`: every analysed model's located binding diagnostics. @@ -232,8 +245,8 @@ pub(crate) fn project_binding_diagnostics<'py>( }; for model in models.as_any().try_iter()? { let model: Bound<'py, PyAny> = model?; - if analyses.positions.contains(model.getattr("name")?)? { - for diagnostic in model.getattr("binding_diagnostics")?.try_iter()? { + if analyses.positions.contains(native_field(&model, "name")?)? { + for diagnostic in native_field(&model, "binding_diagnostics")?.try_iter()? { diagnostics.push(diagnostic?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index a86956db51..6e01d7a4de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -5,25 +5,35 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyStringMethods, PyTu use sqlbuild_analysis::assembly::project::main::configured_namespace::configured_namespace; -use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; +use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ AnalysisContext, analysis_context, and_not, }; use crate::bindings::_helpers::compile_driver::assembly_checks::compiled_sql_scenarios; -use crate::bindings::_helpers::compile_driver::assembly_facts::native_project_facts; +use crate::bindings::_helpers::compile_driver::assembly_facts::{ + native_project_facts, pivot_tables, +}; use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::stage_work::{SEMANTIC_COMPLETION, count_stage}; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::contracts::spec_resolution::{ + resolve_effective_adapter_name, resolve_effective_scenario_config, +}; use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; +use crate::bindings::_helpers::semantic_checks::stage::{ + SemanticCompletion, completed_semantic_project, +}; use pyo3::IntoPyObjectExt; const SOURCE_SLOT: &str = "source"; @@ -84,13 +94,23 @@ pub(crate) fn assembled_project<'py>( || py.None().into_bound(py), |session| session.clone().into_any(), ); + request.before_stage(AssemblyStage::ModelFacts, || { + keywords( + py, + &[ + ("profile", context.profile.clone()), + ("session", native_session.clone()), + ("tables", pivot_tables(&context)?), + ], + ) + })?; let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; - let resources: Bound<'py, PyAny> = native.getattr("resources")?; + let resources: Bound<'py, PyAny> = native_field(&native, "resources")?; let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; report_scope_index_errors(&scope_index)?; let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; let models: Bound<'py, PyTuple> = - compiled_models(inputs, &context, (&native.getattr("models")?, names))?; + compiled_models(inputs, &context, (&native_field(&native, "models")?, names))?; let fields: Bound<'py, PyDict> = project_fields( inputs, &context, @@ -99,22 +119,30 @@ pub(crate) fn assembled_project<'py>( )?; let project: Bound<'py, PyAny> = project_view(inputs, &fields, &models)?; let binding_results: Bound<'py, PyDict> = binding_rows(py, context.analyses.as_ref())?; - call_with( - ®istry::native(py, NativeAttr::CompleteSemanticProject)?, - &[ - ("project", project), - ("profile", context.profile.clone()), - ( - "resource_sql_analysis", - and_not( - &context.sql_analysis_enabled, - &inputs.getattr("no_sql_validation")?, - )?, - ), - ("session", native_session), - ("binding_results", binding_results.into_any()), - ], - ) + let resource_sql_analysis: Bound<'py, PyAny> = and_not( + &context.sql_analysis_enabled, + &native_field(inputs, "no_sql_validation")?, + )?; + request.before_stage(AssemblyStage::SemanticCompletion, || { + keywords( + py, + &[ + ("project", project.clone()), + ("profile", context.profile.clone()), + ("resource_sql_analysis", resource_sql_analysis.clone()), + ("session", native_session.clone()), + ("binding_results", binding_results.clone().into_any()), + ], + ) + })?; + count_stage(py, SEMANTIC_COMPLETION, 1)?; + completed_semantic_project(SemanticCompletion { + project, + profile: context.profile.clone(), + binding_results: binding_results.into_any(), + resource_sql_analysis: resource_sql_analysis.extract()?, + session: Some(native_session), + }) } /// `value.strip()` when `value` is a string with non-blank text, else `None`. @@ -130,7 +158,7 @@ fn str_or_none<'py>(value: &Bound<'py, PyAny>) -> PyResult> { /// The selected target's `field`, or `None` without a selected target. fn target_field<'py>(inputs: &Bound<'py, PyAny>, field: &str) -> PyResult> { - let target: Bound<'py, PyAny> = inputs.getattr("effective_target")?; + let target: Bound<'py, PyAny> = native_field(inputs, "effective_target")?; if target.is_none() { return Ok(target); } @@ -140,7 +168,7 @@ fn target_field<'py>(inputs: &Bound<'py, PyAny>, field: &str) -> PyResult(inputs: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = inputs.py(); - let connection: Bound<'py, PyAny> = inputs.getattr("effective_connection")?; + let connection: Bound<'py, PyAny> = native_field(inputs, "effective_connection")?; let database: Bound<'py, PyAny> = target_field(inputs, "database")?; let database: Bound<'py, PyAny> = if database.is_truthy()? { database @@ -155,10 +183,10 @@ fn effective_target_values<'py>(inputs: &Bound<'py, PyAny>) -> PyResult( ) -> PyResult> { let py: Python<'py> = inputs.py(); let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( - &inputs.getattr("project_config")?, - &inputs.getattr("local_config")?, + &native_field(inputs, "project_config")?, + &native_field(inputs, "local_config")?, ) .and_then(|value| value.into_bound_py_any(py))?; if adapter_name.eq(host::entry(py, AdapterHost::BuiltinAdapter)?.getattr("DUCKDB")?)? { @@ -192,25 +220,25 @@ fn project_fields<'py>( target_values: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let discovered: Bound<'py, PyAny> = inputs.getattr("discovered_inputs")?; - let project_config: Bound<'py, PyAny> = inputs.getattr("project_config")?; + let discovered: Bound<'py, PyAny> = native_field(inputs, "discovered_inputs")?; + let project_config: Bound<'py, PyAny> = native_field(inputs, "project_config")?; let fields: Bound<'py, PyDict> = keywords( py, &[ ( "binding_catalog", - context.profile.getattr("binding_catalog")?, + native_field(&context.profile, "binding_catalog")?, ), - ("run_id", inputs.getattr("run_id")?), + ("run_id", native_field(inputs, "run_id")?), ( "effective_target_name", - inputs.getattr("effective_target_name")?, + native_field(inputs, "effective_target_name")?, ), ( "effective_connection", - inputs.getattr("effective_connection")?, + native_field(inputs, "effective_connection")?, ), - ("effective_vars", inputs.getattr("effective_vars")?), + ("effective_vars", native_field(inputs, "effective_vars")?), ( "effective_target_database", str_or_none(&target_values.call_method1("get", ("database",))?)?, @@ -221,19 +249,22 @@ fn project_fields<'py>( ), ( "sql_analysis_dialect", - context.profile.getattr("sql_analysis_dialect")?, + native_field(&context.profile, "sql_analysis_dialect")?, + ), + ( + "sql_lexical_syntax", + native_field(inputs, "sql_lexical_syntax")?, + ), + ( + "compile_cache_dir", + native_field(inputs, "compile_cache_dir")?, ), - ("sql_lexical_syntax", inputs.getattr("sql_lexical_syntax")?), - ("compile_cache_dir", inputs.getattr("compile_cache_dir")?), - ("settings", inputs.getattr("effective_settings")?), + ("settings", native_field(inputs, "effective_settings")?), ( "scenario", - call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveScenarioConfig)?, - &[ - ("project_config", project_config.clone()), - ("local_config", inputs.getattr("local_config")?), - ], + resolve_effective_scenario_config( + &project_config, + &native_field(inputs, "local_config")?, )?, ), ("models", models.clone().into_any()), @@ -275,7 +306,7 @@ fn project_fields<'py>( fields.set_item("diagnostics", PyTuple::new(py, diagnostics)?)?; fields.set_item( "external_sql_reference_resolver", - inputs.getattr("external_sql_reference_resolver")?, + native_field(inputs, "external_sql_reference_resolver")?, )?; fields.set_item("scope_index", scope_index)?; Ok(fields) @@ -292,7 +323,7 @@ fn compiled_resources<'py>( if count == 0 { return Ok(PyTuple::empty(py)); } - let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; + let owner: Bound<'py, PyAny> = native_field(inputs, "native_project")?; let facts: Bound<'py, PyAny> = resources.getattr(kind.facts)?; owner.call_method1( kind.record, @@ -317,12 +348,12 @@ fn native_resource<'py>( (slot, items): (&str, &Bound<'py, PyAny>), index: usize, ) -> PyResult> { - let failure: Bound<'py, PyAny> = resources.getattr("failure")?; + let failure: Bound<'py, PyAny> = native_field(resources, "failure")?; if !failure.is_none() - && failure.getattr("slot")?.eq(slot)? - && failure.getattr("index")?.eq(index)? + && native_field(&failure, "slot")?.eq(slot)? + && native_field(&failure, "index")?.eq(index)? { - return Err(PyErr::from_value(failure.getattr("error")?)); + return Err(PyErr::from_value(native_field(&failure, "error")?)); } items.get_item(index) } @@ -334,11 +365,10 @@ fn project_view<'py>( models: &Bound<'py, PyTuple>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let owner: Bound<'py, PyAny> = inputs.getattr("native_project")?; - let native_model: Bound<'py, PyAny> = registry::native(py, NativeAttr::NativeCompiledModel)?; + let owner: Bound<'py, PyAny> = native_field(inputs, "native_project")?; let mut all_native: bool = true; for model in models.as_any().try_iter()? { - if !model?.is_instance(&native_model)? { + if !model?.is_instance_of::() { all_native = false; break; } @@ -350,12 +380,12 @@ fn project_view<'py>( return owner.call_method1("project_view", (view_fields,)); } let sql_expansions: Bound<'py, PyDict> = PyDict::new(py); - for model_input in inputs.getattr("model_inputs")?.try_iter()? { + for model_input in native_field(inputs, "model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; - let expansion: Bound<'py, PyAny> = model_input.getattr("sql_expansion")?; + let expansion: Bound<'py, PyAny> = native_field(&model_input, "sql_expansion")?; if !expansion.is_none() { sql_expansions.set_item( - model_input.getattr("model_file")?.getattr("file_path")?, + native_field(&model_input, "model_file")?.getattr("file_path")?, expansion, )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs index 9e768fec7e..0fcb82dafc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs @@ -1,13 +1,17 @@ //! Every SQL test assembled natively; macro-mocked tests' model queries run the user's macros. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyList, PyTuple}; +use pyo3::types::{PyList, PyTuple, PyTupleMethods}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_driver::assembly_checks::test_model_query_overrides; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::diagnostic_collector::report_compile_diagnostic; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compile_driver::stage_work::{SQL_TEST_ASSEMBLY, count_stage}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::compiled_tests::assemble_native_sql_tests; const DECIMAL_MODULE: &str = "decimal"; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -19,12 +23,9 @@ pub(crate) fn assembled_sql_tests<'py>( inputs: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let assemblies: Vec> = call_with( - ®istry::native(py, NativeAttr::AssembleNativeSqlTests)?, - &[("inputs", inputs.clone())], - )? - .try_iter()? - .collect::>()?; + count_stage(py, SQL_TEST_ASSEMBLY, 1)?; + let assemblies: Vec> = + assemble_native_sql_tests(py, inputs)?.iter().collect(); let test_inputs: Vec> = inputs .getattr("test_inputs")? .try_iter()? @@ -34,27 +35,28 @@ pub(crate) fn assembled_sql_tests<'py>( "native SQL test assembly returned a different number of tests", )); } - let report: Bound<'py, PyAny> = registry::native(py, NativeAttr::ReportCompileDiagnostic)?; let mocked_input: Bound<'py, PyAny> = registry::class(py, RecordId::CompileModelSqlTestInputPayload)?; let mut tests: Vec> = Vec::with_capacity(assemblies.len()); for (test_input, assembly) in test_inputs.iter().zip(assemblies) { - let failure: Bound<'py, PyAny> = assembly.getattr("failure")?; + let failure: Bound<'py, PyAny> = native_field(&assembly, "failure")?; let kind: Option = if failure.is_none() { None } else { - Some(failure.getattr("kind")?.extract()?) + Some(native_field(&failure, "kind")?.extract()?) }; - let mut test: Bound<'py, PyAny> = assembly.getattr("test")?; + let mut test: Bound<'py, PyAny> = native_field(&assembly, "test")?; if test.is_none() { return Err(assembly_error(py, &failure, kind.as_deref())?); } - for item in assembly.getattr("diagnostics")?.try_iter()? { + for item in native_field(&assembly, "diagnostics")?.try_iter()? { let (key, diagnostic): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - call_with(&report, &[("key", key), ("diagnostic", diagnostic)])?; + report_compile_diagnostic(&key, &diagnostic)?; } - let payload: Bound<'py, PyAny> = test_input.getattr("payload")?; - if payload.is_instance(&mocked_input)? && payload.getattr("macro_mocks")?.is_truthy()? { + let payload: Bound<'py, PyAny> = native_field(test_input, "payload")?; + if payload.is_instance(&mocked_input)? + && native_field(&payload, "macro_mocks")?.is_truthy()? + { test = with_model_queries( &test, test_model_query_overrides(test_input, inputs)?.as_any(), @@ -79,7 +81,7 @@ fn assembly_error( let message: String = if failure.is_none() { String::new() } else { - failure.getattr("message")?.extract()? + native_field(failure, "message")?.extract()? }; if kind == Some(SQL_TEST_INPUT_FAILURE) { let error: Bound<'_, PyAny> = @@ -95,7 +97,7 @@ fn with_model_queries<'py>( queries: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = test.py(); - let payload: Bound<'py, PyAny> = test.getattr("payload")?; + let payload: Bound<'py, PyAny> = native_field(test, "payload")?; if !payload.is_instance(®istry::class(py, RecordId::CompiledModelSqlTestPayload)?)? { return Ok(test.clone()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 7e1e5c63e4..985f4ae027 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -10,6 +10,7 @@ use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::report::{ format_compile_json, format_compile_text, json_semantic_notice, }; +use crate::bindings::_helpers::compile_command::reuse_recording::write_reusable_compile; use crate::bindings::_helpers::compile_command::reuse_recording::{ CompileArtifactWrites, RecordedCompileOutput, compile_reuse_timings, new_artifact_recorder, }; @@ -17,6 +18,7 @@ use crate::bindings::_helpers::compile_driver::analysis::analyze; use crate::bindings::_helpers::compile_driver::compile_timings::{ CompileTimingsHandle, collect_compile_timings, timing_milliseconds, }; +use crate::bindings::_helpers::compile_driver::input_reads::record_compile_input_reads; use crate::bindings::_helpers::compile_driver::phases::{ elapsed_ms, manifest_payload, monotonic, resolved_dag_path, sql_test_planning_diagnostics, target_dir, write_artifacts, write_dag_artifact, @@ -25,6 +27,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ Status, call_with, imported, keywords, within, }; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; @@ -56,11 +59,11 @@ fn run_compile_command<'py>( reuse_attempt: Option>, ) -> PyResult { let total_start: f64 = match &reuse_attempt { - Some(attempt) => attempt.getattr("started")?.extract()?, + Some(attempt) => native_field(attempt, "started")?.extract()?, None => monotonic(py)?, }; let path_class: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?; - let effective_request: Bound<'py, PyAny> = if request.getattr("project_dir")?.is_none() { + let effective_request: Bound<'py, PyAny> = if native_field(&request, "project_dir")?.is_none() { imported(py, "dataclasses", "replace")?.call( (request.clone(),), Some(&keywords( @@ -98,7 +101,7 @@ fn bypass_attempt<'py>( started: f64, ) -> PyResult> { let path_class: Bound<'py, PyAny> = imported(py, "pathlib", "Path")?; - let project_dir: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = native_field(request, "project_dir")?; let project_dir: Bound<'py, PyAny> = if project_dir.is_truthy()? { project_dir } else { @@ -118,14 +121,14 @@ fn bypass_attempt<'py>( fn compile_status<'py>(request: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = request.py(); let none: Bound<'py, PyAny> = py.None().into_bound(py); - if request.getattr("json_output")?.is_truthy()? { + if native_field(request, "json_output")?.is_truthy()? { return Ok(none); } let stdout: Bound<'py, PyAny> = py.import("sys")?.getattr("stdout")?; if !stdout.hasattr("isatty")? || !stdout.call_method0("isatty")?.is_truthy()? { return Ok(none); } - let use_color: bool = !request.getattr("no_color")?.is_truthy()?; + let use_color: bool = !native_field(request, "no_color")?.is_truthy()?; status_reporter(&stdout, use_color) } @@ -151,7 +154,7 @@ fn recorded_compile<'py>( let compiled: PyResult<(CommandOutcome<'py>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = collect_compile_timings(|detailed_timings| { within( - ®istry::native(py, NativeAttr::RecordCompileInputReads)?.call0()?, + &Bound::new(py, record_compile_input_reads())?.into_any(), |input_reads| { let artifact_writes: Py = new_artifact_recorder(py)?; let manager: Bound<'py, PyAny> = call_with( @@ -192,29 +195,32 @@ fn recorded_compile<'py>( }); close_status(status)?; let (outcome, input_reads, artifact_writes) = compiled?; - let stored: PyResult> = call_with( - ®istry::native(py, NativeAttr::WriteReusableCompile)?, - &[ - ("attempt", attempt.clone()), - ("output", output.clone()), - ("exit_code", outcome.exit_code.into_pyobject(py)?.into_any()), - ("input_reads", input_reads), - ("artifact_writes", artifact_writes), - ( - "compile_cache_enabled", - PyBool::new(py, outcome.compile_cache_enabled) - .to_owned() - .into_any(), - ), - ( - "artifacts_written", - PyBool::new(py, outcome.artifacts_written) - .to_owned() - .into_any(), - ), - ("dag_artifact_path", outcome.dag_artifact_path), - ("json_output", request.getattr("json_output")?), - ], + let stored: PyResult<()> = write_reusable_compile( + py, + Some(keywords( + py, + &[ + ("attempt", attempt.clone()), + ("output", output.clone()), + ("exit_code", outcome.exit_code.into_pyobject(py)?.into_any()), + ("input_reads", input_reads), + ("artifact_writes", artifact_writes), + ( + "compile_cache_enabled", + PyBool::new(py, outcome.compile_cache_enabled) + .to_owned() + .into_any(), + ), + ( + "artifacts_written", + PyBool::new(py, outcome.artifacts_written) + .to_owned() + .into_any(), + ), + ("dag_artifact_path", outcome.dag_artifact_path), + ("json_output", native_field(request, "json_output")?), + ], + )?), ); let printed: PyResult> = output.call_method0("print_stdout"); printed?; @@ -229,7 +235,7 @@ fn compiled_with_status<'py>( (total_start, reuse_timings): (f64, &Bound<'py, PyAny>), ) -> PyResult> { let py: Python<'py> = request.py(); - let project_dir: Bound<'py, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = native_field(request, "project_dir")?; let project_dir: Bound<'py, PyAny> = if project_dir.is_none() { imported(py, "pathlib", "Path")?.call_method0("cwd")? } else { @@ -237,7 +243,7 @@ fn compiled_with_status<'py>( }; let prepared_artifacts: &Bound<'py, PyAny> = &recorders.prepared_artifacts; call_with( - &prepared_artifacts.getattr("remove_abandoned_staging")?, + &native_field(prepared_artifacts, "remove_abandoned_staging")?, &[("target_dir", target_dir(&project_dir)?)], )?; let analysis: Bound<'py, PyAny> = analyze( @@ -249,13 +255,13 @@ fn compiled_with_status<'py>( .getattr("profile_flags")? .getattr("skip_write")? .is_truthy()?; - let manifest_requested: bool = request.getattr("manifest")?.is_truthy()?; + let manifest_requested: bool = native_field(request, "manifest")?.is_truthy()?; let manifest: Bound<'py, PyAny> = manifest_payload(manifest_requested && !rules_failed, &analysis, status)?; let dag_path: Bound<'py, PyAny> = if rules_failed { py.None().into_bound(py) } else { - request.getattr("dag_path")? + native_field(request, "dag_path")? }; let recorder: Option<&Py> = Some(&recorders.artifact_writes); write_dag_artifact((&dag_path, &project_dir), (&analysis, recorder), status)?; @@ -280,28 +286,28 @@ fn compiled_with_status<'py>( analysis .getattr("diagnostics")? .try_iter()? - .chain(written.getattr("diagnostics")?.try_iter()?) + .chain(native_field(&written, "diagnostics")?.try_iter()?) .chain(withheld.try_iter()?) .collect::>>()?, )?; let mut exit_code: i64 = 0; for diagnostic in diagnostics.as_any().try_iter()? { - if diagnostic?.getattr("is_error")?.is_truthy()? { + if native_field(&diagnostic?, "is_error")?.is_truthy()? { exit_code = 1; break; } } close_status(status)?; - if let Some(skipped_note) = - format_skipped_type_proof_rules(analysis.getattr("skipped_type_proof_rules")?.extract()?) - { + if let Some(skipped_note) = format_skipped_type_proof_rules( + native_field(&analysis, "skipped_type_proof_rules")?.extract()?, + ) { output.call_method1("print_stderr_line", (format!("note: {skipped_note}"),))?; } let dag_artifact_path: Bound<'py, PyAny> = - if rules_failed || request.getattr("dag_path")?.is_none() { + if rules_failed || native_field(request, "dag_path")?.is_none() { py.None().into_bound(py) } else { - resolved_dag_path(&project_dir, &request.getattr("dag_path")?)? + resolved_dag_path(&project_dir, &native_field(request, "dag_path")?)? }; let outcome: CommandOutcome<'py> = CommandOutcome { exit_code, @@ -332,20 +338,23 @@ fn analysis_options<'py>( let selection: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompileAnalysisSelection)?, &[ - ("select", request.getattr("select")?), - ("exclude", request.getattr("exclude")?), - ("no_cache", request.getattr("no_cache")?), + ("select", native_field(request, "select")?), + ("exclude", native_field(request, "exclude")?), + ("no_cache", native_field(request, "no_cache")?), ], )?; keywords( py, &[ ("project_dir", project_dir.clone()), - ("no_sql_validation", request.getattr("no_sql_validation")?), - ("selected_target", request.getattr("selected_target")?), - ("lineage_mode", request.getattr("lineage_mode")?), - ("cli_vars", request.getattr("cli_vars")?), - ("profile_flags", request.getattr("profile_flags")?), + ( + "no_sql_validation", + native_field(request, "no_sql_validation")?, + ), + ("selected_target", native_field(request, "selected_target")?), + ("lineage_mode", native_field(request, "lineage_mode")?), + ("cli_vars", native_field(request, "cli_vars")?), + ("profile_flags", native_field(request, "profile_flags")?), ("analysis_selection", selection), ("status", status.0.clone()), ("prepared_artifacts", recorders.prepared_artifacts.clone()), @@ -358,10 +367,10 @@ fn rules_failed(analysis: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = analysis.py(); let rule_phase: Bound<'_, PyAny> = registry::class(py, RecordId::DiagnosticPhase)?.getattr("RULE")?; - for diagnostic in analysis.getattr("diagnostics")?.try_iter()? { + for diagnostic in native_field(analysis, "diagnostics")?.try_iter()? { let diagnostic: Bound<'_, PyAny> = diagnostic?; - if diagnostic.getattr("is_error")?.is_truthy()? - && diagnostic.getattr("phase")?.is(&rule_phase) + if native_field(&diagnostic, "is_error")?.is_truthy()? + && native_field(&diagnostic, "phase")?.is(&rule_phase) { return Ok(true); } @@ -412,23 +421,25 @@ fn record_output<'py>( output: &Bound<'py, PyAny>, ) -> PyResult<()> { let py: Python<'py> = request.py(); - let graph: Bound<'py, PyAny> = analysis.getattr("graph")?; - let selected_keys: Bound<'py, PyAny> = analysis.getattr("selected_keys")?; - let validation_enabled: Bound<'py, PyAny> = - PyBool::new(py, !request.getattr("no_sql_validation")?.is_truthy()?) - .to_owned() - .into_any(); + let graph: Bound<'py, PyAny> = native_field(analysis, "graph")?; + let selected_keys: Bound<'py, PyAny> = native_field(analysis, "selected_keys")?; + let validation_enabled: Bound<'py, PyAny> = PyBool::new( + py, + !native_field(request, "no_sql_validation")?.is_truthy()?, + ) + .to_owned() + .into_any(); let shared: [(&str, Bound<'py, PyAny>); 7] = [ ("graph", graph.clone()), ("written", written.clone()), - ("manifest", request.getattr("manifest")?), - ("lineage", analysis.getattr("lineage")?), - ("lineage_mode", request.getattr("lineage_mode")?), + ("manifest", native_field(request, "manifest")?), + ("lineage", native_field(analysis, "lineage")?), + ("lineage_mode", native_field(request, "lineage_mode")?), ("diagnostics", diagnostics.clone().into_any()), ("selected_keys", selected_keys.clone()), ]; let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = shared.to_vec(); - let rendered: Bound<'py, PyString> = if request.getattr("json_output")?.is_truthy()? { + let rendered: Bound<'py, PyString> = if native_field(request, "json_output")?.is_truthy()? { let notice: Option> = json_semantic_notice(&graph, &selected_keys, validation_enabled.is_truthy()?)?; if let Some(notice) = notice { @@ -442,7 +453,7 @@ fn record_output<'py>( ) } else { let use_color: bool = - !request.getattr("no_color")?.is_truthy()? && stdout_supports_color(py)?; + !native_field(request, "no_color")?.is_truthy()? && stdout_supports_color(py)?; arguments.push(( "use_color", PyBool::new(py, use_color).to_owned().into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index 1dd915b13f..641a7c4d1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -6,10 +6,16 @@ use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_ta use crate::bindings::_helpers::compile_driver::python_calls::call_with; -use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::contracts::project_config::{ + resolve_config_file_path, validate_local_sql_analysis, +}; +use crate::bindings::_helpers::contracts::spec_resolution::{ + resolve_effective_adapter_name, resolve_target_config, +}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; @@ -49,7 +55,7 @@ fn target_names(config: &Bound<'_, PyAny>) -> PyResult> { /// Reject a local SQL analysis opt-out the project forbids. fn reject_local_sql_analysis_opt_out(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); - let local_settings: Bound<'_, PyAny> = config.local_config.getattr("settings")?; + let local_settings: Bound<'_, PyAny> = native_field(config.local_config, "settings")?; let flags: Bound<'_, PyTuple> = PyTuple::new( py, [ @@ -65,18 +71,16 @@ fn reject_local_sql_analysis_opt_out(config: &ProjectConfiguration<'_, '_>) -> P local_settings.getattr(SQL_ANALYSIS_KEY)?, ], )?; - registry::native(py, NativeAttr::ValidateLocalSqlAnalysis)? - .call1((config.project_dir, flags))?; - Ok(()) + validate_local_sql_analysis(py, config.project_dir, flags.extract()?) } /// The first target, by name, that selects a command-group warehouse. fn first_configured_group(config: &Bound<'_, PyAny>) -> PyResult> { - let targets: Bound<'_, PyAny> = config.getattr("targets")?; + let targets: Bound<'_, PyAny> = native_field(config, "targets")?; let mut names: Vec = target_names(config)?; names.sort(); for name in names { - let warehouses: Bound<'_, PyAny> = targets.get_item(&name)?.getattr("warehouses")?; + let warehouses: Bound<'_, PyAny> = native_field(&targets.get_item(&name)?, "warehouses")?; for group in WAREHOUSE_GROUPS { let value: Bound<'_, PyAny> = warehouses.getattr(group)?; if !value.is_none() { @@ -109,8 +113,7 @@ fn reject_group( (target, group, value): (String, String, String), ) -> PyErr { let raised = (|| -> PyResult { - let file_path: String = registry::native(py, NativeAttr::ResolveConfigFilePath)? - .call1((project_dir, local))? + let file_path: String = resolve_config_file_path(py, project_dir, local)? .str()? .extract()?; let file_name: &str = if local { @@ -132,7 +135,7 @@ fn reject_group( fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) -> PyResult<()> { let py: Python<'_> = config.project_dir.py(); if let Some(group) = first_configured_group(config.project_config)? { - let adapter_name: Bound<'_, PyAny> = config.project_config.getattr("adapter")?; + let adapter_name: Bound<'_, PyAny> = native_field(config.project_config, "adapter")?; if !session_warehouse_support(py, &adapter_name, config.project_dir)? { return Err(reject_group( py, @@ -168,16 +171,13 @@ fn reject_invalid_target_connections(config: &ProjectConfiguration<'_, '_>) -> P names.extend(target_names(config.local_config)?); names.sort(); names.dedup(); - let resolve: Bound<'_, PyAny> = registry::native(py, NativeAttr::ResolveTargetConfig)?; let spec_error: Bound<'_, PyAny> = registry::error(py, ErrorId::SpecConfig)?; for name in names { - let resolved: PyResult> = call_with( - &resolve, - &[ - ("project_config", config.project_config.clone()), - ("local_config", config.local_config.clone()), - ("target_name", PyString::new(py, &name).into_any()), - ], + let resolved: PyResult> = resolve_target_config( + py, + config.project_config, + config.local_config, + PyString::new(py, &name).as_any(), ); let Err(error) = resolved else { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs index 21dc42cdd8..b71ccba1b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs @@ -11,6 +11,7 @@ use sqlbuild_model_config::templates::models::DeferredSegment; use crate::bindings::_helpers::authored_values::project_vars::render_project_var_text; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; @@ -72,8 +73,8 @@ fn deferred_locations<'py>( for resource in project.getattr(attribute)?.try_iter()? { let resource: Bound<'py, PyAny> = resource?; let location: Bound<'py, PyAny> = - deferred_location(&resource.getattr("destination")?, naming)?; - locations.set_item(resource.getattr("name")?, location)?; + deferred_location(&native_field(&resource, "destination")?, naming)?; + locations.set_item(native_field(&resource, "name")?, location)?; } } Ok(locations) @@ -85,15 +86,15 @@ fn deferred_location<'py>( naming: &DeferredNaming<'py>, ) -> PyResult> { let py: Python<'py> = target.py(); - let logical_schema: Bound<'py, PyAny> = target.getattr("logical_schema")?; - let logical_database: Bound<'py, PyAny> = target.getattr("logical_database")?; + let logical_schema: Bound<'py, PyAny> = native_field(target, "logical_schema")?; + let logical_database: Bound<'py, PyAny> = native_field(target, "logical_database")?; let mut schema: Bound<'py, PyAny> = target_field( - &naming.target_config.getattr("schema")?, + &native_field(&naming.target_config, "schema")?, &logical_schema, &naming.effective_vars, )?; let mut database: Bound<'py, PyAny> = target_field( - &naming.target_config.getattr("database")?, + &native_field(&naming.target_config, "database")?, &logical_database, &naming.effective_vars, )?; @@ -103,7 +104,7 @@ fn deferred_location<'py>( if database.is_none() { database = naming.default_database.clone(); } - let name: Bound<'py, PyAny> = target.getattr("name")?; + let name: Bound<'py, PyAny> = native_field(target, "name")?; let qualified_name: Bound<'py, PyAny> = call_with( &naming.render_qualified_name, &[ @@ -205,7 +206,8 @@ fn deferred_target_config<'py>( current_target_name: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = discovered.py(); - let targets: Bound<'py, PyAny> = discovered.getattr("project_config")?.getattr("targets")?; + let targets: Bound<'py, PyAny> = + native_field(discovered, "project_config")?.getattr("targets")?; let name: String = defer_to.str()?.to_cow()?.into_owned(); if !targets.contains(defer_to)? { return Err(planner_error( @@ -253,14 +255,14 @@ fn deferred_relations<'py>( let mut database: Bound<'py, PyAny> = py.None().into_bound(py); for location in locations.call_method0("values")?.try_iter()? { let location: Bound<'py, PyAny> = location?; - let schema: Bound<'py, PyAny> = location.getattr("schema")?; + let schema: Bound<'py, PyAny> = native_field(&location, "schema")?; if !schema.is_none() { let schema: String = schema.extract()?; if !schemas.contains(&schema) { schemas.push(schema); } } - let location_database: Bound<'py, PyAny> = location.getattr("database")?; + let location_database: Bound<'py, PyAny> = native_field(&location, "database")?; if !location_database.is_none() && database.is_none() { database = location_database; } @@ -271,7 +273,7 @@ fn deferred_relations<'py>( } schemas.sort(); let relations: Bound<'py, PyAny> = call_with( - &adapter.getattr("list_relations")?, + &native_field(adapter, "list_relations")?, &[ ("connection", connection.clone()), ("database", database), @@ -280,7 +282,7 @@ fn deferred_relations<'py>( )?; for relation in relations.try_iter()? { let relation: Bound<'py, PyAny> = relation?; - relations_by_name.set_item(relation.getattr("name")?, relation)?; + relations_by_name.set_item(native_field(&relation, "name")?, relation)?; } Ok(relations_by_name) } @@ -317,16 +319,16 @@ fn resolve_model_deferral_inputs<'py>( let target_config: Bound<'py, PyAny> = deferred_target_config( &arguments.value("discovered_inputs")?, &defer_to, - &project.getattr("effective_target_name")?, + &native_field(&project, "effective_target_name")?, )?; let locations: Bound<'py, PyDict> = deferred_locations( &project, &DeferredNaming { target_config, - effective_vars: project.getattr("effective_vars")?, + effective_vars: native_field(&project, "effective_vars")?, default_schema: adapter.call_method0("default_schema")?, default_database: adapter.call_method0("default_database")?, - render_qualified_name: adapter.getattr("render_qualified_name")?, + render_qualified_name: native_field(&adapter, "render_qualified_name")?, }, )?; let relations: Bound<'py, PyDict> = deferred_relations( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs index b5fbf09e5b..a4ce5df811 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diagnostic_collector.rs @@ -8,6 +8,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{PyErr, pyclass, pyfunction, pymethods, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{context_variable, imported}; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; @@ -105,13 +106,15 @@ pub(crate) fn report_compile_diagnostic( if let Ok(collector) = collector.cast::() { return collector.get().add(py, key, diagnostic); } - let location: Bound<'_, PyAny> = diagnostic.getattr("location")?; - let path: Bound<'_, PyAny> = diagnostic.getattr("path")?; + let location: Bound<'_, PyAny> = native_field(diagnostic, "location")?; + let path: Bound<'_, PyAny> = native_field(diagnostic, "path")?; let pointer: String = if !location.is_none() { format!( "\n --> {}:{}", - location.getattr("path")?.call_method0("as_posix")?.str()?, - location.getattr("line")?.str()? + native_field(&location, "path")? + .call_method0("as_posix")? + .str()?, + native_field(&location, "line")?.str()? ) } else if !path.is_none() { format!("\n --> {}", path.call_method0("as_posix")?.str()?) @@ -119,9 +122,9 @@ pub(crate) fn report_compile_diagnostic( String::new() }; let keywords: Bound<'_, PyDict> = PyDict::new(py); - keywords.set_item("code", diagnostic.getattr("code")?)?; - keywords.set_item("help", diagnostic.getattr("help")?)?; - let message: String = format!("{}{pointer}", diagnostic.getattr("message")?.str()?); + keywords.set_item("code", native_field(diagnostic, "code")?)?; + keywords.set_item("help", native_field(diagnostic, "help")?)?; + let message: String = format!("{}{pointer}", native_field(diagnostic, "message")?.str()?); Err(PyErr::from_value( registry::error(py, ErrorId::CompileInput)?.call((message,), Some(&keywords))?, )) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs index f01496a11d..533040b9f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs @@ -14,15 +14,13 @@ use sqlbuild_analysis::diff_selection::models::DiffGraph; use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::project::build_compiled_project; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, imported, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::graph::views::project_lineage_views; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - /// Indexes of the `CompiledObjectKey`s a diff graph names, assigned on first sight. struct KeyIndexes<'py> { indexes: Bound<'py, PyDict>, @@ -67,15 +65,16 @@ fn diff_graph<'py>(project: &Bound<'py, PyAny>, keys: &KeyIndexes<'py>) -> PyRes for attribute in ["models", "sources", "seeds"] { for resource in project.getattr(attribute)?.try_iter()? { let resource: Bound<'py, PyAny> = resource?; - let name: String = resource.getattr("name")?.extract()?; + let name: String = native_field(&resource, "name")?.extract()?; graph .names - .push((name, keys.index(&resource.getattr("key")?)?)); + .push((name, keys.index(&native_field(&resource, "key")?)?)); } } - let upstream: Vec<(usize, Vec)> = adjacency(&lineage.getattr("upstream_deps")?, keys)?; + let upstream: Vec<(usize, Vec)> = + adjacency(&native_field(&lineage, "upstream_deps")?, keys)?; let downstream: Vec<(usize, Vec)> = - adjacency(&lineage.getattr("downstream_deps")?, keys)?; + adjacency(&native_field(&lineage, "downstream_deps")?, keys)?; for item in lineage .getattr("tag_index")? .call_method0("items")? @@ -163,16 +162,16 @@ fn diff_model_names<'py>( let selected: HashSet = resolve_diff_selection(&graph, (&select, &exclude)) .map_err(|error| selector_error(py, error))?; let model_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut names: Vec> = Vec::new(); - for model in project.getattr("models")?.try_iter()? { + for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; let key: Bound<'py, PyAny> = model_reads::model_key(&model)?; let chosen: bool = match keys.indexes.get_item(&key)? { Some(index) => selected.contains(&index.extract::()?), None => false, }; - if chosen && key.getattr("resource_type")?.eq(&model_type)? { + if chosen && native_field(&key, "resource_type")?.eq(&model_type)? { names.push(model_reads::name_object(&model)?); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 256315d1ea..92e7aeb24d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -11,12 +11,21 @@ use crate::bindings::_helpers::compile_driver::integration_loaders::integration_ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, keywords, within, }; +use crate::bindings::_helpers::compile_driver::stage_work::{PROJECT_DISCOVERIES, count_stage}; +use crate::bindings::_helpers::contracts::project_config::{ + load_local_config_file, load_project_config_file, +}; use crate::bindings::_helpers::discovery::declaration_records::{ discover_adapter_file, discover_audit_files, discover_constant_files, discover_enum_files, discover_macro_files, discover_model_schema_files, discover_seed_files, discover_sql_function_files, discover_sql_hook_files, retained_discovery_session, }; -use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::discovery::imported_providers::{ + discover_event_exporter_functions, discover_provider_classes, +}; +use crate::bindings::_helpers::discovery::imported_python_nodes::{ + discover_hook_functions, discover_materialization_files, discover_python_node_functions, +}; use crate::bindings::_helpers::discovery::model_records::{ ModelFileSelection, discovered_model_files, }; @@ -34,7 +43,8 @@ use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, produ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -105,19 +115,19 @@ const FILE_DISCOVERERS: [(&str, FileDiscoverer); 14] = [ const PYTHON_DISCOVERERS: [(&str, Discoverer); 5] = [ ( "python_nodes", - Discoverer::Native(NativeAttr::DiscoverPythonNodeFunctions), + Discoverer::Native(discover_python_node_functions), ), ( "materialization_files", - Discoverer::Native(NativeAttr::DiscoverMaterializationFiles), + Discoverer::Native(discover_materialization_files), ), ( "hook_functions", - Discoverer::Native(NativeAttr::DiscoverHookFunctions), + Discoverer::Native(discover_hook_functions), ), ( "event_exporters", - Discoverer::Native(NativeAttr::DiscoverEventExporterFunctions), + Discoverer::Native(discover_event_exporter_functions), ), ( "command_output_sinks", @@ -125,18 +135,32 @@ const PYTHON_DISCOVERERS: [(&str, Discoverer); 5] = [ ), ]; +/// A native discoverer of project Python that takes `project_dir` and `providers`. +type PythonDiscoverer = + for<'py> fn(Bound<'py, PyAny>, Option>) -> PyResult>; + /// Where a project-Python discoverer lives. #[derive(Clone, Copy)] enum Discoverer { - Native(NativeAttr), + Native(PythonDiscoverer), Host(UserCodeHost), } impl Discoverer { - fn function<'py>(self, py: Python<'py>) -> PyResult> { + fn discovered<'py>( + self, + project_dir: &Bound<'py, PyAny>, + providers: &Bound<'py, PyAny>, + ) -> PyResult> { match self { - Self::Native(id) => registry::native(py, id), - Self::Host(entry) => host::entry(py, entry), + Self::Native(discover) => discover(project_dir.clone(), Some(providers.clone())), + Self::Host(entry) => call_with( + &host::entry(project_dir.py(), entry)?, + &[ + ("project_dir", project_dir.clone()), + ("providers", providers.clone()), + ], + ), } } } @@ -181,6 +205,7 @@ pub(crate) fn discover_project_inputs<'py>( ], ), )?; + count_stage(py, PROJECT_DISCOVERIES, 1)?; let request: DiscoveryRequest = DiscoveryRequest { project_dir: arguments.value("project_dir")?.unbind(), sql_analysis_enabled_override: arguments.value("sql_analysis_enabled_override")?.unbind(), @@ -203,9 +228,8 @@ fn assembled_inputs<'py>( ) -> PyResult> { let project_dir: &Bound<'py, PyAny> = request.project_dir.bind(py); let project_config: Bound<'py, PyAny> = - registry::native(py, NativeAttr::LoadProjectConfigFile)?.call1((project_dir,))?; - let local_config: Bound<'py, PyAny> = - registry::native(py, NativeAttr::LoadLocalConfigFile)?.call1((project_dir,))?; + load_project_config_file(py, project_dir)?.into_bound(py); + let local_config: Bound<'py, PyAny> = load_local_config_file(py, project_dir)?.into_bound(py); reject_invalid_project_configuration(&ProjectConfiguration { project_dir, project_config: &project_config, @@ -219,7 +243,7 @@ fn assembled_inputs<'py>( .getattr("setting_overrides")? .contains(SQL_ANALYSIS_SETTING_KEY)? { - local_config.getattr("settings")?.getattr("sql_analysis")? + native_field(&local_config, "settings")?.getattr("sql_analysis")? } else { project_config .getattr("settings")? @@ -320,7 +344,7 @@ fn completed(lifecycle: &Bound<'_, PyAny>, item_count: usize) -> PyResult<()> { &[("item_count", item_count.into_pyobject(py)?.into_any())], )?; call_with( - &lifecycle.getattr("completed")?, + &native_field(lifecycle, "completed")?, &[("metadata", metadata.into_any())], )?; Ok(()) @@ -361,16 +385,7 @@ fn imported_python<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult = PyDict::new(py); python.set_item("providers", &providers)?; for (field, discoverer) in PYTHON_DISCOVERERS { - python.set_item( - field, - call_with( - &discoverer.function(py)?, - &[ - ("project_dir", project_dir.clone()), - ("providers", providers.clone()), - ], - )?, - )?; + python.set_item(field, discoverer.discovered(project_dir, &providers)?)?; } Ok(python) } @@ -395,7 +410,7 @@ fn imported_python_paths(python: &Bound<'_, PyDict>) -> PyResult { } for group in groups { for item in group.try_iter()? { - paths.add(item?.getattr("relative_path")?)?; + paths.add(native_field(&item?, "relative_path")?)?; } } Ok(paths.len()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs index bccd5ac5c6..cc7bc8a336 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/input_reads.rs @@ -69,7 +69,7 @@ pub(crate) fn context_read(py: Python<'_>, name: &str) -> PyResult<()> { /// Note a provider settings class instantiated from the environment and files. #[pyfunction] -fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> PyResult<()> { +pub(crate) fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> PyResult<()> { for recorder in recorders(settings_class.py())? { recorder.get().add_settings_class(settings_class); } @@ -78,7 +78,7 @@ fn record_settings_class_read(settings_class: &Bound<'_, PyAny>) -> PyResult<()> /// Records environment variables and run-identity reads while entered. #[pyclass(frozen, module = "sqlbuild._native")] -struct CompileInputRecording { +pub(crate) struct CompileInputRecording { recorder: Mutex>>, } @@ -111,7 +111,7 @@ impl CompileInputRecording { /// Record template input reads from every thread until the returned block exits. #[pyfunction] -fn record_compile_input_reads() -> CompileInputRecording { +pub(crate) fn record_compile_input_reads() -> CompileInputRecording { CompileInputRecording { recorder: Mutex::new(None), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index a5d9d34675..562e243f7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -4,19 +4,24 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_attachment::declaration_scope::build_declaration_scope; use crate::bindings::_helpers::compile_attachment::effective_config::{ build_effective_connection, build_effective_settings, build_effective_vars, }; +use crate::bindings::_helpers::compile_attachment::model_inputs::build_model_inputs; +use crate::bindings::_helpers::compile_driver::diagnostic_collector::collect_compile_diagnostics; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, within, }; use crate::bindings::_helpers::compiled_project::attachment::{attach_checks, attach_resources}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::records::{compile_inputs_view, inputs_with}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::model_config::template_data::run_id; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_indexes::{ public_declaration_indexes, public_model_schema_index, @@ -114,13 +119,12 @@ pub(crate) fn attached_inputs<'py>( py: Python<'py>, request: &InputsRequest, ) -> PyResult> { - let collector: Bound<'py, PyAny> = - registry::native(py, NativeAttr::CollectCompileDiagnostics)?.call0()?; + let collector: Bound<'py, PyAny> = collect_compile_diagnostics(py)?.into_any(); let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = within(&collector, |violations| { Ok((compile_inputs(py, request)?, violations.clone())) })?; - let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; + let collected: Bound<'py, PyAny> = native_field(&violations, "diagnostics")?; if !collected.is_truthy()? { return Ok(built); } @@ -145,18 +149,15 @@ fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult = target_context(request, &discovered, &adapter_context)?; let run_id: Bound<'py, PyAny> = run_id(&request.run_id.bind(py).clone())?; let loaded_macros: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadProjectMacros)? - .call1((discovered.getattr("macro_files")?,))?; - let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; - let declaration_scope: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildDeclarationScope)?, - &[ - ("discovered_inputs", discovered.clone()), - ("loaded_macros", loaded_macros), - ("sql_lexical_syntax", syntax.clone()), - ("compile_cache_dir", target.compile_cache_dir.clone()), - ], + .call1((native_field(&discovered, "macro_files")?,))?; + let syntax: Bound<'py, PyAny> = native_field(&adapter_context, "sql_lexical_syntax")?; + let declaration_scope: Bound<'py, PyAny> = build_declaration_scope( + &discovered, + &loaded_macros, + &syntax, + Some(target.compile_cache_dir.clone()), )?; - let value_renderer: Bound<'py, PyAny> = adapter_context.getattr("value_renderer")?; + let value_renderer: Bound<'py, PyAny> = native_field(&adapter_context, "value_renderer")?; let model_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::ModelInputBuildContext)?, &[ @@ -166,16 +167,19 @@ fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult(py: Python<'py>, request: &InputsRequest) -> PyResult = keywords( py, &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), + ( + "project_config", + native_field(&discovered, "project_config")?, + ), + ("local_config", native_field(&discovered, "local_config")?), ("discovered_inputs", discovered.clone()), ("sql_lexical_syntax", syntax), ("run_id", run_id), @@ -208,7 +215,10 @@ fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult(py: Python<'py>, request: &InputsRequest) -> PyResult( )?; let target_name: Bound<'py, PyAny> = selected.get_item(0)?; let target: Bound<'py, PyAny> = selected.get_item(1)?; - let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; let cli_vars: Bound<'py, PyAny> = request.cli_vars.bind(py).clone(); let cli_vars: Bound<'py, PyAny> = if cli_vars.is_none() { PyDict::new(py).into_any() @@ -269,16 +279,19 @@ fn target_context<'py>( ®istry::class(py, RecordId::MacroContext)?, &[ ("adapter_name", adapter_name), - ("sql_analysis_enabled", settings.getattr("sql_analysis")?), + ( + "sql_analysis_enabled", + native_field(&settings, "sql_analysis")?, + ), ("target_name", target_name.clone()), ("vars", vars.clone()), ( "_value_renderer", - adapter_context.getattr("value_renderer")?, + native_field(adapter_context, "value_renderer")?, ), ( "_collection_rendering", - adapter_context.getattr("collection_rendering")?, + native_field(adapter_context, "collection_rendering")?, ), ( "_enforce_explicit_references", @@ -310,7 +323,7 @@ fn model_build<'py>( (enums.into_any(), constants.into_any()); let schemas: Bound<'py, PyAny> = public_model_schema_index(discovered.clone())?.into_any(); let native_project: Bound<'py, PyAny> = - registry::native(py, NativeAttr::NativeCompiledProject)?.call0()?; + Bound::new(py, NativeCompiledProject::default())?.into_any(); let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; let model_context: Bound<'py, PyAny> = replace.call( (context,), @@ -323,26 +336,30 @@ fn model_build<'py>( ], )?), )?; - let inputs: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildModelInputs)?, - &[ - ("discovered_inputs", discovered.clone()), - ("context", model_context), - ( - "no_sql_validation", - request.no_sql_validation.bind(py).clone(), - ), - ( - "defer_model_sql_validation", - request.defer_model_sql_validation.bind(py).clone(), - ), - ( - "external_sql_reference_resolver", - request.external_sql_reference_resolver.bind(py).clone(), - ), - ("native_project", native_project.clone()), - ], - )?; + let inputs: Bound<'py, PyAny> = build_model_inputs( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("context", model_context), + ( + "no_sql_validation", + request.no_sql_validation.bind(py).clone(), + ), + ( + "defer_model_sql_validation", + request.defer_model_sql_validation.bind(py).clone(), + ), + ( + "external_sql_reference_resolver", + request.external_sql_reference_resolver.bind(py).clone(), + ), + ("native_project", native_project.clone()), + ], + )?), + )? + .into_any(); let context: Bound<'py, PyAny> = replace.call( (context,), Some(&keywords( @@ -371,7 +388,7 @@ fn attached_resources<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let context: &Bound<'py, PyAny> = &models.context; - let settings: Bound<'py, PyAny> = context.getattr("effective_settings")?; + let settings: Bound<'py, PyAny> = native_field(context, "effective_settings")?; let resources: Bound<'py, PyAny> = attach_resources( py, Some(keywords( @@ -385,8 +402,8 @@ fn attached_resources<'py>( py, [ settings.clone(), - context.getattr("target_config")?, - context.getattr("macro_context")?.getattr("adapter_name")?, + native_field(context, "target_config")?, + native_field(context, "macro_context")?.getattr("adapter_name")?, ], )? .into_any(), @@ -397,7 +414,10 @@ fn attached_resources<'py>( ), ( "python_functions_inherit_default_namespace", - adapter_context.getattr("python_functions_inherit_default_namespace")?, + native_field( + adapter_context, + "python_functions_inherit_default_namespace", + )?, ), ], )?), @@ -417,13 +437,16 @@ fn attached_resources<'py>( py, [ settings, - adapter_context.getattr("value_renderer")?, - context.getattr("sql_lexical_syntax")?, + native_field(adapter_context, "value_renderer")?, + native_field(context, "sql_lexical_syntax")?, ], )? .into_any(), ), - ("scan_cache", declaration_scope.getattr("sql_test_scans")?), + ( + "scan_cache", + native_field(declaration_scope, "sql_test_scans")?, + ), ("source_inputs", sources.clone()), ], )?), @@ -448,8 +471,8 @@ fn effective_connection<'py>( return Ok(resolved); } build_effective_connection( - &discovered.getattr("project_config")?, - &discovered.getattr("local_config")?, + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, &target.target.clone(), &target.vars.clone(), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs index 77adbbe5b2..07eb7e45d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs @@ -8,6 +8,7 @@ use sqlbuild_discovery::yaml_entries::main::integration_loader_name::integration use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::third_party::ThirdPartyHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -43,15 +44,18 @@ pub(crate) fn integration_loader_functions<'py>( let factory: Bound<'py, PyAny> = host::entry(py, integration.loader_factory)?; for source_file in source_files.try_iter()? { let source_file: Bound<'py, PyAny> = source_file?; - for source_entry in source_file.getattr("source_entries")?.try_iter()? { + for source_entry in native_field(&source_file, "source_entries")?.try_iter()? { let source_entry: Bound<'py, PyAny> = source_entry?; - let declared: Bound<'py, PyAny> = source_entry.getattr("integration_loader")?; - if declared.is_none() || declared.getattr("kind")?.extract::()? != kind { + let declared: Bound<'py, PyAny> = + native_field(&source_entry, "integration_loader")?; + if declared.is_none() + || native_field(&declared, "kind")?.extract::()? != kind + { continue; } let name: String = integration_loader_name( &kind, - &source_entry.getattr("name")?.extract::()?, + &native_field(&source_entry, "name")?.extract::()?, ); let function: Bound<'py, PyAny> = call_with( &factory, @@ -63,8 +67,11 @@ pub(crate) fn integration_loader_functions<'py>( loaders.push(call_with( &record_class, &[ - ("file_path", source_file.getattr("file_path")?), - ("relative_path", source_file.getattr("relative_path")?), + ("file_path", native_field(&source_file, "file_path")?), + ( + "relative_path", + native_field(&source_file, "relative_path")?, + ), ("name", name.into_pyobject(py)?.into_any()), ("function", function), ("connection_mode", external.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs index ef8fdc85aa..b450d2fac9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/mod.rs @@ -27,5 +27,6 @@ pub(crate) mod python_plan_entries; pub(crate) mod registration; pub(crate) mod relation_targets; pub(crate) mod selection_phases; +pub(crate) mod stage_work; pub(crate) mod target_defaults; pub(crate) mod targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 234514b0e5..0c8d7bb8db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -9,10 +9,14 @@ use crate::bindings::_helpers::compile_command::reuse_recording::ArtifactRecorde use crate::bindings::_helpers::compile_command::target_writer::{ Prepared, planning_diagnostics, write_static_target, }; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::manifest::builder::build_manifest; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; const EMPTY_DAG_PATH: &str = ""; @@ -52,26 +56,25 @@ pub(crate) fn manifest_payload<'py>( } let started: f64 = monotonic(py)?; status.start("Building manifest...")?; - let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; - let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let graph: Bound<'py, PyAny> = analysis.getattr("graph")?; - let adapter_type: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", discovered.getattr("local_config")?), - ], + let discovered: Bound<'py, PyAny> = native_field(analysis, "discovered_inputs")?; + let project_config: Bound<'py, PyAny> = native_field(&discovered, "project_config")?; + let graph: Bound<'py, PyAny> = native_field(analysis, "graph")?; + let adapter_type: Bound<'py, PyAny> = resolve_effective_adapter_name( + &project_config, + &native_field(&discovered, "local_config")?, )?; - let payload: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildManifest)?, + let payload: Bound<'py, PyAny> = build_manifest(Some(&keywords( + py, &[ - ("project", graph.getattr("project")?), + ("project", native_field(&graph, "project")?), ("plan_output", py.None().into_bound(py)), - ("project_name", project_config.getattr("name")?), + ("project_name", native_field(&project_config, "name")?), ("adapter_type", adapter_type), - ("graph", graph.getattr("native")?), + ("graph", native_field(&graph, "native")?), ], - )?; + )?))? + .into_bound(py) + .into_any(); status.complete(&format!( "Built manifest. ({}s)", seconds_since(py, started)? @@ -107,7 +110,7 @@ pub(crate) fn write_dag_artifact<'py>( let py: Python<'py> = analysis.py(); let started: f64 = monotonic(py)?; status.start("Writing DAG artifact...")?; - let discovered: Bound<'py, PyAny> = analysis.getattr("discovered_inputs")?; + let discovered: Bound<'py, PyAny> = native_field(analysis, "discovered_inputs")?; let python_graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let path: Bound<'py, PyAny> = resolved_dag_path(project_dir, dag_path)?; let project_name: String = discovered @@ -115,7 +118,7 @@ pub(crate) fn write_dag_artifact<'py>( .getattr("name")? .extract()?; let dag_json: String = build_dag_json( - &analysis.getattr("graph")?, + &native_field(analysis, "graph")?, &project_name, Some(&python_graph), )?; @@ -146,8 +149,8 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( Ok(planning_diagnostics( &target_dir(project_dir)?, ( - &analysis.getattr("adapter")?, - &analysis.getattr("graph")?.getattr("project")?, + &native_field(analysis, "adapter")?, + &native_field(analysis, "graph")?.getattr("project")?, ), Prepared { planned_tests: None, @@ -168,7 +171,7 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( } let reference_error: Bound<'py, PyAny> = registry::error(py, ErrorId::SqlTestReference)?; let location: Bound<'py, PyAny> = if value.is_instance(&reference_error)? { - value.getattr("location")? + native_field(&value, "location")? } else { py.None().into_bound(py) }; @@ -183,9 +186,9 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( "severity", registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?, ), - ("code", value.getattr("code")?), + ("code", native_field(&value, "code")?), ("message", value.str()?.into_any()), - ("help", value.getattr("help")?), + ("help", native_field(&value, "help")?), ("location", location), ], )?; @@ -221,7 +224,7 @@ pub(crate) fn write_artifacts<'py>( py.None().into_bound(py) } else { call_with( - &prepared_artifacts.getattr("publish")?, + &native_field(prepared_artifacts, "publish")?, &[ ("target_dir", target.clone()), ("manifest", manifest.clone()), @@ -232,8 +235,8 @@ pub(crate) fn write_artifacts<'py>( write_static_target( &target, ( - &analysis.getattr("adapter")?, - &analysis.getattr("graph")?.getattr("project")?, + &native_field(analysis, "adapter")?, + &native_field(analysis, "graph")?.getattr("project")?, ), manifest, Prepared { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs index f79c418d9b..b5de764392 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline_entries::{ compile_analysis_selection, project_graph, @@ -11,23 +12,29 @@ use crate::bindings::_helpers::compile_driver::pipeline_entries::{ use crate::bindings::_helpers::compile_driver::plan_result::{ PlanningInputs, connection_with_hooks, pipeline_result, }; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, }; +use crate::bindings::_helpers::compile_driver::selection_phases::{ + apply_unit_test_selectors, resolve_configured_rule_selection, +}; +use crate::bindings::_helpers::diagnostics::trackers::current_tracker; use crate::bindings::_helpers::frontier::stage_capture::write_plan_capture; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::results::{format_rules_result, rules_progress}; +use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; use crate::bindings::_helpers::rules::rules_config::load_rules_config; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const GENERIC_DIALECT: &str = "generic"; /// The current build's phase timing tracker, when one is recording. fn timing_tracker(py: Python<'_>) -> PyResult> { - registry::native(py, NativeAttr::BuildPhaseTimingTracker)?.call_method0("current") + current_tracker(py) } /// Report `message` through `on_progress` when a callback is set. @@ -41,7 +48,7 @@ fn progress(on_progress: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { /// Compile a project without opening a warehouse planning connection. #[pyfunction] #[pyo3(signature = (**options))] -fn compile_project_phase<'py>( +pub(crate) fn compile_project_phase<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -76,15 +83,12 @@ fn compiled_phase<'py>( } else { registry::class(py, RecordId::CompilePipelineOptions)?.call0()? }; - let configured: Bound<'py, PyAny> = options.getattr("connection_config")?; + let configured: Bound<'py, PyAny> = native_field(&options, "connection_config")?; let connection_config: Bound<'py, PyAny> = if configured.is_none() { - call_with( - ®istry::native(py, NativeAttr::BuildEffectiveConnectionConfig)?, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", options.getattr("selected_target")?), - ("cli_vars", options.getattr("cli_vars")?), - ], + build_effective_connection_config( + discovered, + Some(native_field(&options, "selected_target")?), + Some(native_field(&options, "cli_vars")?), )? } else { configured @@ -93,21 +97,30 @@ fn compiled_phase<'py>( let started: f64 = monotonic(py)?; let compiled: PyResult> = (|| { let selection: Bound<'py, PyAny> = compile_analysis_selection(&options, discovered)?; - call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("selected_target", options.getattr("selected_target")?), - ("no_sql_validation", options.getattr("no_sql_validation")?), - ("cli_vars", options.getattr("cli_vars")?), - ( - "external_sql_reference_resolver", - options.getattr("external_sql_reference_resolver")?, - ), - ("resolved_connection", connection_config.clone()), - ("analysis_selection", selection), - ], + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ( + "selected_target", + native_field(&options, "selected_target")?, + ), + ( + "no_sql_validation", + native_field(&options, "no_sql_validation")?, + ), + ("cli_vars", native_field(&options, "cli_vars")?), + ( + "external_sql_reference_resolver", + native_field(&options, "external_sql_reference_resolver")?, + ), + ("resolved_connection", connection_config.clone()), + ("analysis_selection", selection), + ], + )?), ) })(); let compile_seconds: f64 = monotonic(py)? - started; @@ -141,14 +154,14 @@ fn reject_compile_errors( on_progress: &Bound<'_, PyAny>, ) -> PyResult<()> { let py: Python<'_> = project.py(); - let plan_sql_tests: bool = options.getattr("plan_sql_tests")?.is_truthy()?; + let plan_sql_tests: bool = native_field(options, "plan_sql_tests")?.is_truthy()?; let sql_test: Bound<'_, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SQL_TEST")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SQL_TEST")?; let mut errors: Vec> = Vec::new(); - for diagnostic in project.getattr("diagnostics")?.try_iter()? { + for diagnostic in native_field(project, "diagnostics")?.try_iter()? { let diagnostic: Bound<'_, PyAny> = diagnostic?; - if diagnostic.getattr("is_error")?.is_truthy()? - && (plan_sql_tests || diagnostic.getattr("resource_type")?.ne(&sql_test)?) + if native_field(&diagnostic, "is_error")?.is_truthy()? + && (plan_sql_tests || native_field(&diagnostic, "resource_type")?.ne(&sql_test)?) { errors.push(diagnostic); } @@ -170,13 +183,13 @@ fn reject_compile_errors( }; lines.push(format!( "[{}] {}:{}:{}: {}", - diagnostic.getattr("code")?.str()?, - diagnostic.getattr("path")?.str()?, + native_field(diagnostic, "code")?.str()?, + native_field(diagnostic, "path")?.str()?, or_one("line")?, or_one("column")?, - diagnostic.getattr("message")?.str()?, + native_field(diagnostic, "message")?.str()?, )); - helps.add(diagnostic.getattr("help")?)?; + helps.add(native_field(diagnostic, "help")?)?; } let help: Bound<'_, PyAny> = if helps.len() == 1 { helps.pop().unwrap_or_else(|| py.None().into_bound(py)) @@ -187,7 +200,7 @@ fn reject_compile_errors( (lines.join("\n"),), Some(&keywords( py, - &[("code", first.getattr("code")?), ("help", help)], + &[("code", native_field(first, "code")?), ("help", help)], )?), )?; Err(PyErr::from_value(error)) @@ -208,23 +221,18 @@ fn run_compile_pipeline<'py>( )?; let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; let adapter: Bound<'py, PyAny> = arguments.value("adapter")?; - let options: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ApplyUnitTestSelectors)?, - &[ - ("options", arguments.value("options")?), - ("discovered_inputs", discovered.clone()), - ], - )?; + let options: Bound<'py, PyAny> = + apply_unit_test_selectors(&arguments.value("options")?, &discovered)?; let hooks: Bound<'py, PyAny> = arguments.value("hooks")?; let hooks: Bound<'py, PyAny> = if hooks.is_none() { host::entry(py, ExecutorHost::ConnectionHooks)?.call0()? } else { hooks }; - let on_progress: Bound<'py, PyAny> = hooks.getattr("on_progress")?; + let on_progress: Bound<'py, PyAny> = native_field(&hooks, "on_progress")?; let compiled: Bound<'py, PyAny> = compiled_phase((&discovered, &adapter), &options, &on_progress)?; - let project: Bound<'py, PyAny> = compiled.getattr("project")?; + let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; let graph: Bound<'py, PyAny> = project_graph(&project)?; run_rules_gate( (&discovered, &adapter), @@ -242,7 +250,7 @@ fn run_rules_gate<'py>( ) -> PyResult<()> { let py: Python<'py> = discovered.py(); let rules_error: Bound<'py, PyAny> = registry::error(py, ErrorId::Rules)?; - let project_dir: Bound<'py, PyAny> = discovered.getattr("project_dir")?; + let project_dir: Bound<'py, PyAny> = native_field(discovered, "project_dir")?; if project_dir.is_none() { return Err(PyErr::from_value( rules_error.call1(("compiler rules require a resolved project directory",))?, @@ -250,7 +258,7 @@ fn run_rules_gate<'py>( } let mut config: Bound<'py, PyAny> = load_rules_config(py, project_dir.extract()?)?.into_bound(py); - if options.getattr("no_cache")?.is_truthy()? { + if native_field(options, "no_cache")?.is_truthy()? { let cache: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::RulesCacheConfig)?, &[("enabled", false.into_pyobject(py)?.to_owned().into_any())], @@ -266,32 +274,32 @@ fn run_rules_gate<'py>( } else { GENERIC_DIALECT.into_pyobject(py)?.into_any() }; - let selected_keys: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveConfiguredRuleSelection)?, - &[ - ("options", options.clone()), - ("discovered_inputs", discovered.clone()), - ("project", project.clone()), - ], - )?; - let result: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::EvaluateRules)?, - &[ - ("graph", graph.clone()), - ("discovered_inputs", discovered.clone()), - ("config", config), - ("project_dir", project_dir), - ("dialect", dialect), - ("selected_keys", selected_keys), - ("no_sql_analysis", options.getattr("no_sql_validation")?), - ], + let selected_keys: Bound<'py, PyAny> = + resolve_configured_rule_selection(options, discovered, project)?; + let result: Bound<'py, PyAny> = evaluate_rules( + py, + Some(keywords( + py, + &[ + ("graph", graph.clone()), + ("discovered_inputs", discovered.clone()), + ("config", config), + ("project_dir", project_dir), + ("dialect", dialect), + ("selected_keys", selected_keys), + ( + "no_sql_analysis", + native_field(options, "no_sql_validation")?, + ), + ], + )?), )?; if !on_progress.is_none() { let elapsed: f64 = monotonic(py)? - started; let message: String = rules_progress(elapsed, &result, true)?; on_progress.call1((message,))?; } - let findings: Bound<'py, PyAny> = result.getattr("findings")?; + let findings: Bound<'py, PyAny> = native_field(&result, "findings")?; if !findings.is_truthy()? { return Ok(()); } @@ -299,13 +307,16 @@ fn run_rules_gate<'py>( ®istry::class(py, RecordId::RulesResult)?, &[ ("findings", findings), - ("evaluated_models", result.getattr("evaluated_models")?), + ( + "evaluated_models", + native_field(&result, "evaluated_models")?, + ), ( "unevaluated_resources", - result.getattr("unevaluated_resources")?, + native_field(&result, "unevaluated_resources")?, ), - ("cache_hits", result.getattr("cache_hits")?), - ("cache_misses", result.getattr("cache_misses")?), + ("cache_hits", native_field(&result, "cache_hits")?), + ("cache_misses", native_field(&result, "cache_misses")?), ], )?; let rendered: String = format_rules_result(&summary, false)?; @@ -325,8 +336,11 @@ fn planned<'py>( let started: f64 = monotonic(py)?; let mut connection: Option> = None; let result: PyResult> = (|| { - let opened: Bound<'py, PyAny> = - connection_with_hooks(adapter, &compiled.getattr("connection_config")?, hooks)?; + let opened: Bound<'py, PyAny> = connection_with_hooks( + adapter, + &native_field(compiled, "connection_config")?, + hooks, + )?; connection = Some(opened.clone()); pipeline_result( &PlanningInputs { @@ -334,9 +348,9 @@ fn planned<'py>( adapter, connection: &opened, }, - &compiled.getattr("project")?, + &native_field(compiled, "project")?, options, - &hooks.getattr("on_progress")?, + &native_field(hooks, "on_progress")?, ) })(); if let Some(connection) = connection.filter(|connection| !connection.is_none()) { @@ -347,13 +361,16 @@ fn planned<'py>( tracker.setattr("planning_seconds", planning_seconds)?; } let result: Bound<'py, PyAny> = result?; - write_plan_capture(&result.getattr("plan_output")?)?; + write_plan_capture(&native_field(&result, "plan_output")?)?; imported(py, "dataclasses", "replace")?.call( (result,), Some(&keywords( py, &[ - ("compile_seconds", compiled.getattr("compile_seconds")?), + ( + "compile_seconds", + native_field(compiled, "compile_seconds")?, + ), ( "planning_seconds", planning_seconds.into_pyobject(py)?.into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs index 64231949f7..4a3a3da6c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/pipeline_entries.rs @@ -4,8 +4,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyFloat}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, keywords, +}; +use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// Discovered Python node functions whose selectors need the Python graph. @@ -58,7 +63,7 @@ pub(crate) fn compile_analysis_selection<'py>( } } let selection: Bound<'py, PyAny> = registry::class(py, RecordId::CompileAnalysisSelection)?; - let no_cache: Bound<'py, PyAny> = options.getattr("no_cache")?; + let no_cache: Bound<'py, PyAny> = native_field(options, "no_cache")?; if (options .getattr("resolve_python_run_selectors")? .is_truthy()? @@ -73,9 +78,12 @@ pub(crate) fn compile_analysis_selection<'py>( call_with( &selection, &[ - ("select", options.getattr("select")?), - ("exclude", options.getattr("exclude")?), - ("auto_load_sources", options.getattr("auto_load_sources")?), + ("select", native_field(options, "select")?), + ("exclude", native_field(options, "exclude")?), + ( + "auto_load_sources", + native_field(options, "auto_load_sources")?, + ), ("no_cache", no_cache), ], ) @@ -99,13 +107,13 @@ pub(crate) fn build_project_graph_from_compiled_project<'py>( /// The project's natively held graph; dict views are built only when a boundary reads them. pub(crate) fn project_graph<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - call_with( - ®istry::class(project.py(), RecordId::ProjectGraph)?, - &[ - ("project", project.clone()), - ("native", project.getattr("lineage_graph")?), - ], - ) + let native: Bound<'py, NativeProjectGraph> = + native_field(project, "lineage_graph")?.cast_into()?; + Ok(Bound::new( + project.py(), + CompiledProjectGraph::new(project.clone().unbind(), native.unbind()), + )? + .into_any()) } /// Return whether a planned build has runtime work to execute. @@ -150,9 +158,12 @@ pub(crate) fn build_static_pipeline_result<'py>( call_with( ®istry::class(py, RecordId::CompilePipelineResult)?, &[ - ("project", context.getattr("project")?), + ("project", native_field(&context, "project")?), ("plan_output", arguments.value("plan_output")?), - ("compile_seconds", context.getattr("compile_seconds")?), + ( + "compile_seconds", + native_field(&context, "compile_seconds")?, + ), ("planning_seconds", PyFloat::new(py, 0.0).into_any()), ], ) @@ -174,24 +185,30 @@ fn run_compile_only_pipeline<'py>( &["on_progress"], ), )?; - let compiled: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CompileProjectPhase)?, - &[ - ("discovered_inputs", arguments.value("discovered_inputs")?), - ("adapter", arguments.value("adapter")?), - ("options", arguments.value("options")?), - ("on_progress", arguments.value("on_progress")?), - ], + let compiled: Bound<'py, PyAny> = compile_project_phase( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", arguments.value("discovered_inputs")?), + ("adapter", arguments.value("adapter")?), + ("options", arguments.value("options")?), + ("on_progress", arguments.value("on_progress")?), + ], + )?), )?; call_with( ®istry::class(py, RecordId::CompilePipelineResult)?, &[ - ("project", compiled.getattr("project")?), + ("project", native_field(&compiled, "project")?), ( "plan_output", registry::class(py, RecordId::PlanOutput)?.call0()?, ), - ("compile_seconds", compiled.getattr("compile_seconds")?), + ( + "compile_seconds", + native_field(&compiled, "compile_seconds")?, + ), ("planning_seconds", PyFloat::new(py, 0.0).into_any()), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 89409309b2..12ced32ae3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -5,7 +5,10 @@ use crate::bindings::_helpers::compile_driver::deferral::gather_deferred_relatio use crate::bindings::_helpers::compile_driver::deferral::resolve_deferred_target_config; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_project_graph_from_compiled_project; use crate::bindings::_helpers::compile_driver::python_calls::keywords; -use crate::bindings::_helpers::compile_driver::python_plan_entries::python_read_source_names_for; +use crate::bindings::_helpers::compile_driver::python_plan_entries::{ + build_python_run_plan_outputs, python_read_source_names_for, +}; +use crate::bindings::_helpers::planner_entries::execution_plan::build_execution_plan; use crate::bindings::_helpers::planner_phases::objects::object; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -17,7 +20,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, use crate::bindings::_helpers::graph::python_node_graph::run_selection_from_inputs; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const SINGLE_CONNECTION: i64 = 1; @@ -41,17 +45,17 @@ pub(crate) fn pipeline_result<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let discovered: &Bound<'py, PyAny> = inputs.discovered_inputs; - let select: Bound<'py, PyAny> = options.getattr("select")?; - let exclude: Bound<'py, PyAny> = options.getattr("exclude")?; + let select: Bound<'py, PyAny> = native_field(options, "select")?; + let exclude: Bound<'py, PyAny> = native_field(options, "exclude")?; let (deferred_locations, deferred_relations): Deferral<'py> = - deferral(inputs, project, &options.getattr("defer_to")?)?; + deferral(inputs, project, &native_field(options, "defer_to")?)?; let empty = || -> PyResult> { Ok(PyFrozenSet::empty(py)?.into_any()) }; let mut selected_sql_keys: Bound<'py, PyAny> = py.None().into_bound(py); let mut selected_python_node_names: Bound<'py, PyAny> = empty()?; let mut python_read_source_names: Bound<'py, PyAny> = empty()?; let mut run_selection: Bound<'py, PyAny> = py.None().into_bound(py); - let sql_test_selection: Bound<'py, PyAny> = options.getattr("sql_test_selection")?; - if !sql_test_selection.getattr("models_selected")?.is_truthy()? { + let sql_test_selection: Bound<'py, PyAny> = native_field(options, "sql_test_selection")?; + if !native_field(&sql_test_selection, "models_selected")?.is_truthy()? { selected_sql_keys = empty()?; } else if options .getattr("resolve_python_run_selectors")? @@ -63,8 +67,8 @@ pub(crate) fn pipeline_result<'py>( )?; run_selection = run_selection_from_inputs((select.extract()?, exclude.extract()?), &graph, discovered)?; - selected_sql_keys = run_selection.getattr("sql_keys")?; - selected_python_node_names = run_selection.getattr("python_node_names")?; + selected_sql_keys = native_field(&run_selection, "sql_keys")?; + selected_python_node_names = native_field(&run_selection, "python_node_names")?; python_read_source_names = python_read_source_names_for( py, Some(keywords( @@ -122,46 +126,55 @@ pub(crate) fn pipeline_result<'py>( ], )?, )?; - let plan_output: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildExecutionPlan)?, - &[ - ("project", project.clone()), - ("adapter", inputs.adapter.clone()), - ("connection", inputs.connection.clone()), - ("selection", selection), - ("overrides", overrides), - ("deferral", deferral), - ("policies", policies), - ("on_progress", on_progress.clone()), - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ], + let plan_output: Bound<'py, PyAny> = build_execution_plan( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", inputs.adapter.clone()), + ("connection", inputs.connection.clone()), + ("selection", selection), + ("overrides", overrides), + ("deferral", deferral), + ("policies", policies), + ("on_progress", on_progress.clone()), + ( + "project_config", + native_field(discovered, "project_config")?, + ), + ("local_config", native_field(discovered, "local_config")?), + ], + )?), )?; let custom_materializations: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadCustomMaterializations)? - .call1((discovered.getattr("materialization_files")?,))?; - let python_outputs: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildPythonRunPlanOutputs)?, - &[ - ("discovered_inputs", discovered.clone()), - ("plan_output", plan_output), - ("run_selection", run_selection), - ("selected_python_node_names", selected_python_node_names), - ], + .call1((native_field(discovered, "materialization_files")?,))?; + let python_outputs: Bound<'py, PyAny> = build_python_run_plan_outputs( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("plan_output", plan_output), + ("run_selection", run_selection), + ("selected_python_node_names", selected_python_node_names), + ], + )?), )?; call_with( ®istry::class(py, RecordId::CompilePipelineResult)?, &[ ("project", project.clone()), - ("plan_output", python_outputs.getattr("plan_output")?), + ("plan_output", native_field(&python_outputs, "plan_output")?), ("custom_materializations", custom_materializations), ( "python_node_names", - python_outputs.getattr("selected_python_node_names")?, + native_field(&python_outputs, "selected_python_node_names")?, ), ( "python_plan_entries", - python_outputs.getattr("python_plan_entries")?, + native_field(&python_outputs, "python_plan_entries")?, ), ], ) @@ -198,7 +211,7 @@ fn deferral<'py>( ("defer_to", defer_to.clone()), ( "current_target_name", - project.getattr("effective_target_name")?, + native_field(project, "effective_target_name")?, ), ], )?), @@ -212,7 +225,7 @@ fn deferral<'py>( &[ ("project", project.clone()), ("deferred_target_config", target_config), - ("effective_vars", project.getattr("effective_vars")?), + ("effective_vars", native_field(project, "effective_vars")?), ("default_schema", adapter.call_method0("default_schema")?), ( "default_database", @@ -220,7 +233,7 @@ fn deferral<'py>( ), ( "render_qualified_name", - adapter.getattr("render_qualified_name")?, + native_field(adapter, "render_qualified_name")?, ), ], )?), @@ -250,7 +263,7 @@ pub(crate) fn connection_with_hooks<'py>( hooks: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = adapter.py(); - let on_start: Bound<'py, PyAny> = hooks.getattr("on_connection_start")?; + let on_start: Bound<'py, PyAny> = native_field(hooks, "on_connection_start")?; if !on_start.is_none() { on_start.call1((SINGLE_CONNECTION,))?; } @@ -264,14 +277,14 @@ pub(crate) fn connection_with_hooks<'py>( { Ok(connection) => connection, Err(error) => { - let on_error: Bound<'py, PyAny> = hooks.getattr("on_connection_error")?; + let on_error: Bound<'py, PyAny> = native_field(hooks, "on_connection_error")?; if error.is_instance_of::(hooks.py()) && !on_error.is_none() { on_error.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; } return Err(error); } }; - let on_complete: Bound<'py, PyAny> = hooks.getattr("on_connection_complete")?; + let on_complete: Bound<'py, PyAny> = native_field(hooks, "on_connection_complete")?; if !on_complete.is_none() { on_complete.call((SINGLE_CONNECTION,), Some(&elapsed(started)?))?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index ca9031ae7c..60a2019f3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -20,10 +20,14 @@ use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridg use crate::bindings::_helpers::compile_driver::assembly::assemble_project; use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::selection_phases::resolve_analysis_model_names; +use crate::bindings::_helpers::compile_driver::stage_work::{PROMOTION_CONFLICTS, count_stage}; use crate::bindings::_helpers::compiled_project::records::project_with; +use crate::bindings::_helpers::contracts::contract_diagnostics::promotion_conflict_diagnostics; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; @@ -134,26 +138,21 @@ fn python_compiled_project<'py>( named_target_schema_strategy(discovered, adapter, request.selected_target.bind(py))?; managed_loader_target_isolation(discovered, adapter, request.resolved_connection.bind(py))?; let selection: &Bound<'py, PyAny> = request.analysis_selection.bind(py); - let no_cache: bool = !selection.is_none() && selection.getattr("no_cache")?.is_truthy()?; + let no_cache: bool = + !selection.is_none() && native_field(selection, "no_cache")?.is_truthy()?; let compile_inputs: Bound<'py, PyAny> = record_compile_timing("attachment_ms", || { frontier_compile_inputs(py, request, no_cache) })?; if !selection.is_none() { - let on_inputs_ready: Bound<'py, PyAny> = selection.getattr("on_inputs_ready")?; + let on_inputs_ready: Bound<'py, PyAny> = native_field(selection, "on_inputs_ready")?; if !on_inputs_ready.is_none() { call_with(&on_inputs_ready, &[("inputs", compile_inputs.clone())])?; } } let inference_profile: Bound<'py, PyAny> = adapter.call_method0("expression_inference_profile")?; - let analysis_model_names: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveAnalysisModelNames)?, - &[ - ("compile_inputs", compile_inputs.clone()), - ("inference_profile", inference_profile.clone()), - ("selection", selection.clone()), - ], - )?; + let analysis_model_names: Bound<'py, PyAny> = + resolve_analysis_model_names(&compile_inputs, &inference_profile, selection)?; let project: Bound<'py, PyAny> = assemble_project( py, Some(keywords( @@ -171,7 +170,7 @@ fn python_compiled_project<'py>( ), ( "analysis_cache_dir", - compile_inputs.getattr("compile_cache_dir")?, + native_field(&compile_inputs, "compile_cache_dir")?, ), ("analysis_model_names", analysis_model_names), ], @@ -190,7 +189,7 @@ fn targeted_project<'py>( let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); managed_write_schemas(adapter, project)?; - let effective_schema: Bound<'py, PyAny> = project.getattr("effective_target_schema")?; + let effective_schema: Bound<'py, PyAny> = native_field(project, "effective_target_schema")?; let default_schema: Bound<'py, PyAny> = if effective_schema.is_truthy()? { effective_schema } else { @@ -199,13 +198,13 @@ fn targeted_project<'py>( let defaults: TargetDefaults<'py> = TargetDefaults { default_schema, default_database: adapter.call_method0("default_database")?, - render_qualified_name: adapter.getattr("render_qualified_name")?, + render_qualified_name: native_field(adapter, "render_qualified_name")?, python_functions_inherit_default_namespace: adapter .call_method0("python_functions_inherit_default_namespace")?, }; let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; - let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let project_config: Bound<'py, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'py, PyAny> = native_field(discovered, "local_config")?; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name(&project_config, &local_config.clone()) .and_then(|value| value.into_bound_py_any(py))?; @@ -220,17 +219,14 @@ fn targeted_project<'py>( } else { pyo3::types::PyString::new(py, PROJECT_CONFIG_FILENAME).into_any() }; - let conflicts: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::PromotionConflictDiagnostics)?, - &[ - ("project", project.clone()), - ( - "adapter_default", - adapter.call_method0("default_table_promotion_mode")?, - ), - ("settings_file", settings_file), - ], - )?; + count_stage(py, PROMOTION_CONFLICTS, 1)?; + let conflicts: Bound<'py, PyAny> = promotion_conflict_diagnostics( + py, + &project, + &adapter.call_method0("default_table_promotion_mode")?, + settings_file.extract()?, + )? + .into_any(); let diagnostics: Bound<'py, PyTuple> = PyTuple::new( py, project @@ -254,7 +250,7 @@ fn frontier_compile_inputs<'py>( let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); let collection_rendering: Bound<'py, PyAny> = resolve_effective_collection_rendering( - &discovered.getattr("project_config")?, + &native_field(discovered, "project_config")?, &py.None().into_bound(py), ) .and_then(|value| value.into_bound_py_any(py))?; @@ -267,7 +263,10 @@ fn frontier_compile_inputs<'py>( "python_functions_inherit_default_namespace", adapter.call_method0("python_functions_inherit_default_namespace")?, ), - ("sql_lexical_syntax", adapter.getattr("sql_lexical_syntax")?), + ( + "sql_lexical_syntax", + native_field(adapter, "sql_lexical_syntax")?, + ), ], )?; let inputs: InputsRequest = InputsRequest { @@ -299,7 +298,7 @@ fn frontier_compile_inputs<'py>( /// Build a static graph with an explicit deep-analysis selection. #[pyfunction] #[pyo3(signature = (**options))] -fn build_project_graph_with_analysis_selection<'py>( +pub(crate) fn build_project_graph_with_analysis_selection<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 625a1a4cac..dd794cb26c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -31,8 +32,8 @@ impl<'py> NodeKinds<'py> { fn load(py: Python<'py>) -> PyResult { let kinds: Bound<'py, PyAny> = host::entry(py, UserCodeHost::PythonNodeKind)?; Ok(Self { - task: kinds.getattr("TASK")?, - asset: kinds.getattr("ASSET")?, + task: native_field(&kinds, "TASK")?, + asset: native_field(&kinds, "ASSET")?, }) } @@ -89,14 +90,14 @@ pub(crate) fn python_read_source_names_for<'py>( host::entry(py, UserCodeHost::SqlResourceRefKind)?.getattr("SOURCE")?; let mut names: Vec> = Vec::new(); for reference in call_with( - &graph.getattr("selected_sql_refs")?, + &native_field(&graph, "selected_sql_refs")?, &[("selected_names", selected)], )? .try_iter()? { let reference: Bound<'py, PyAny> = reference?; - if reference.getattr("kind")?.eq(&source)? { - names.push(reference.getattr("name")?); + if native_field(&reference, "kind")?.eq(&source)? { + names.push(native_field(&reference, "name")?); } } frozenset.call1((PyTuple::new(py, names)?,)) @@ -158,7 +159,7 @@ pub(crate) fn build_python_run_plan_outputs<'py>( let selection: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::PythonSqlRunSelection)?, &[ - ("sql_keys", plan_output.getattr("selected_keys")?), + ("sql_keys", native_field(&plan_output, "selected_keys")?), ("python_node_names", selected.clone()), ], )?; @@ -166,7 +167,7 @@ pub(crate) fn build_python_run_plan_outputs<'py>( let entries: Bound<'py, PyTuple> = plan_entries( &lifecycle, (&graph, &kinds), - &plan_output.getattr("python_identity_fingerprints")?, + &native_field(&plan_output, "python_identity_fingerprints")?, )?; call_with( &outputs, @@ -217,7 +218,7 @@ fn plan_entries<'py>( let phases: Bound<'py, PyAny> = registry::class(py, RecordId::PythonRunPhase)?; let entry_class: Bound<'py, PyAny> = registry::class(py, RecordId::PythonPlanEntry)?; let upstream: HashMap> = upstream_names(graph)?; - let nodes: Bound<'py, PyAny> = graph.getattr("nodes_by_name")?; + let nodes: Bound<'py, PyAny> = native_field(graph, "nodes_by_name")?; let mut entries: Vec> = Vec::new(); for (attribute, phase) in [ ("ingress_python_node_names", "PRE_SQL_INGRESS"), @@ -230,11 +231,11 @@ fn plan_entries<'py>( .collect::>()?; for name in ordered_python_names(&selected, &upstream) { let node: Bound<'py, PyAny> = nodes.get_item(name.as_str())?; - let kind: Bound<'py, PyAny> = node.getattr("kind")?; + let kind: Bound<'py, PyAny> = native_field(&node, "kind")?; if !kinds.runs(&kind)? { continue; } - let identity: Bound<'py, PyAny> = node.getattr("identity")?; + let identity: Bound<'py, PyAny> = native_field(&node, "identity")?; let previous: Bound<'py, PyAny> = previous_identity(&identity, previous_identities)?; let attribute_or_none = |value: &Bound<'py, PyAny>, name: &str| { if value.is_none() { @@ -246,7 +247,7 @@ fn plan_entries<'py>( entries.push(call_with( &entry_class, &[ - ("name", node.getattr("name")?), + ("name", native_field(&node, "name")?), ("kind", kind), ("phase", phases.getattr(phase)?), ( @@ -269,7 +270,7 @@ fn plan_entries<'py>( "previous_metadata_json", attribute_or_none(&previous, "metadata_json")?, ), - ("provider_usages", node.getattr("provider_usages")?), + ("provider_usages", native_field(&node, "provider_usages")?), ], )?); } @@ -288,8 +289,8 @@ fn previous_identity<'py>( let key: Bound<'py, PyTuple> = PyTuple::new( identity.py(), [ - identity.getattr("node_type")?, - identity.getattr("node_name")?, + native_field(identity, "node_type")?, + native_field(identity, "node_name")?, ], )?; previous_identities.call_method1("get", (key,)) @@ -303,19 +304,19 @@ fn identity_status<'py>( let statuses: Bound<'py, PyAny> = registry::class(identity.py(), RecordId::PythonIdentityStatus)?; if identity.is_none() { - return statuses.getattr("UNKNOWN"); + return native_field(&statuses, "UNKNOWN"); } let previous: Bound<'py, PyAny> = previous_identity(identity, previous_identities)?; if previous.is_none() { - return statuses.getattr("NEW"); + return native_field(&statuses, "NEW"); } if previous .getattr("version_hash")? - .eq(identity.getattr("version_hash")?)? + .eq(native_field(identity, "version_hash")?)? { - return statuses.getattr("UNCHANGED"); + return native_field(&statuses, "UNCHANGED"); } - statuses.getattr("CHANGED") + native_field(&statuses, "CHANGED") } /// Warnings for skipped task/asset deps of planned source loaders. @@ -326,15 +327,15 @@ fn skipped_ingress_warnings<'py>( ) -> PyResult>> { let py: Python<'py> = plan_output.py(); let mut loaders: Vec = Vec::new(); - for entry in plan_output.getattr("source_load_entries")?.try_iter()? { - let loader: String = entry?.getattr("loader")?.extract()?; + for entry in native_field(plan_output, "source_load_entries")?.try_iter()? { + let loader: String = native_field(&entry?, "loader")?.extract()?; if !loaders.contains(&loader) { loaders.push(loader); } } loaders.sort(); - let nodes: Bound<'py, PyAny> = graph.getattr("nodes_by_name")?; - let selected: Bound<'py, PyAny> = run_selection.getattr("python_node_names")?; + let nodes: Bound<'py, PyAny> = native_field(graph, "nodes_by_name")?; + let selected: Bound<'py, PyAny> = native_field(run_selection, "python_node_names")?; let upstream: HashMap> = upstream_names(graph)?; let warning_class: Bound<'py, PyAny> = registry::class(py, RecordId::PlanWarning)?; let severity: Bound<'py, PyAny> = @@ -349,7 +350,7 @@ fn skipped_ingress_warnings<'py>( continue; } let kind: Bound<'py, PyAny> = - nodes.get_item(upstream_name.as_str())?.getattr("kind")?; + native_field(&nodes.get_item(upstream_name.as_str())?, "kind")?; if !kinds.runs(&kind)? { continue; } @@ -357,7 +358,7 @@ fn skipped_ingress_warnings<'py>( "Source loader '{loader}' has unselected upstream {} '{upstream_name}'. SQLBuild \ will not run it and cannot verify side effects or payload availability; use \ +source:{loader} to refresh upstream ingress dependencies.", - kind.getattr("value")?.str()? + native_field(&kind, "value")?.str()? ); warnings.push(call_with( &warning_class, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs index 917edd104b..620f94f94a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/registration.rs @@ -1,11 +1,12 @@ //! Register the native compile command driver. -use pyo3::prelude::{Bound, PyModule, PyResult}; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; use crate::bindings::_helpers::compile_driver::{ analysis, analysis_cache, assembly, command, deferral, diagnostic_collector, diff, discovery, input_reads, inputs, pipeline, pipeline_entries, plan_result, project, python_plan_entries, - relation_targets, selection_phases, targets, + relation_targets, selection_phases, stage_work, targets, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -26,5 +27,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { python_plan_entries::register(module)?; relation_targets::register(module)?; selection_phases::register(module)?; + module.add_function(wrap_pyfunction!( + stage_work::compile_stage_work_info, + module + )?)?; targets::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs index 03045bde39..92cadbfb21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::references::source_relations::render_source_relation; const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; @@ -27,9 +28,9 @@ impl<'py> RefKinds<'py> { fn load(py: Python<'py>) -> PyResult { let kinds: Bound<'py, PyAny> = host::entry(py, UserCodeHost::SqlResourceRefKind)?; Ok(Self { - model: kinds.getattr("MODEL")?, - seed: kinds.getattr("SEED")?, - source: kinds.getattr("SOURCE")?, + model: native_field(&kinds, "MODEL")?, + seed: native_field(&kinds, "SEED")?, + source: native_field(&kinds, "SOURCE")?, }) } } @@ -145,8 +146,8 @@ fn resolved_relation<'py>( if !planned.is_none() { return Ok(planned); } - let kind: Bound<'py, PyAny> = reference.getattr("kind")?; - let name: Bound<'py, PyAny> = reference.getattr("name")?; + let kind: Bound<'py, PyAny> = native_field(reference, "kind")?; + let name: Bound<'py, PyAny> = native_field(reference, "name")?; if kind.eq(&kinds.source)? { return Err(shared_error( py, @@ -164,8 +165,8 @@ fn resolved_relation<'py>( let mut location: Option> = None; for resource in resources.try_iter()? { let resource: Bound<'py, PyAny> = resource?; - if resource.getattr("name")?.eq(&name)? { - location = Some(resource.getattr("destination")?); + if native_field(&resource, "name")?.eq(&name)? { + location = Some(native_field(&resource, "destination")?); break; } } @@ -174,7 +175,7 @@ fn resolved_relation<'py>( py, format!( "Python node references unknown {} '{}'", - kind.getattr("value")?.str()?, + native_field(&kind, "value")?.str()?, name.str()? ), )); @@ -204,8 +205,8 @@ fn planned_relation<'py>( kinds: &RefKinds<'py>, ) -> PyResult> { let py: Python<'py> = adapter.py(); - let kind: Bound<'py, PyAny> = reference.getattr("kind")?; - let name: Bound<'py, PyAny> = reference.getattr("name")?; + let kind: Bound<'py, PyAny> = native_field(reference, "kind")?; + let name: Bound<'py, PyAny> = native_field(reference, "name")?; if kind.eq(&kinds.source)? { let source: Bound<'py, PyAny> = plan_output .getattr("python_source_entries")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs index 7776309998..edde609702 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/selection_phases.rs @@ -5,15 +5,28 @@ use pyo3::types::{PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::compile_driver::assembly::assemble_project; +use crate::bindings::_helpers::compile_driver::deferral::{ + build_deferred_locations, resolve_deferred_target_config, +}; +use crate::bindings::_helpers::compile_driver::pipeline::compile_project_phase; +use crate::bindings::_helpers::compile_driver::pipeline_entries::compile_analysis_selection; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_entries::command_plans::{ + resolve_static_command_scope, resolve_static_relation_context, +}; +use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; +use crate::bindings::_helpers::planner_selection::unit_tests::split_project_unit_test_selectors; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const DATACLASSES_MODULE: &str = "dataclasses"; /// Whether a selection names nothing to select or exclude. fn unselective(selection: &Bound<'_, PyAny>) -> PyResult { - Ok(!selection.getattr("select")?.is_truthy()? && !selection.getattr("exclude")?.is_truthy()?) + Ok(!native_field(selection, "select")?.is_truthy()? + && !native_field(selection, "exclude")?.is_truthy()?) } /// Options whose selectors name only resources, with unit tests selected apart. @@ -30,26 +43,23 @@ fn unit_test_selectors_applied<'py>( if unselective(&resolved)? { return Ok(resolved); } - let split: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::SplitProjectUnitTestSelectors)?, - &[ - ("select", resolved.getattr("select")?), - ("exclude", resolved.getattr("exclude")?), - ("discovered_inputs", discovered.clone()), - ( - "accepts_unit_tests", - resolved.getattr("accepts_unit_test_selectors")?, - ), - ], + let split: Bound<'py, PyAny> = split_project_unit_test_selectors( + &native_field(&resolved, "select")?, + &native_field(&resolved, "exclude")?, + discovered, + native_field(&resolved, "accepts_unit_test_selectors")?.extract()?, )?; imported(py, DATACLASSES_MODULE, "replace")?.call( (resolved,), Some(&keywords( py, &[ - ("select", split.getattr("select")?), - ("exclude", split.getattr("exclude")?), - ("sql_test_selection", split.getattr("sql_test_selection")?), + ("select", native_field(&split, "select")?), + ("exclude", native_field(&split, "exclude")?), + ( + "sql_test_selection", + native_field(&split, "sql_test_selection")?, + ), ], )?), ) @@ -61,28 +71,22 @@ fn planner_scopes<'py>( selection: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project.py(); - call_with( - ®istry::native(py, NativeAttr::ResolvePlannerScopes)?, - &[ - ("project", project.clone()), - ( - "selection", - call_with( - ®istry::class(py, RecordId::PlannerSelection)?, - &[ - ("select", selection.getattr("select")?), - ("exclude", selection.getattr("exclude")?), - ], - )?, - ), - ( - "policies", - call_with( - ®istry::class(py, RecordId::PlannerPolicies)?, - &[("auto_load_sources", selection.getattr("auto_load_sources")?)], - )?, - ), - ], + resolve_planner_scopes( + project, + &call_with( + ®istry::class(py, RecordId::PlannerSelection)?, + &[ + ("select", native_field(selection, "select")?), + ("exclude", native_field(selection, "exclude")?), + ], + )?, + &call_with( + ®istry::class(py, RecordId::PlannerPolicies)?, + &[( + "auto_load_sources", + native_field(selection, "auto_load_sources")?, + )], + )?, ) } @@ -96,7 +100,7 @@ fn scoped_models<'py>(scopes: &Bound<'py, PyAny>) -> PyResult> /// Return options whose selectors name only resources, with unit tests selected apart. #[pyfunction] #[pyo3(signature = (*, options, discovered_inputs))] -fn apply_unit_test_selectors<'py>( +pub(crate) fn apply_unit_test_selectors<'py>( options: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, ) -> PyResult> { @@ -106,7 +110,7 @@ fn apply_unit_test_selectors<'py>( /// Align configured-rule subjects with models that received deep SQL analysis. #[pyfunction] #[pyo3(signature = (*, options, discovered_inputs, project))] -fn resolve_configured_rule_selection<'py>( +pub(crate) fn resolve_configured_rule_selection<'py>( options: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, project: &Bound<'py, PyAny>, @@ -119,13 +123,7 @@ fn resolve_configured_rule_selection<'py>( { return Ok(PyFrozenSet::empty(py)?.into_any()); } - let selection: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveCompileAnalysisSelection)?, - &[ - ("options", options.clone()), - ("discovered_inputs", discovered_inputs.clone()), - ], - )?; + let selection: Bound<'py, PyAny> = compile_analysis_selection(options, discovered_inputs)?; if selection.is_none() || unselective(&selection)? { return Ok(py.None().into_bound(py)); } @@ -133,7 +131,7 @@ fn resolve_configured_rule_selection<'py>( let keys: Vec> = models .call_method0("values")? .try_iter()? - .map(|model| model?.getattr("key")) + .map(|model| native_field(&model?, "key")) .collect::>()?; Ok(PyFrozenSet::new(py, keys)?.into_any()) } @@ -141,7 +139,7 @@ fn resolve_configured_rule_selection<'py>( /// Return the models selected for deep analysis, or None to analyse every model. #[pyfunction] #[pyo3(signature = (*, compile_inputs, inference_profile, selection))] -fn resolve_analysis_model_names<'py>( +pub(crate) fn resolve_analysis_model_names<'py>( compile_inputs: &Bound<'py, PyAny>, inference_profile: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, @@ -150,17 +148,20 @@ fn resolve_analysis_model_names<'py>( if selection.is_none() || unselective(selection)? { return Ok(py.None().into_bound(py)); } - let structural_project: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AssembleProject)?, - &[ - ("inputs", compile_inputs.clone()), - ("inference_profile", inference_profile.clone()), - ( - "skip_column_inference", - pyo3::types::PyBool::new(py, true).to_owned().into_any(), - ), - ("analysis_model_names", PyFrozenSet::empty(py)?.into_any()), - ], + let structural_project: Bound<'py, PyAny> = assemble_project( + py, + Some(keywords( + py, + &[ + ("inputs", compile_inputs.clone()), + ("inference_profile", inference_profile.clone()), + ( + "skip_column_inference", + pyo3::types::PyBool::new(py, true).to_owned().into_any(), + ), + ("analysis_model_names", PyFrozenSet::empty(py)?.into_any()), + ], + )?), )?; let models: Bound<'py, PyAny> = scoped_models(&planner_scopes(&structural_project, selection)?)?; @@ -174,38 +175,45 @@ fn deferred_locations<'py>( project: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project.py(); - let defer_to: Bound<'py, PyAny> = options.getattr("defer_to")?; + let defer_to: Bound<'py, PyAny> = native_field(options, "defer_to")?; if defer_to.is_none() { return Ok(py.None().into_bound(py)); } - let deferred_target: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveDeferredTargetConfig)?, - &[ - ("discovered_inputs", discovered.clone()), - ("defer_to", defer_to), - ( - "current_target_name", - project.getattr("effective_target_name")?, - ), - ], + let deferred_target: Bound<'py, PyAny> = resolve_deferred_target_config( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("defer_to", defer_to), + ( + "current_target_name", + native_field(project, "effective_target_name")?, + ), + ], + )?), )?; - call_with( - ®istry::native(py, NativeAttr::BuildDeferredLocations)?, - &[ - ("project", project.clone()), - ("deferred_target_config", deferred_target), - ("effective_vars", project.getattr("effective_vars")?), - ("default_schema", adapter.call_method0("default_schema")?), - ( - "default_database", - adapter.call_method0("default_database")?, - ), - ( - "render_qualified_name", - adapter.getattr("render_qualified_name")?, - ), - ], - ) + Ok(build_deferred_locations( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("deferred_target_config", deferred_target), + ("effective_vars", native_field(project, "effective_vars")?), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ( + "render_qualified_name", + native_field(adapter, "render_qualified_name")?, + ), + ], + )?), + )? + .into_any()) } /// Run canonical compile, selector, and relation phases without build planning. @@ -231,67 +239,76 @@ pub(crate) fn compile_static_command_context<'py>( { selected_keys = PyFrozenSet::empty(py)?.into_any(); } - let compiled: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CompileProjectPhase)?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", adapter.clone()), - ("options", options.clone()), - ( - "on_progress", - request.optional("on_progress")?.unwrap_or_else(none), - ), - ], + let compiled: Bound<'py, PyAny> = compile_project_phase( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter.clone()), + ("options", options.clone()), + ( + "on_progress", + request.optional("on_progress")?.unwrap_or_else(none), + ), + ], + )?), )?; - let project: Bound<'py, PyAny> = compiled.getattr("project")?; + let project: Bound<'py, PyAny> = native_field(&compiled, "project")?; let deferred: Bound<'py, PyAny> = deferred_locations((&discovered, &adapter), &options, &project)?; - let scope: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveStaticCommandScope)?, - &[ - ("project", project.clone()), - ( - "selection", - call_with( - ®istry::class(py, RecordId::PlannerSelection)?, - &[ - ("select", options.getattr("select")?), - ("exclude", options.getattr("exclude")?), - ("selected_keys", selected_keys), - ("sql_test_selection", options.getattr("sql_test_selection")?), - ], - )?, - ), - ("auto_load_sources", options.getattr("auto_load_sources")?), - ], + let scope: Bound<'py, PyAny> = resolve_static_command_scope( + &project, + &call_with( + ®istry::class(py, RecordId::PlannerSelection)?, + &[ + ("select", native_field(&options, "select")?), + ("exclude", native_field(&options, "exclude")?), + ("selected_keys", selected_keys), + ( + "sql_test_selection", + native_field(&options, "sql_test_selection")?, + ), + ], + )?, + native_field(&options, "auto_load_sources")?.extract()?, )?; - let relations: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveStaticRelationContext)?, - &[ - ("project", project.clone()), - ("adapter", adapter), - ("scope", scope.clone()), - ( - "deferral", - call_with( - ®istry::class(py, RecordId::DeferralInputs)?, - &[ - ("deferred_locations", deferred), - ("defer_sources_to", options.getattr("defer_sources_to")?), - ( - "source_deferral_enabled", - options.getattr("source_deferral_enabled")?, - ), - ], - )?, - ), - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ( - "relation_keys", - request.optional("relation_keys")?.unwrap_or_else(none), - ), - ], + let relations: Bound<'py, PyAny> = resolve_static_relation_context( + py, + Some(keywords( + py, + &[ + ("project", project.clone()), + ("adapter", adapter), + ("scope", scope.clone()), + ( + "deferral", + call_with( + ®istry::class(py, RecordId::DeferralInputs)?, + &[ + ("deferred_locations", deferred), + ( + "defer_sources_to", + native_field(&options, "defer_sources_to")?, + ), + ( + "source_deferral_enabled", + native_field(&options, "source_deferral_enabled")?, + ), + ], + )?, + ), + ( + "project_config", + native_field(&discovered, "project_config")?, + ), + ("local_config", native_field(&discovered, "local_config")?), + ( + "relation_keys", + request.optional("relation_keys")?.unwrap_or_else(none), + ), + ], + )?), )?; call_with( ®istry::class(py, RecordId::StaticCommandContext)?, @@ -299,8 +316,14 @@ pub(crate) fn compile_static_command_context<'py>( ("project", project), ("scope", scope), ("relations", relations), - ("connection_config", compiled.getattr("connection_config")?), - ("compile_seconds", compiled.getattr("compile_seconds")?), + ( + "connection_config", + native_field(&compiled, "connection_config")?, + ), + ( + "compile_seconds", + native_field(&compiled, "compile_seconds")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs new file mode 100644 index 0000000000..2537090419 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/stage_work.rs @@ -0,0 +1,63 @@ +//! How often each compile stage ran in this process, for tests that assert stage work. + +use std::collections::BTreeMap; + +use pyo3::prelude::{Py, PyRefMut, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::{pyclass, pyfunction}; + +pub(crate) const EXPRESSION_SHAPES: &str = "infer_native_expression_source_shapes"; +pub(crate) const EXPRESSION_SHAPE_BATCHES: &str = "expression_shape_batches"; +pub(crate) const EXPRESSION_SHAPES_INFERRED: &str = "expression_shapes_inferred"; +pub(crate) const COLUMN_LINEAGE: &str = "build_project_column_lineage"; +pub(crate) const MODEL_CONTRACTS: &str = "evaluate_model_contracts"; +pub(crate) const PROJECT_DISCOVERIES: &str = "discover_project_inputs"; +pub(crate) const MODEL_ANALYSIS: &str = "analyze_native_model_sql"; +pub(crate) const SEMANTIC_COMPLETION: &str = "complete_semantic_project"; +pub(crate) const SQL_TEST_ASSEMBLY: &str = "assemble_native_sql_tests"; +pub(crate) const PROMOTION_CONFLICTS: &str = "promotion_conflict_diagnostics"; +pub(crate) const PIVOT_PROOFS: &str = "pivot_proofs"; +pub(crate) const SESSION_PIVOT_PROOFS: &str = "session_pivot_proofs"; +pub(crate) const DECLARATION_SCOPES: &str = "build_declaration_scope"; +pub(crate) const LINEAGE_GRAPH_BUILDS: &str = "lineage_graph_builds"; + +/// Compile stage runs since the last reset, by stage. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct StageWork { + runs: BTreeMap<&'static str, u64>, +} + +static STAGE_WORK: PyOnceLock> = PyOnceLock::new(); + +fn stage_work(py: Python<'_>) -> PyResult> { + Ok(STAGE_WORK + .get_or_try_init(py, || Py::new(py, StageWork::default()))? + .bind(py) + .borrow_mut()) +} + +/// Add `count` runs of `stage`. +pub(crate) fn count_stage(py: Python<'_>, stage: &'static str, count: u64) -> PyResult<()> { + *stage_work(py)?.runs.entry(stage).or_default() += count; + Ok(()) +} + +/// Report the runs of each compile stage so far, optionally resetting them. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn compile_stage_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult> { + let mut work = stage_work(py)?; + let info: BTreeMap = work + .runs + .iter() + .map(|(stage, count)| ((*stage).to_owned(), *count)) + .collect(); + if reset { + *work = StageWork::default(); + } + Ok(info) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index e0cc7f661d..e15fb7b9c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -4,8 +4,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::function_views::NativeCompiledFunction; use crate::bindings::_helpers::compiled_project::records::project_with; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compiled_project::seed_views::NativeCompiledSeed; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// The adapter defaults every compiled location falls back to. @@ -18,11 +22,11 @@ pub(crate) struct TargetDefaults<'py> { impl<'py> TargetDefaults<'py> { /// `(default_schema, default_database)`. - fn pair(&self) -> PyResult> { - PyTuple::new( - self.default_schema.py(), - [self.default_schema.clone(), self.default_database.clone()], - ) + fn pair(&self) -> PyResult<(Option, Option)> { + Ok(( + self.default_schema.extract()?, + self.default_database.extract()?, + )) } } @@ -41,38 +45,46 @@ pub(crate) fn apply_target_defaults<'py>( defaults: &TargetDefaults<'py>, ) -> PyResult> { let py: Python<'py> = project.py(); - let compiled_model: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledModel)?; - let native_seed: Bound<'py, PyAny> = registry::native(py, NativeAttr::NativeCompiledSeed)?; let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; let mut models: Vec> = Vec::new(); - for model in project.getattr("models")?.try_iter()? { + for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - models.push(if model.is_instance(&compiled_model)? { + models.push(if let Ok(native) = model.cast::() { + Bound::new( + py, + native.get().with_target_defaults( + py, + defaults.pair()?, + defaults.render_qualified_name.clone(), + )?, + )? + .into_any() + } else { replace.call( (&model,), Some(&keywords( py, &[( "destination", - resolved_target(&model.getattr("destination")?, defaults)?, + resolved_target(&native_field(&model, "destination")?, defaults)?, )], )?), )? - } else { - model.call_method1( - "with_target_defaults", - (defaults.pair()?, &defaults.render_qualified_name), - )? }); } let mut seeds: Vec> = Vec::new(); - for seed in project.getattr("seeds")?.try_iter()? { + for seed in native_field(project, "seeds")?.try_iter()? { let seed: Bound<'py, PyAny> = seed?; - seeds.push(if seed.is_instance(&native_seed)? { - seed.call_method1( - "with_target_defaults", - (defaults.pair()?, &defaults.render_qualified_name), + seeds.push(if let Ok(native) = seed.cast::() { + Bound::new( + py, + native.get().with_target_defaults( + py, + defaults.pair()?, + defaults.render_qualified_name.clone(), + )?, )? + .into_any() } else { replace.call( (&seed,), @@ -80,22 +92,30 @@ pub(crate) fn apply_target_defaults<'py>( py, &[( "destination", - resolved_target(&seed.getattr("destination")?, defaults)?, + resolved_target(&native_field(&seed, "destination")?, defaults)?, )], )?), )? }); } let mut functions: Vec> = Vec::new(); - for function in project.getattr("functions")?.try_iter()? { - functions.push(function?.call_method1( - "with_target_defaults", - ( - defaults.pair()?, - &defaults.render_qualified_name, - &defaults.python_functions_inherit_default_namespace, - ), - )?); + for function in native_field(project, "functions")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + let native: &Bound<'py, NativeCompiledFunction> = function.cast()?; + functions.push( + Bound::new( + py, + native.get().with_target_defaults( + py, + defaults.pair()?, + defaults.render_qualified_name.clone(), + defaults + .python_functions_inherit_default_namespace + .extract()?, + )?, + )? + .into_any(), + ); } project_with( &project.clone(), @@ -105,14 +125,14 @@ pub(crate) fn apply_target_defaults<'py>( ( "effective_target_database", or_else( - project.getattr("effective_target_database")?, + native_field(project, "effective_target_database")?, &defaults.default_database, )?, ), ( "effective_target_schema", or_else( - project.getattr("effective_target_schema")?, + native_field(project, "effective_target_schema")?, &defaults.default_schema, )?, ), @@ -130,8 +150,8 @@ fn resolved_target<'py>( defaults: &TargetDefaults<'py>, ) -> PyResult> { let py: Python<'py> = target.py(); - let target_schema: Bound<'py, PyAny> = target.getattr("schema")?; - let target_database: Bound<'py, PyAny> = target.getattr("database")?; + let target_schema: Bound<'py, PyAny> = native_field(target, "schema")?; + let target_database: Bound<'py, PyAny> = native_field(target, "database")?; let schema: Bound<'py, PyAny> = if target_schema.is_none() { defaults.default_schema.clone() } else { @@ -142,7 +162,7 @@ fn resolved_target<'py>( } else { target_database.clone() }; - let name: Bound<'py, PyAny> = target.getattr("name")?; + let name: Bound<'py, PyAny> = native_field(target, "name")?; let qualified_name: Bound<'py, PyAny> = call_with( &defaults.render_qualified_name, &[ @@ -153,7 +173,7 @@ fn resolved_target<'py>( )?; if schema.eq(&target_schema)? && database.eq(&target_database)? - && qualified_name.eq(target.getattr("qualified_name")?)? + && qualified_name.eq(native_field(target, "qualified_name")?)? { return Ok(target.clone()); } @@ -164,8 +184,11 @@ fn resolved_target<'py>( ("schema", schema), ("name", name), ("qualified_name", qualified_name), - ("logical_schema", target.getattr("logical_schema")?), - ("logical_database", target.getattr("logical_database")?), + ("logical_schema", native_field(target, "logical_schema")?), + ( + "logical_database", + native_field(target, "logical_database")?, + ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 9e1e0ecd0d..26249342fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -42,6 +42,7 @@ use crate::bindings::_helpers::compile_attachment::effective_config::build_effec use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::discovery::guidance::setting_help; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -128,8 +129,8 @@ pub(crate) fn named_target_schema_strategy( { return Ok(()); } - let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; + let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; let target_name: Bound<'_, PyAny> = resolve_target_name( py, &project_config.clone(), @@ -143,7 +144,7 @@ pub(crate) fn named_target_schema_strategy( let target_config: Bound<'_, PyAny> = resolve_target_config(py, &project_config, &local_config, &target_name.clone()) .and_then(|value| value.into_bound_py_any(py))?; - let schema: Option = text_or_none(&target_config.getattr("schema")?)?; + let schema: Option = text_or_none(&native_field(&target_config, "schema")?)?; if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { return Ok(()); } @@ -185,10 +186,10 @@ pub(crate) fn managed_loader_target_isolation( ) -> PyResult<()> { let py: Python<'_> = discovered.py(); let mut managed_sources: Vec> = Vec::new(); - for source_file in discovered.getattr("source_files")?.try_iter()? { - for entry in source_file?.getattr("source_entries")?.try_iter()? { + for source_file in native_field(discovered, "source_files")?.try_iter()? { + for entry in native_field(&source_file?, "source_entries")?.try_iter()? { let entry: Bound<'_, PyAny> = entry?; - if entry.getattr("managed")?.is_truthy()? { + if native_field(&entry, "managed")?.is_truthy()? { managed_sources.push(entry); } } @@ -196,11 +197,11 @@ pub(crate) fn managed_loader_target_isolation( if managed_sources.is_empty() { return Ok(()); } - let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; + let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; let mut target_names: Vec = Vec::new(); for config in [&project_config, &local_config] { - for name in config.getattr("targets")?.try_iter()? { + for name in native_field(config, "targets")?.try_iter()? { let name: String = name?.extract()?; if !target_names.contains(&name) { target_names.push(name); @@ -261,8 +262,8 @@ fn target_namespaces( let name: Bound<'_, PyAny> = PyString::new(py, target_name).into_any(); let target_config: Bound<'_, PyAny> = resolve_target_config( py, - &discovered.getattr("project_config")?, - &discovered.getattr("local_config")?, + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, &name.clone(), ) .and_then(|value| value.into_bound_py_any(py))?; @@ -329,17 +330,17 @@ fn managed_source_namespace( let py: Python<'_> = source_entry.py(); let database: Bound<'_, PyAny> = first_truthy( &[ - source_entry.getattr("database")?, - target_config.getattr("database")?, + native_field(source_entry, "database")?, + native_field(target_config, "database")?, connection.call_method1("get", ("database",))?, ], || adapter.call_method0("default_database"), )?; let schema: Bound<'_, PyAny> = first_truthy( &[ - source_entry.getattr("schema")?, - target_config.getattr("loader_schema")?, - target_config.getattr("schema")?, + native_field(source_entry, "schema")?, + native_field(target_config, "loader_schema")?, + native_field(target_config, "schema")?, connection.call_method1("get", ("schema",))?, ], || adapter.call_method0("default_schema"), @@ -405,8 +406,11 @@ pub(crate) fn project_targets( let mut targets: Vec<(String, Bound<'_, PyAny>)> = Vec::new(); for resource in project.getattr(attribute)?.try_iter()? { let resource: Bound<'_, PyAny> = resource?; - let resource_name: String = resource.getattr("name")?.str()?.to_cow()?.into_owned(); - let destination: Bound<'_, PyAny> = resource.getattr("destination")?; + let resource_name: String = native_field(&resource, "name")? + .str()? + .to_cow()? + .into_owned(); + let destination: Bound<'_, PyAny> = native_field(&resource, "destination")?; match targets .iter_mut() .find(|(existing, _)| *existing == resource_name) @@ -429,8 +433,8 @@ fn required_target_parts( target: &Bound<'_, PyAny>, ) -> PyResult<()> { let py: Python<'_> = target.py(); - let database: Option = optional_text(&target.getattr("database")?)?; - let schema: Option = optional_text(&target.getattr("schema")?)?; + let database: Option = optional_text(&native_field(target, "database")?)?; + let schema: Option = optional_text(&native_field(target, "schema")?)?; let missing: Vec<&str> = missing_target_parts(database.as_deref(), schema.as_deref()); let Some(first) = missing.first() else { return Ok(()); @@ -516,24 +520,27 @@ pub(crate) fn managed_write_schemas( for resource in project.getattr(attribute)?.try_iter()? { let resource: Bound<'_, PyAny> = resource?; managed_write_schema( - &format!("{label} '{}'", resource.getattr("name")?.str()?.to_cow()?), - &resource.getattr("destination")?.getattr("schema")?, + &format!( + "{label} '{}'", + native_field(&resource, "name")?.str()?.to_cow()? + ), + &native_field(&resource, "destination")?.getattr("schema")?, connection_schema.as_deref(), )?; } } - for source in project.getattr("sources")?.try_iter()? { + for source in native_field(project, "sources")?.try_iter()? { let source: Bound<'_, PyAny> = source?; - let entry: Bound<'_, PyAny> = source.getattr("source_entry")?; - if !entry.getattr("managed")?.is_truthy()? { + let entry: Bound<'_, PyAny> = native_field(&source, "source_entry")?; + if !native_field(&entry, "managed")?.is_truthy()? { continue; } managed_write_schema( &format!( "Managed source '{}'", - source.getattr("name")?.str()?.to_cow()? + native_field(&source, "name")?.str()?.to_cow()? ), - &entry.getattr("schema")?, + &native_field(&entry, "schema")?, connection_schema.as_deref(), )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index fc12fc6178..f6cf74c7a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -944,6 +944,12 @@ const R_DISCOVERED_LOADER_FUNCTION: RecordSpec = RecordSpec { FieldDefault::NativeMember(RecordId::LoaderConnectionMode, "SQLBUILD"), ), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "identity", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DISCOVERED_TASK_FUNCTION: RecordSpec = RecordSpec { @@ -961,6 +967,12 @@ const R_DISCOVERED_TASK_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("meta", FieldDefault::None), FieldSpec::new("retry", FieldDefault::None), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "identity", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DISCOVERED_ASSET_FUNCTION: RecordSpec = RecordSpec { @@ -980,6 +992,12 @@ const R_DISCOVERED_ASSET_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("column_lineage", FieldDefault::None), FieldSpec::new("retry", FieldDefault::None), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "identity", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DISCOVERED_CHECK_FUNCTION: RecordSpec = RecordSpec { @@ -1000,6 +1018,12 @@ const R_DISCOVERED_CHECK_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("description", FieldDefault::None), FieldSpec::new("meta", FieldDefault::None), FieldSpec::new("provider_usages", FieldDefault::EmptyTuple), + FieldSpec { + name: "identity", + default: FieldDefault::None, + compared: false, + shown: false, + }, ], }; const R_DISCOVERED_HOOK_FUNCTION: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index f5960d374e..a06b43dfc4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -806,7 +806,7 @@ impl NativeCompiledModel { } /// This model with the adapter's default schema and database filled into its destination. - fn with_target_defaults( + pub(crate) fn with_target_defaults( &self, py: Python<'_>, defaults: (Option, Option), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs index d4544b9531..7c265a45e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ }; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::graph::project_graph::ResourceRow; const TABLE_FUNCTION_TYPE: &str = "table_fn"; const SCALAR_FUNCTION_TYPE: &str = "udf"; @@ -424,6 +425,25 @@ impl NativeCompiledFunction { Self::defaulted(owner, index, None) } + /// The function's project graph row: its key, the resources it reads and its tags. + pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { + read_function(&self.owner, self.index, py, |function, _| { + let deps: &[ObjectKeyRow] = function + .deps + .as_deref() + .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; + Ok(( + ( + function.attached.node_type().to_owned(), + function.attached.name.clone(), + ), + deps.to_vec(), + function.attached.tags.clone(), + None, + )) + }) + } + fn defaulted( owner: Py, index: usize, @@ -535,7 +555,7 @@ function_getters!(NativeCompiledFunction, { } /// This function with target defaults; a non-inheriting Python function is name-qualified. - fn with_target_defaults( + pub(crate) fn with_target_defaults( &self, py: Python<'_>, defaults: (Option, Option), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index 88ff02b850..59376c3924 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -6,13 +6,17 @@ use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{pyclass, pymethods}; +use crate::bindings::_helpers::compile_driver::stage_work::{LINEAGE_GRAPH_BUILDS, count_stage}; use crate::bindings::_helpers::compiled_project::compiled_model_view::{ NativeCompiledModel, model_graph_row, }; +use crate::bindings::_helpers::compiled_project::function_views::NativeCompiledFunction; use crate::bindings::_helpers::compiled_project::model_input_view::{ InputViewClasses, authored_expansion, view_classes, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::seed_views::NativeCompiledSeed; +use crate::bindings::_helpers::compiled_project::source_views::NativeCompiledSource; use crate::bindings::_helpers::graph::project_graph::{ NativeProjectGraph, ResourceRow, project_graph, }; @@ -521,21 +525,36 @@ impl NativeProjectView { } let compiler: &CompilerHandles = &self.fields.compiler; for source in compiler.sources.bind(py).try_iter()? { - rows.push(resource_row(&source?, None)?); + let source: Bound<'_, PyAny> = source?; + rows.push(match source.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => resource_row(&source, None)?, + }); } for seed in compiler.seeds.bind(py).try_iter()? { let seed: Bound<'_, PyAny> = seed?; - let tags: Bound<'_, PyAny> = seed - .getattr(SCHEMA_ENTRY_ATTRIBUTE)? - .getattr(TAGS_ATTRIBUTE)?; - rows.push(resource_row(&seed, Some(&tags))?); + rows.push(match seed.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => { + let tags: Bound<'_, PyAny> = seed + .getattr(SCHEMA_ENTRY_ATTRIBUTE)? + .getattr(TAGS_ATTRIBUTE)?; + resource_row(&seed, Some(&tags))? + } + }); } for function in compiler.functions.bind(py).try_iter()? { let function: Bound<'_, PyAny> = function?; - let tags: Bound<'_, PyAny> = function.getattr(TAGS_ATTRIBUTE)?; - rows.push(resource_row(&function, Some(&tags))?); + rows.push(match function.cast::() { + Ok(native) => native.get().graph_row(py)?, + Err(_) => { + let tags: Bound<'_, PyAny> = function.getattr(TAGS_ATTRIBUTE)?; + resource_row(&function, Some(&tags))? + } + }); } self.owner.bind(py).borrow_mut().record_graph_build(); + count_stage(py, LINEAGE_GRAPH_BUILDS, 1)?; Py::new(py, project_graph(rows)?) }) .map(|graph| graph.clone_ref(py)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs index 327b430fe6..e270cd4b0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs @@ -14,8 +14,10 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ InputViewClasses, view_classes, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::graph::project_graph::ResourceRow; const SEED_RESOURCE: &str = "seed"; +const TAGS_ATTRIBUTE: &str = "tags"; const NAME_ATTRIBUTE: &str = "name"; const FILE_PATH_ATTRIBUTE: &str = "file_path"; const STEM_ATTRIBUTE: &str = "stem"; @@ -192,6 +194,27 @@ impl NativeCompiledSeed { } } + /// The seed's project graph row: its key and its declared tags. + pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { + read_seed(&self.owner, self.index, py, |seed, py| { + let mut tags: Vec = Vec::new(); + for tag in seed + .schema_entry + .bind(py) + .getattr(TAGS_ATTRIBUTE)? + .try_iter()? + { + tags.push(tag?.str()?.extract()?); + } + Ok(( + (SEED_RESOURCE.to_owned(), seed.name.clone()), + Vec::new(), + tags, + None, + )) + }) + } + fn current_destination(&self, py: Python<'_>) -> PyResult { if let Some(defaulted) = &self.defaulted { return Ok(defaulted.clone()); @@ -262,7 +285,7 @@ impl NativeCompiledSeed { } /// This seed with the adapter's default schema and database filled into its destination. - fn with_target_defaults( + pub(crate) fn with_target_defaults( &self, py: Python<'_>, defaults: (Option, Option), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index f21ce5a178..62c118242b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ InputViewClasses, view_classes, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::records::fields::field as native_field; const SOURCE_RESOURCE: &str = "source"; @@ -203,6 +204,18 @@ impl NativeCompiledSource { pub(crate) fn new(owner: Py, index: usize) -> Self { Self { owner, index } } + + /// The source's project graph row: its key, with no dependencies, tags or folder. + pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { + read_source(&self.owner, self.index, py, |source, _| { + Ok(( + (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), + Vec::new(), + Vec::new(), + None, + )) + }) + } } #[pymethods] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs index 17cb132b3c..f31b913f53 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_enums/constants.rs @@ -79,6 +79,21 @@ const E_COLUMN_LINEAGE_CONFIDENCE: EnumSpec = EnumSpec { ("UNKNOWN", "unknown"), ], }; +const E_COMPILED_RESOURCE_TYPE: EnumSpec = EnumSpec { + name: "CompiledResourceType", + doc: "", + members: &[ + ("MODEL", "model"), + ("SOURCE", "source"), + ("SEED", "seed"), + ("UDF", "udf"), + ("TABLE_FN", "table_fn"), + ("DBT_REF", "dbt_ref"), + ("AUDIT", "audit"), + ("SQL_TEST", "sql_test"), + ("SQL_SCENARIO", "sql_scenario"), + ], +}; const E_INFERRED_NULLABILITY: EnumSpec = EnumSpec { name: "InferredNullability", doc: "Conservative nullability state inferred for an output column.", @@ -725,6 +740,7 @@ pub(crate) const COMPILER_ENUMS: &[EnumSpec] = &[ E_COLUMN_LINEAGE_CONFIDENCE, E_COLUMN_LINEAGE_MODE, E_INFERRED_NULLABILITY, + E_COMPILED_RESOURCE_TYPE, E_POLYGLOT_ANALYSIS_DIALECT, E_MIGRATION_DISCOVERY, E_MIGRATION_COMPATIBILITY, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index f4d373bc63..6cef6bd314 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -209,33 +209,6 @@ const R_MANUAL_LOCATION: RecordSpec = RecordSpec { FieldSpec::required("reason"), ], }; -const R_NATIVE_SQL_TEST_PLANNING_REQUEST: RecordSpec = RecordSpec { - name: "NativeSqlTestPlanningRequest", - doc: "Compiled models and tests to plan, with the adapter facts native planning cannot derive.", - fields: &[ - FieldSpec::required("models"), - FieldSpec::required("rendered_model_sql"), - FieldSpec::required("functions"), - FieldSpec::required("tests"), - FieldSpec::required("rendered_test_overrides"), - FieldSpec::required("sql_analysis_enabled"), - FieldSpec::required("sql_analysis_dialect"), - FieldSpec::required("set_difference_operator"), - FieldSpec::required("requires_derived_table_aliases"), - FieldSpec::required("lexical_syntax"), - FieldSpec::required("render_sql"), - FieldSpec::required("include_plan"), - ], -}; -const R_NATIVE_SQL_TEST_CHAIN_REQUEST: RecordSpec = RecordSpec { - name: "NativeSqlTestChainRequest", - doc: "Compiled models and the tests whose unmocked model chains to resolve.", - fields: &[ - FieldSpec::required("models"), - FieldSpec::required("tests"), - FieldSpec::required("lexical_syntax"), - ], -}; const R_NATIVE_SQL_TEST_FAILURE: RecordSpec = RecordSpec { name: "NativeSqlTestFailure", doc: "The error one test's assembly raises: its kind and message.", @@ -889,19 +862,6 @@ const R_SQL_LEXICAL_SYNTAX: RecordSpec = RecordSpec { ), ], }; -const R_PROJECT_GRAPH: RecordSpec = RecordSpec { - name: "ProjectGraph", - doc: "Static compiled project graph held natively; dict views are built only when read.", - fields: &[ - FieldSpec::required("project"), - FieldSpec { - name: "native", - default: FieldDefault::Required, - compared: false, - shown: false, - }, - ], -}; const R_COMPILE_PIPELINE_RESULT: RecordSpec = RecordSpec { name: "CompilePipelineResult", doc: "Complete output from the compile-and-plan pipeline.", @@ -936,8 +896,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_MODEL_COLUMN_LINEAGE, R_REFACTOR_PROJECT, R_MANUAL_LOCATION, - R_NATIVE_SQL_TEST_PLANNING_REQUEST, - R_NATIVE_SQL_TEST_CHAIN_REQUEST, R_NATIVE_SQL_TEST_FAILURE, R_NATIVE_SQL_TEST_ASSEMBLY, R_HEADER_SPAN, @@ -993,6 +951,5 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_PYTHON_IDENTITY_DEPENDENCY, R_PYTHON_NODE_IDENTITY, R_SQL_LEXICAL_SYNTAX, - R_PROJECT_GRAPH, R_COMPILE_PIPELINE_RESULT, ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs index cadaab4d69..b6599f754c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -20,6 +20,9 @@ use sqlbuild_core::contract_command::models::{ use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::contract_command::write::{ContractWrite, write_contracts}; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; @@ -80,19 +83,22 @@ fn select_resources<'py>( ®istry::class(py, RecordId::CompileAnalysisSelection)?, &[("no_cache", PyBool::new(py, true).to_owned().into_any())], )?; - let project: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter", native_field(context, "adapter")?), - ("selected_target", native_field(request, "from_target")?), - ("cli_vars", native_field(request, "cli_vars")?), - ( - "resolved_connection", - native_field(context, "connection_config")?, - ), - ("analysis_selection", analysis), - ], + let project: Bound<'py, PyAny> = build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", native_field(context, "adapter")?), + ("selected_target", native_field(request, "from_target")?), + ("cli_vars", native_field(request, "cli_vars")?), + ( + "resolved_connection", + native_field(context, "connection_config")?, + ), + ("analysis_selection", analysis), + ], + )?), )?; let graph: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::BuildProjectGraphFromCompiledProject)?, @@ -366,9 +372,9 @@ fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult< } else { imported(py, "pathlib", "Path")?.call_method0("cwd")? }; - let discovered: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'_, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; call_with( ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs index 2233529140..5bc9f1632c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs @@ -4,12 +4,14 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const UTF8: &str = "utf-8"; /// The repository the edits apply to and how they apply. @@ -32,7 +34,7 @@ pub(crate) fn write_contracts<'py>( let models: Bound<'py, PyDict> = by_name(&native_field(&project, "models")?)?; let sources: Bound<'py, PyDict> = by_name(&native_field(&project, "sources")?)?; let model_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let updates: Bound<'py, PyDict> = PyDict::new(py); let updated_evidence: Bound<'py, PyList> = PyList::empty(py); for evidence in native_field(result, "evidence")?.try_iter()? { @@ -130,27 +132,30 @@ fn applied<'py>( ) -> PyResult<()> { let py: Python<'py> = result.py(); write_all(updates)?; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", write.project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", write.project_dir.clone())])?), )?; if write.adapter.is_none() { return Ok(()); } - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("no_cache", true)?; - let selection: Bound<'py, PyAny> = - registry::class(py, RecordId::CompileAnalysisSelection)?.call((), Some(&keywords))?; - call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", discovered), - ("adapter", write.adapter.clone()), - ("selected_target", native_field(result, "from_target")?), - ("resolved_connection", write.resolved_connection.clone()), - ("cli_vars", write.cli_vars.clone()), - ("analysis_selection", selection), - ], + let selection_keywords: Bound<'py, PyDict> = PyDict::new(py); + selection_keywords.set_item("no_cache", true)?; + let selection: Bound<'py, PyAny> = registry::class(py, RecordId::CompileAnalysisSelection)? + .call((), Some(&selection_keywords))?; + build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered), + ("adapter", write.adapter.clone()), + ("selected_target", native_field(result, "from_target")?), + ("resolved_connection", write.resolved_connection.clone()), + ("cli_vars", write.cli_vars.clone()), + ("analysis_selection", selection), + ], + )?), )?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 66a5399376..0a8084a1e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -16,7 +16,6 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sqltext::model_headers::{ColumnLocation, location_rows}; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PATHLIB_MODULE: &str = "pathlib"; const IMPLICIT_COLUMN_CONTRACTS: &str = "implicit"; const GENERIC_DIALECT: &str = "generic"; @@ -155,15 +154,13 @@ fn contract_diagnostic<'py>( } fn model_resource_type(py: Python<'_>) -> PyResult> { - py.import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")? - .call1((MODEL_RESOURCE,)) + registry::class(py, RecordId::CompiledResourceType)?.call1((MODEL_RESOURCE,)) } /// Every model's column contract diagnostics, in model order. #[pyfunction] #[pyo3(signature = (*, project, dialect = None))] -fn evaluate_model_contracts<'py>( +pub(crate) fn evaluate_model_contracts<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, dialect: Option<&Bound<'py, PyAny>>, @@ -206,7 +203,7 @@ fn evaluate_model_contracts<'py>( /// One K011 error per enforced-contract table model under immediate promotion. #[pyfunction] #[pyo3(signature = (*, project, adapter_default, settings_file))] -fn promotion_conflict_diagnostics<'py>( +pub(crate) fn promotion_conflict_diagnostics<'py>( py: Python<'py>, project: &Bound<'py, PyAny>, adapter_default: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs index 7370c4c364..9d5236873c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/project_config.rs @@ -186,7 +186,7 @@ pub(crate) fn load_local_config_file( /// The project or local configuration file path, rejecting an unconverted legacy file. #[pyfunction] -fn resolve_config_file_path<'py>( +pub(crate) fn resolve_config_file_path<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, local: bool, @@ -200,7 +200,7 @@ fn resolve_config_file_path<'py>( /// `validate_local_sql_analysis_policy` from the project requirement and the local setting. #[pyfunction] -fn validate_local_sql_analysis( +pub(crate) fn validate_local_sql_analysis( py: Python<'_>, project_dir: &Bound<'_, PyAny>, flags: (bool, bool, bool), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs index f19c8b5671..212835ff19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/resource_identities.rs @@ -15,7 +15,7 @@ const UDF_NODE_TYPE: &str = "udf"; /// Reject one identity unless it uses canonical lowercase snake_case. #[pyfunction] #[pyo3(signature = (*, name, kind, path, private_identity = false))] -fn validate_resource_identity( +pub(crate) fn validate_resource_identity( name: &Bound<'_, PyString>, kind: &str, path: &Bound<'_, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs index 8be885312e..a262bdf308 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs @@ -143,7 +143,7 @@ fn override_or<'py>(local: Bound<'py, PyAny>, project: Bound<'py, PyAny>) -> Bou /// Resolve scenario config, allowing local overrides to replace project rules. #[pyfunction] #[pyo3(signature = (*, project_config, local_config))] -fn resolve_effective_scenario_config<'py>( +pub(crate) fn resolve_effective_scenario_config<'py>( project_config: &Bound<'py, PyAny>, local_config: &Bound<'py, PyAny>, ) -> PyResult> { @@ -246,7 +246,7 @@ fn loader_destination_parts<'py>( loader_parts(py, destination, (default_database, default_schema)) } -fn loader_parts<'py>( +pub(crate) fn loader_parts<'py>( py: Python<'py>, destination: &str, (default_database, default_schema): (Option>, Option>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index 4d0feb89e9..6911b73225 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -10,6 +10,8 @@ use sqlbuild_core::cost_command::main::accepts_cost_refresh::accepts_cost_refres use sqlbuild_core::cost_command::models::CostCoverage; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::cost_telemetry::values::cost_aware; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -107,9 +109,9 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { return Ok(()); } let target_name: Bound<'_, PyAny> = native_field(&record, "target_name")?; - let discovered: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'_, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let target_config: Bound<'_, PyAny> = call_with( ®istry::native(py, NativeAttr::BuildEffectiveTargetConfig)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs index a45f185962..42aede93ad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs @@ -139,7 +139,10 @@ impl CostContext { #[staticmethod] #[pyo3(signature = (**options))] - fn resource_scope(py: Python<'_>, options: Option>) -> PyResult { + pub(crate) fn resource_scope( + py: Python<'_>, + options: Option>, + ) -> PyResult { let arguments: KeywordArguments<'_> = KeywordArguments::read( py, "resource_scope", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs index a4a36ea427..84e847763f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dag_artifact/artifact.rs @@ -2,7 +2,7 @@ use pyo3::exceptions::{PyKeyError, PyRecursionError, PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult}; -use pyo3::{PyRef, pyfunction, wrap_pyfunction}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::dag_artifact::errors::DagArtifactError; use sqlbuild_analysis::dag_artifact::main::build_dag_json::build_dag_json as build; use sqlbuild_analysis::dag_artifact::models::{DagProjectInput, DagPythonGraphInput}; @@ -10,8 +10,7 @@ use sqlbuild_core::json::errors::JsonEmitError; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::dag_artifact::rows::{project_input, python_graph_input}; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::graph::project_graph::CompiledProjectGraph; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; @@ -39,15 +38,19 @@ pub(crate) fn build_dag_json( ) -> PyResult { compiler_guard(|| { let py = graph.py(); - let project: DagProjectInput = project_input(&native_field(graph, "project")?)?; + let graph: &Bound<'_, CompiledProjectGraph> = graph.cast()?; + let project: DagProjectInput = project_input(graph.get().project(py))?; let python: Option = python_graph .filter(|python_graph| !python_graph.is_none()) .map(python_graph_input) .transpose()?; - let native: Bound<'_, PyAny> = native_field(graph, "native")?; - let native: PyRef<'_, NativeProjectGraph> = native.extract()?; - build(native.graph(), &project, python.as_ref(), project_name) - .map_err(|error| python_error(error, py)) + build( + graph.get().native(py).get().graph(), + &project, + python.as_ref(), + project_name, + ) + .map_err(|error| python_error(error, py)) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs index efd3badb26..5a66b8c059 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs @@ -9,6 +9,8 @@ use sqlbuild_core::dbt_cli::main::dbt_debug_args::dbt_debug_args; use std::time::Instant; use crate::bindings::_helpers::build_command::support::{boolean, none, text}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::dbt_interop::pipeline_prologue::progress; use crate::bindings::_helpers::dbt_interop::runtime_config::config_overrides; @@ -322,10 +324,12 @@ fn plan_dbt_interop_from_project<'py>( #[pyo3(signature = (**options))] fn debug_dbt_from_project(py: Python<'_>, options: Option>) -> PyResult { let args = KeywordArguments::read(py, "debug_dbt_from_project", options, DEBUG_OPTIONS)?; - let discovered = native( + let discovered = discover_project_inputs( py, - NativeAttr::DiscoverProjectInputs, - &[("project_dir", args.value("project_dir")?)], + Some(keywords( + py, + &[("project_dir", args.value("project_dir")?)], + )?), )?; let dbt_args: Vec = args .value("args")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs index a6c4a7b2ac..207cf71e76 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_prologue.rs @@ -1,5 +1,8 @@ //! Discover, compile and resolve the combined dbt pipeline through its existing boundaries. +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::project::build_compiled_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -78,9 +81,9 @@ pub(crate) fn resolve_dbt_execution_invocation<'py>( &[("command", command), ("parsed", parsed)], )?; let project_dir = native_field(request, "project_dir")?; - let discovered = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let dbt_args: Vec = routed .getattr("dbt_args")? @@ -150,12 +153,7 @@ fn load_compiled_dbt_manifest<'py>( let detail = registry::native(py, NativeAttr::DbtFailureDetail)?.call1((result,))?; let error = registry::error(py, ErrorId::DbtInteropRuntime)?.call( ("dbt compile failed",), - Some( - &crate::bindings::_helpers::compile_driver::python_calls::keywords( - py, - &[("help", detail)], - )?, - ), + Some(&keywords(py, &[("help", detail)])?), )?; return Err(PyErr::from_value(error)); } @@ -199,15 +197,18 @@ fn compile_dbt_interop_project<'py>( DbtCompileReferenceResolver::new(Some(args.value("manifest")?.unbind())), )? .into_any(); - let project = call_with( - ®istry::native(py, NativeAttr::BuildCompiledProject)?, - &[ - ("discovered_inputs", discovered), - ("adapter", adapter.clone()), - ("no_sql_validation", args.value("no_sql_validation")?), - ("cli_vars", args.value("dbt_vars")?), - ("external_sql_reference_resolver", resolver), - ], + let project = build_compiled_project( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter.clone()), + ("no_sql_validation", args.value("no_sql_validation")?), + ("cli_vars", args.value("dbt_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?), )?; completed(&callback, "Compiled SQLBuild project.", start)?; call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs index 364c3915cf..c5c6fb0c02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs @@ -30,6 +30,8 @@ use crate::bindings::_helpers::command_runtime::warehouse_scope::command_warehou use crate::bindings::_helpers::command_runtime::warehouses::{ CommandScope, WarehouseInputs, apply_warehouse, command_scope, find_authored, resolve_warehouse, }; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; @@ -259,9 +261,12 @@ fn build_report( ) -> PyResult { let project_dir_object: Bound<'_, PyAny> = imported(py, "pathlib", "Path")?.call1((project_dir,))?; - let discovered_inputs: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir_object.clone())], + let discovered_inputs: Bound<'_, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[("project_dir", project_dir_object.clone())], + )?), )?; let files: DebugConfigFiles = debug_config_files(project_dir); let adapter_name: Bound<'_, PyAny> = call_with( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs index d1455f1450..4dab1204b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs @@ -30,6 +30,11 @@ fn current_variable(py: Python<'_>) -> PyResult> { context_variable(py, &CURRENT_TRACKER, TRACKER_VARIABLE) } +/// The active build timing tracker, or None. +pub(crate) fn current_tracker(py: Python<'_>) -> PyResult> { + current_variable(py)?.call_method0("get") +} + /// `monotonic`, or `time.monotonic` when none is given. fn clock<'py>( py: Python<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs index a147a603f4..27dc07328f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/command.rs @@ -17,7 +17,9 @@ use sqlbuild_cli::diff_report::types::QueryDiffPhase; use sqlbuild_core::json::main::dumps::dumps; use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::diff_report::facts::diff_facts; use crate::bindings::_helpers::diff_report::query::{ execute_query_diff, execution_options, prepare_query_diff, project_adapter, @@ -83,9 +85,9 @@ fn resolve_invocation<'py>( let discovered: Bound<'py, PyAny> = if query_mode { discover_project_configuration(py, &project_dir)? } else { - call( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", &project_dir)], + discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )? }; call( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs index a3364bcffc..3b6b868ec7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs @@ -1,12 +1,15 @@ //! Project inputs holding only the project and local configuration. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyString; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; +use crate::bindings::_helpers::contracts::project_config::{ + load_local_config_file, load_project_config_file, +}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// `DiscoveredProjectInputs` with the loaded configuration, inside the project discovery lifecycle. @@ -31,12 +34,11 @@ pub(crate) fn discover_project_configuration<'py>( ("project_dir", project_dir.clone()), ( "project_config", - registry::native(py, NativeAttr::LoadProjectConfigFile)? - .call1((project_dir,))?, + load_project_config_file(py, project_dir)?.into_bound(py), ), ( "local_config", - registry::native(py, NativeAttr::LoadLocalConfigFile)?.call1((project_dir,))?, + load_local_config_file(py, project_dir)?.into_bound(py), ), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs index 2fb0491bd0..dc659ce42d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/description_inputs.rs @@ -1,12 +1,20 @@ //! Model description resolution for header-only tooling, without importing project Python. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyCFunction, PyFrozenSet}; +use pyo3::types::{PyCFunction, PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::project_config::load_project_config_file; +use crate::bindings::_helpers::discovery::declaration_records::discover_model_schema_files; +use crate::bindings::_helpers::discovery::model_records::{ + ModelFileSelection, discovered_model_files, +}; +use crate::bindings::_helpers::discovery::snapshot::in_snapshot; +use crate::bindings::_helpers::model_config::described_models::externally_described_models as described_models; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::render::declaration_indexes::public_model_schema_index; /// Models and model schemas, skipping files that cannot be read. #[pyfunction] @@ -15,20 +23,14 @@ pub(crate) fn discover_model_description_inputs<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = project_dir.py(); - let native = |name: NativeAttr| registry::native(py, name); let project_config: Bound<'py, PyAny> = - native(NativeAttr::LoadProjectConfigFile)?.call1((project_dir,))?; + load_project_config_file(py, project_dir)?.into_bound(py); let skip: Bound<'py, PyAny> = PyCFunction::new_closure(py, None, None, |_arguments, _keywords| -> PyResult<()> { Ok(()) })? .into_any(); - let disabled: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); - let scope: Bound<'py, PyAny> = call_with( - &native(NativeAttr::DirectorySnapshotScope)?, - &[("project_dir", project_dir.clone())], - )?; - within(&scope, |_| { + in_snapshot(project_dir, || { call_with( ®istry::class(py, RecordId::DiscoveredProjectInputs)?, &[ @@ -39,22 +41,19 @@ pub(crate) fn discover_model_description_inputs<'py>( ), ( "model_files", - call_with( - &native(NativeAttr::DiscoverModelFiles)?, - &[ - ("project_dir", project_dir.clone()), - ("extract_implicit_alias_columns", disabled.clone()), - ("extract_output_column_locations", disabled), - ("on_fault", skip.clone()), - ], - )?, + discovered_model_files( + project_dir, + (false, false), + ModelFileSelection { + selected_model_names: None, + on_fault: Some(skip.clone()), + }, + )? + .into_any(), ), ( "model_schema_files", - call_with( - &native(NativeAttr::DiscoverModelSchemaFiles)?, - &[("project_dir", project_dir.clone()), ("on_fault", skip)], - )?, + discover_model_schema_files(project_dir, Some(skip))?.into_any(), ), ], ) @@ -67,11 +66,8 @@ pub(crate) fn externally_described_models<'py>( ) -> PyResult> { let py: Python<'py> = project_dir.py(); let inputs: Bound<'py, PyAny> = discover_model_description_inputs(project_dir)?; - let schemas: Bound<'py, PyAny> = - registry::native(py, NativeAttr::PublicModelSchemaIndex)?.call1((&inputs,))?; - let described: Bound<'py, PyAny> = - registry::native(py, NativeAttr::ExternallyDescribedModels)?.call1((&inputs, schemas))?; - PyFrozenSet::new(py, described.try_iter()?.collect::>>()?) + let schemas: Bound<'py, PyDict> = public_model_schema_index(inputs.clone())?; + PyFrozenSet::new(py, described_models(inputs, schemas)?) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs index c10008829f..ff016dd6db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -6,20 +6,14 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sqltext::model_headers::{ + ColumnLocation, ParsedModelHeader, header_failure, model_output_columns, parse_model_headers, +}; const HEADER_CACHE_SIZE: usize = 4096; -/// `(name, line, column, end line, end column)`. -type LocationRow<'py> = ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, -); - /// Parsed headers by text: `(values or None, error or None)`. static HEADER_CACHE: PyOnceLock> = PyOnceLock::new(); @@ -49,14 +43,9 @@ fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { cache.clear(); } let names: Bound<'_, PyList> = missing.keys(); - let parsed: Bound<'_, PyAny> = - registry::native(py, NativeAttr::ParseModelHeaders)?.call1((&names,))?; - for (header, row) in names.iter().zip(parsed.try_iter()?) { - let row: Bound<'_, PyAny> = row?; - cache.set_item( - header, - PyTuple::new(py, [row.get_item(0)?, row.get_item(2)?])?, - )?; + let parsed: Vec = parse_model_headers(py, names.extract()?)?; + for (header, (values, _offsets, error)) in names.iter().zip(parsed) { + cache.set_item(header, (values, error))?; } Ok(()) } @@ -79,11 +68,12 @@ fn parse_header_values<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult = request.get("header_line")?; - let (detail, help, line): (String, Option, Option) = - registry::native(py, NativeAttr::ModelHeaderFailure)? - .call1((error.str()?, &header, &header_line))? - .extract()?; + let header_line: Option = request.get("header_line")?.extract()?; + let (detail, help, line): (String, Option, Option) = header_failure( + &error.str()?.extract::()?, + &header.extract::()?, + header_line, + ); let file_path: String = request.get("file_path")?.str()?.extract()?; let location: String = match line { Some(line) => format!("{file_path}:{line}"), @@ -107,14 +97,13 @@ fn parse_header_values<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult( - rows: &Bound<'py, PyAny>, + rows: Vec, relative_path: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = rows.py(); + let py: Python<'py> = relative_path.py(); let location_class: Bound<'py, PyAny> = registry::class(py, RecordId::SourceLocation)?; let locations: Bound<'py, PyDict> = PyDict::new(py); - for row in rows.try_iter()? { - let (name, line, column, end_line, end_column): LocationRow<'py> = row?.extract()?; + for (name, line, column, end_line, end_column) in rows { let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item("path", relative_path)?; arguments.set_item("line", line)?; @@ -133,14 +122,13 @@ fn model_output_column_locations<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("model_output_column_locations", request)?; - let py: Python<'py> = request.py(); let implicit: bool = match request.optional("extract_implicit_alias_columns")? { Some(value) => value.is_truthy()?, None => true, }; - let rows: Bound<'py, PyAny> = registry::native(py, NativeAttr::ModelOutputColumns)? - .call1((request.get("contents")?, implicit))?; - source_locations(&rows, &request.get("relative_path")?) + let contents: String = request.get("contents")?.extract()?; + let rows: Vec = model_output_columns(&contents, implicit)?; + source_locations(rows, &request.get("relative_path")?) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index 8134adb1b4..ba8affd828 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -2,15 +2,16 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::discovery::locations::project_relative_path; +use crate::bindings::_helpers::discovery::python_module_files::discover_python_module_files; use crate::bindings::_helpers::event_exporting::scope::current_scope; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObjectExt; @@ -49,22 +50,9 @@ impl<'py> ProjectDir<'py> { root: &str, public_only: bool, ) -> PyResult>> { - let py: Python<'py> = self.py(); - call_with( - ®istry::native(py, NativeAttr::DiscoverPythonModuleFiles)?, - &[ - ("project_dir", self.path.clone()), - ("root", PyString::new(py, root).into_any()), - ( - "public_only", - pyo3::types::PyBool::new(py, public_only) - .to_owned() - .into_any(), - ), - ], - )? - .try_iter()? - .collect() + Ok(discover_python_module_files(&self.path, root, public_only)? + .iter() + .collect()) } /// `shim.(file_path=file_path, project_dir=project_dir)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs index 16b9f904bb..4653475d06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_providers.rs @@ -4,9 +4,11 @@ use std::collections::HashSet; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::types::{PyDict, PyDictMethods}; +use crate::bindings::_helpers::compile_driver::input_reads::record_settings_class_read; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::contracts::resource_identities::validate_resource_identity; use crate::bindings::_helpers::discovery::imported_modules::{ ProjectDir, discovered, discovery_error, discovery_error_from, module_functions, provider_by_name, provider_usages, shim, tuple_of, @@ -14,8 +16,7 @@ use crate::bindings::_helpers::discovery::imported_modules::{ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; const PROVIDER_ERROR: ErrorId = ErrorId::ProviderDiscovery; const SINK_ERROR: ErrorId = ErrorId::EventExporterDiscovery; @@ -31,16 +32,7 @@ fn reject_invalid_identity( name: &Bound<'_, PyAny>, relative_path: &Bound<'_, PyAny>, ) -> PyResult<()> { - let py: Python<'_> = name.py(); - call_with( - ®istry::native(py, NativeAttr::ValidateResourceIdentity)?, - &[ - ("name", name.clone()), - ("kind", PyString::new(py, "provider").into_any()), - ("path", relative_path.clone()), - ], - )?; - Ok(()) + validate_resource_identity(name.downcast()?, "provider", relative_path, false) } /// The provider's validated name, from its explicit `provider_name` or `name()`. @@ -139,11 +131,10 @@ pub(crate) fn discover_provider_classes<'py>( ) -> PyResult> { let py: Python<'py> = project_dir.py(); let project = ProjectDir { path: project_dir }; - let record_read: Bound<'py, PyAny> = registry::native(py, NativeAttr::RecordSettingsClassRead)?; if let Some(cached) = project.cached_extensions()? { let providers: Bound<'py, PyAny> = cached.get_item(0)?; for provider in providers.try_iter()? { - record_read.call1((native_field(&provider?, "provider_class")?,))?; + record_settings_class_read(&native_field(&provider?, "provider_class")?)?; } return Ok(providers); } @@ -178,7 +169,7 @@ pub(crate) fn discover_provider_classes<'py>( )); } seen.push((name_text, file_path.clone())); - record_read.call1((&provider_class,))?; + record_settings_class_read(&provider_class)?; let settings: Bound<'py, PyAny> = provider_instance(&provider_class, (&name, &relative_path))?; providers.push(discovered( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs index 5a113f57be..22de979c8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::discovery::imported_modules::{ use crate::bindings::_helpers::discovery::imported_providers; use crate::bindings::_helpers::discovery::python_hook_files::discover_python_hook_files; +use crate::bindings::_helpers::graph::python_node_graph::node_identity; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::fields::field as native_field; @@ -33,6 +34,8 @@ struct NodeKind { reader: UserCodeHost, record: RecordId, bucket: usize, + /// The node kind and whether its identity hashes documentation fields. + identity: (&'static str, bool), fields: &'static [&'static str], } @@ -42,6 +45,7 @@ const NODE_KINDS: [NodeKind; 4] = [ reader: UserCodeHost::ReadLoaderDefinition, record: RecordId::DiscoveredLoaderFunction, bucket: 0, + identity: ("LOADER", false), fields: &[ "depends_on", "destination", @@ -57,6 +61,7 @@ const NODE_KINDS: [NodeKind; 4] = [ reader: UserCodeHost::ReadTaskDefinition, record: RecordId::DiscoveredTaskFunction, bucket: 1, + identity: ("TASK", true), fields: &[ "depends_on", "tags", @@ -70,6 +75,7 @@ const NODE_KINDS: [NodeKind; 4] = [ reader: UserCodeHost::ReadAssetDefinition, record: RecordId::DiscoveredAssetFunction, bucket: 2, + identity: ("ASSET", true), fields: &[ "depends_on", "tags", @@ -85,6 +91,7 @@ const NODE_KINDS: [NodeKind; 4] = [ reader: UserCodeHost::ReadCheckDefinition, record: RecordId::DiscoveredCheckFunction, bucket: 3, + identity: ("CHECK", true), fields: &[ "depends_on", "severity", @@ -131,6 +138,14 @@ impl<'py> Buckets<'py> { pairs.push((field, read.getattr(*field)?)); } pairs.push(("provider_usages", provider_usages(function, providers)?)); + let get = |name: &str| -> PyResult> { + match pairs.iter().find(|(field, _)| *field == name) { + Some((_, value)) => Ok(value.clone()), + None => Ok(py.None().into_bound(py)), + } + }; + let identity: Bound<'py, PyAny> = node_identity(py, kind.identity, &get)?; + pairs.push(("identity", identity)); self.nodes[kind.bucket].push(discovered(py, kind.record, &pairs)?); return Ok(true); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs index 2836e3582e..be655ab2fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_module_files.rs @@ -14,7 +14,7 @@ use crate::bindings::models::NativeProjectTree; /// `sorted((project_dir / root).rglob("*.py"))` without `__init__` modules, or `_` parts too. #[pyfunction] #[pyo3(signature = (*, project_dir, root, public_only))] -fn discover_python_module_files<'py>( +pub(crate) fn discover_python_module_files<'py>( project_dir: &Bound<'py, PyAny>, root: &str, public_only: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs index 3eb9cdf6ee..b7717fc98c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs @@ -40,6 +40,7 @@ use sqlbuild_state::execution_history::models::{ use sqlbuild_state::execution_history::types::UtcTime; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::execution_history::appends::prepared_batch; use crate::bindings::_helpers::execution_history::conversion::{ checked, event_filter_spec, exact_int, history_error_of_kind, new_namespace, now, @@ -498,20 +499,17 @@ impl SqliteExecutionHistory { let sqlite = py.import(SQLITE_MODULE)?; let opened: Bound<'_, PyAny> = native_field(&sqlite, "connect")?.call( (database.as_str(),), - Some( - &crate::bindings::_helpers::compile_driver::python_calls::keywords( - py, - &[ - ( - "timeout", - (busy_timeout_ms as f64 / MILLIS_PER_SECOND) - .into_bound_py_any(py)?, - ), - ("isolation_level", py.None().into_bound(py)), - ("check_same_thread", false.into_bound_py_any(py)?), - ], - )?, - ), + Some(&keywords( + py, + &[ + ( + "timeout", + (busy_timeout_ms as f64 / MILLIS_PER_SECOND).into_bound_py_any(py)?, + ), + ("isolation_level", py.None().into_bound(py)), + ("check_same_thread", false.into_bound_py_any(py)?), + ], + )?), )?; connection = Some(opened.clone()); slf.setattr("_connection", &opened)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs index 33a3fde9bc..67f7efd85d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -16,6 +16,8 @@ use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; @@ -27,8 +29,6 @@ use crate::bindings::_helpers::format_command::selection::{LintInputs, resolve_l use crate::bindings::_helpers::frontier::encoder::emit_error; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; const DEFAULT_DIALECT: &str = "generic"; @@ -117,9 +117,9 @@ fn resolve_inputs<'py>( } } if fix { - resolved.discovered = Some(call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + resolved.discovered = Some(discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?); } Ok(resolved) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs index 202e716c67..631ccf7ebc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -15,12 +15,13 @@ use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const SELECTION_CODE: &str = "S007"; /// The adapter, the selected files when a selector scopes the run, and the discovered inputs. @@ -171,7 +172,7 @@ fn model_paths( ], )?; let model_type: Bound<'_, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut names: BTreeSet = BTreeSet::new(); for key in keys.try_iter()? { let key: Bound<'_, PyAny> = key?; @@ -334,13 +335,16 @@ pub(crate) fn resolve_lint_inputs<'py>( return path_scoped_inputs(project_dir, &directory, paths); } let disabled: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("sql_analysis_enabled_override", disabled.clone()), - ("extract_output_column_locations", disabled), - ], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", disabled.clone()), + ("extract_output_column_locations", disabled), + ], + )?), )?; call_with( ®istry::native(py, NativeAttr::RejectUnitTestSelectors)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs index 67a8d50ee1..3a493440b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs @@ -27,9 +27,9 @@ use crate::bindings::_helpers::planner_selection::selectors::{ }; use crate::bindings::_helpers::planner_selection::unit_tests::reject_unit_test_selectors; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; @@ -81,7 +81,7 @@ fn source_names_for_keys( let mut names: BTreeSet = BTreeSet::new(); let py: Python<'_> = graph.py(); let source_type: Bound<'_, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SOURCE")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?; let upstream_deps: Bound<'_, PyAny> = native_field(graph, "upstream_deps")?; for key in keys.try_iter()? { let key: Bound<'_, PyAny> = key?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs index 0d27260653..7f3fb613a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/planning.rs @@ -7,9 +7,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PySet, PySetMethods, PyTup use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_state::freshness::main::configuration_error::configuration_error; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::freshness::exception_types::SourceFreshnessObservationError; use crate::bindings::_helpers::freshness::observation::{ Observer, observation_record, observation_source, observe_adapter_batch, observe_configured, strategy, unknown, @@ -20,12 +19,11 @@ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_identity::graph::build_downstream_model_name_closure; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::freshness::{age_policy, equivalent}; const ADAPTER_STRATEGY: &str = "adapter"; -const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; const NO_CONFIG_MESSAGE: &str = "no freshness config and adapter metadata unavailable"; /// The message of an `AdapterUserError` or a source freshness observation error, if it is one. @@ -34,10 +32,7 @@ fn handled_message<'py>(py: Python<'py>, error: &PyErr) -> PyResult(py) { return Ok(Some(value.str()?.into_any())); } Ok(None) @@ -311,7 +306,7 @@ fn downstream_models<'py>( &[ ( "resource_type", - imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?, + registry::class(py, RecordId::CompiledResourceType)?.getattr("SOURCE")?, ), ("name", source_name.clone()), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index 1d7d91d50b..5606688024 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -8,11 +8,12 @@ use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; +use crate::bindings::_helpers::cost_telemetry::context::CostContext; use crate::bindings::_helpers::freshness::state_io::write_source_freshness_records; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; /// Observed records grouped by `(database, schema)` in first-seen order. @@ -187,7 +188,6 @@ fn write_records<'py>( grouped: &RecordsByLocation<'py>, ) -> PyResult<()> { let py: Python<'py> = adapter.py(); - let cost_context: Bound<'py, PyAny> = registry::native(py, NativeAttr::CostContext)?; for ((database, schema), records) in grouped { let mut by_source: Vec<(Bound<'py, PyAny>, Vec>)> = Vec::new(); for record in records { @@ -205,17 +205,24 @@ fn write_records<'py>( } } for (source_name, source_records) in by_source { - let scope: Bound<'py, PyAny> = call_with( - &native_field(&cost_context, "resource_scope")?, - &[ - ("resource_type", PyString::new(py, "source").into_any()), - ("resource_name", source_name), - ( - "phase", - PyString::new(py, "freshness_finalization").into_any(), - ), - ], - )?; + let scope: Bound<'py, PyAny> = Bound::new( + py, + CostContext::resource_scope( + py, + Some(keywords( + py, + &[ + ("resource_type", PyString::new(py, "source").into_any()), + ("resource_name", source_name), + ( + "phase", + PyString::new(py, "freshness_finalization").into_any(), + ), + ], + )?), + )?, + )? + .into_any(); within(&scope, |_| { write_source_freshness_records( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs index 2f7e75a44e..2450e4e5c1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/support.rs @@ -6,23 +6,21 @@ use pyo3::types::{PyDict, PyString}; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::freshness::exception_types::{ + SourceFreshnessInputError, SourceFreshnessObservationError, +}; use crate::bindings::_helpers::state_rows::timestamps::{date_time_typed, in_utc, offset_aware}; -pub(crate) const OBSERVATION_ERROR: NativeAttr = NativeAttr::SourceFreshnessObservationError; -pub(crate) const INPUT_ERROR: NativeAttr = NativeAttr::SourceFreshnessInputError; +pub(crate) const OBSERVATION_ERROR: fn(String) -> PyErr = SourceFreshnessObservationError::new_err; +pub(crate) const INPUT_ERROR: fn(String) -> PyErr = SourceFreshnessInputError::new_err; /// The native error `class(message)` as a raisable error, with an optional `__cause__`. pub(crate) fn freshness_error( py: Python<'_>, - (class, message): (NativeAttr, &str), + (class, message): (fn(String) -> PyErr, &str), cause: Option, ) -> PyErr { - let error: PyErr = match registry::native(py, class).and_then(|class| class.call1((message,))) { - Ok(error) => PyErr::from_value(error), - Err(error) => return error, - }; + let error: PyErr = class(message.to_owned()); if let Some(cause) = cause { error.set_cause(py, Some(cause)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index a439a58255..319aeb6f8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -12,7 +12,6 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PATHLIB_MODULE: &str = "pathlib"; /// The model config keys whose entries are hooks, in lifecycle order. const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -123,9 +122,7 @@ pub(crate) fn sql_ref_key<'py>( py: Python<'py>, reference: &Bound<'py, PyAny>, ) -> PyResult> { - let resource_type: Bound<'py, PyAny> = py - .import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")? + let resource_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)? .call1((native_field(reference, "kind")?.getattr("value")?,))?; keyword_call( ®istry::class(py, RecordId::CompiledObjectKey)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs index 704ef6c721..a5d71ca98f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs @@ -6,8 +6,6 @@ use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - /// One `CompiledObjectKey` per native pair, so shared keys convert once. pub(crate) struct KeyCache<'py> { key_type: Bound<'py, PyAny>, @@ -19,9 +17,7 @@ impl<'py> KeyCache<'py> { pub(crate) fn new(py: Python<'py>) -> PyResult { Ok(Self { key_type: registry::class(py, RecordId::CompiledObjectKey)?, - resource_type: py - .import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")?, + resource_type: registry::class(py, RecordId::CompiledResourceType)?, keys: PyDict::new(py), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index c4415b453d..7ab69a9101 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -2,7 +2,8 @@ use std::collections::HashMap; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::{pyclass, pymethods}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; @@ -20,6 +21,7 @@ use sqlbuild_analysis::graph::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::graph::views::lineage_graph_views; /// `(key, deps, tags, model folder)`. pub(crate) type ResourceRow = (ObjectKey, Vec, Vec, Option); @@ -45,6 +47,14 @@ pub(crate) struct NativeProjectGraph { graph: ProjectGraph, } +/// Static compiled project graph held natively; dict views are built only when read. +#[pyclass(name = "ProjectGraph", module = "sqlbuild._native", frozen)] +pub(crate) struct CompiledProjectGraph { + project: Py, + native: Py, + views: PyOnceLock>, +} + fn dependency_edges(key: &ObjectKey, deps: &[ObjectKey]) -> Vec<(ObjectKey, ObjectKey)> { deps.iter().map(|dep| (dep.clone(), key.clone())).collect() } @@ -72,6 +82,85 @@ impl NativeProjectGraph { } } +impl CompiledProjectGraph { + /// The graph of `project` over its native lineage graph. + pub(crate) fn new(project: Py, native: Py) -> Self { + Self { + project, + native, + views: PyOnceLock::new(), + } + } + + /// The compiled project this graph indexes. + pub(crate) fn project<'py>(&self, py: Python<'py>) -> &Bound<'py, PyAny> { + self.project.bind(py) + } + + /// The native lineage graph. + pub(crate) fn native<'py>(&self, py: Python<'py>) -> &Bound<'py, NativeProjectGraph> { + self.native.bind(py) + } + + fn view(&self, py: Python<'_>, name: &str) -> PyResult> { + Ok(self.lineage_views(py)?.bind(py).getattr(name)?.unbind()) + } +} + +#[pymethods] +impl CompiledProjectGraph { + #[new] + #[pyo3(signature = (*, project, native))] + fn py_new(project: Py, native: Py) -> Self { + Self::new(project, native) + } + + #[getter(project)] + fn project_attribute(&self, py: Python<'_>) -> Py { + self.project.clone_ref(py) + } + + #[getter(native)] + fn native_attribute(&self, py: Python<'_>) -> Py { + self.native.clone_ref(py) + } + + /// Lineage edges and selector indexes, converted once. + #[getter] + fn lineage_views(&self, py: Python<'_>) -> PyResult> { + self.views + .get_or_try_init(py, || { + Ok(lineage_graph_views(py, self.native.bind(py).borrow())?.unbind()) + }) + .map(|views| views.clone_ref(py)) + } + + #[getter] + fn upstream_deps(&self, py: Python<'_>) -> PyResult> { + self.view(py, "upstream_deps") + } + + #[getter] + fn downstream_deps(&self, py: Python<'_>) -> PyResult> { + self.view(py, "downstream_deps") + } + + #[getter] + fn tag_index(&self, py: Python<'_>) -> PyResult> { + self.view(py, "tag_index") + } + + #[getter] + fn path_index(&self, py: Python<'_>) -> PyResult> { + self.view(py, "path_index") + } + + #[getter] + fn all_keys(&self, py: Python<'_>) -> PyResult> { + self.view(py, "all_keys") + } +} + /// The project graph indexes of these resources, in project order. pub(crate) fn project_graph(resources: Vec) -> PyResult { compiler_guard(|| { @@ -220,5 +309,6 @@ impl NativeProjectGraph { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::() + module.add_class::()?; + module.add_class::() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs index a07b3878b6..0eb247bf0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_node_graph.rs @@ -32,7 +32,6 @@ struct Vocabulary<'py> { node: Bound<'py, PyAny>, kinds: Bound<'py, PyAny>, sql_ref: Bound<'py, PyAny>, - identity: Bound<'py, PyAny>, providers: Bound<'py, PyDict>, } @@ -47,13 +46,69 @@ fn optional_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> native_field(value, "value") } -/// The kind-specific decorator configuration and metadata record of one discovered function. -fn kind_parts<'py>( +/// The decorator configuration the identity of one node of `kind` hashes, read through `get`. +fn decorator_config<'py>( + py: Python<'py>, + (kind, documented): (&str, bool), + get: &dyn Fn(&str) -> PyResult>, +) -> PyResult> { + let config: Bound<'py, PyDict> = PyDict::new(py); + match kind { + "LOADER" => { + config.set_item("columns", get("columns")?)?; + config.set_item("contract", get("contract")?)?; + config.set_item("cursor_column", get("cursor_column")?)?; + config.set_item("destination", get("destination")?)?; + config.set_item("unique_key", get("unique_key")?)?; + config.set_item("write_strategy", optional_value(&get("write_strategy")?)?)?; + } + "ASSET" => { + config.set_item("columns", get("columns")?)?; + config.set_item("column_lineage", get("column_lineage")?)?; + } + "CHECK" => { + config.set_item("severity", get("severity")?.getattr("value")?)?; + } + _ => {} + } + if documented { + for field in ["description", "group", "meta", "tags"] { + config.set_item(field, get(field)?)?; + } + } + Ok(config) +} + +/// The identity of one discovered node of `(kind, documented)`, hashed once by the user-code host. +pub(crate) fn node_identity<'py>( + py: Python<'py>, + (kind, documented): (&str, bool), + get: &dyn Fn(&str) -> PyResult>, +) -> PyResult> { + let kind_value: Bound<'py, PyAny> = host::entry(py, UserCodeHost::PythonNodeKind)? + .getattr(kind)? + .getattr("value")?; + call_with( + &host::entry(py, UserCodeHost::PythonNodeIdentity)?, + &[ + ("kind", kind_value), + ("name", get("name")?), + ("function", get("function")?), + ("file_path", get("file_path")?), + ( + "decorator_config", + decorator_config(py, (kind, documented), get)?.into_any(), + ), + ], + ) +} + +/// The kind's metadata field and record of one discovered function. +fn kind_metadata<'py>( py: Python<'py>, kind: &str, discovered: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyDict>, &'static str, Bound<'py, PyAny>)> { - let config: Bound<'py, PyDict> = PyDict::new(py); +) -> PyResult<(&'static str, Bound<'py, PyAny>)> { let get = |name: &str| attribute_value(discovered, name); let metadata = |class: RecordId, fields: &[&str]| -> PyResult> { let pairs: Vec<(&str, Bound<'py, PyAny>)> = fields @@ -63,14 +118,9 @@ fn kind_parts<'py>( call_with(®istry::class(py, class)?, &pairs) }; Ok(match kind { - "LOADER" => { - config.set_item("columns", get("columns")?)?; - config.set_item("contract", get("contract")?)?; - config.set_item("cursor_column", get("cursor_column")?)?; - config.set_item("destination", get("destination")?)?; - config.set_item("unique_key", get("unique_key")?)?; - config.set_item("write_strategy", optional_value(&get("write_strategy")?)?)?; - let record = metadata( + "LOADER" => ( + "loader", + metadata( RecordId::DiscoveredPythonLoaderMetadata, &[ "destination", @@ -81,28 +131,23 @@ fn kind_parts<'py>( "contract", "connection_mode", ], - )?; - (config, "loader", record) - } + )?, + ), "TASK" => ( - config, "task", metadata(RecordId::DiscoveredPythonTaskMetadata, &["retry"])?, ), - "ASSET" => { - config.set_item("columns", get("columns")?)?; - config.set_item("column_lineage", get("column_lineage")?)?; - let record = metadata( + "ASSET" => ( + "asset", + metadata( RecordId::DiscoveredPythonAssetMetadata, &["columns", "column_lineage", "retry"], - )?; - (config, "asset", record) - } - _ => { - config.set_item("severity", get("severity")?.getattr("value")?)?; - let record = metadata(RecordId::DiscoveredPythonCheckMetadata, &["severity"])?; - (config, "check", record) - } + )?, + ), + _ => ( + "check", + metadata(RecordId::DiscoveredPythonCheckMetadata, &["severity"])?, + ), }) } @@ -114,24 +159,13 @@ fn python_node<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let get = |name: &str| attribute_value(discovered, name); - let (config, metadata_field, metadata) = kind_parts(py, kind, discovered)?; - if documented { - for field in ["description", "group", "meta", "tags"] { - config.set_item(field, get(field)?)?; - } - } + let (metadata_field, metadata) = kind_metadata(py, kind, discovered)?; let kind_member: Bound<'py, PyAny> = vocabulary.kinds.getattr(kind)?; let usages: Bound<'py, PyAny> = provider_usages(&get("function")?, &vocabulary.providers)?; - let identity: Bound<'py, PyAny> = call_with( - &vocabulary.identity, - &[ - ("kind", native_field(&kind_member, "value")?), - ("name", get("name")?), - ("function", get("function")?), - ("file_path", get("file_path")?), - ("decorator_config", config.into_any()), - ], - )?; + let mut identity: Bound<'py, PyAny> = get("identity")?; + if identity.is_none() { + identity = node_identity(py, (kind, documented), &get)?; + } let depends_on: Bound<'py, PyAny> = get("depends_on")?; let mut sql_deps: Vec> = Vec::new(); for dependency in depends_on.try_iter()? { @@ -224,7 +258,6 @@ pub(crate) fn python_node_graph<'py>( node: registry::class(py, RecordId::DiscoveredPythonNode)?, kinds: host::entry(py, UserCodeHost::PythonNodeKind)?, sql_ref: host::entry(py, UserCodeHost::SqlResourceRef)?, - identity: host::entry(py, UserCodeHost::PythonNodeIdentity)?, providers, }; let mut nodes: Vec> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index a0761289a9..ac2aef966f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -17,7 +17,7 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; @@ -101,9 +101,8 @@ fn selection_graph(graph: &Bound<'_, PyAny>) -> PyResult { } /// `(name, loader)` of every source in the compiled project. -fn project_sources(project_graph: &Bound<'_, PyAny>) -> PyResult)>> { - project_graph - .getattr("project")? +fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>> { + project .getattr("sources")? .try_iter()? .map(|source| { @@ -168,9 +167,9 @@ pub(crate) fn resolve_python_sql_run_selection( exclude: Vec, ) -> PyResult<(Vec, Vec)> { let py: Python<'_> = project_graph.py(); - let native: Bound<'_, NativeProjectGraph> = - native_field(project_graph, "native")?.downcast_into()?; - let sources: Vec<(String, Option)> = project_sources(project_graph)?; + let project_graph: &Bound<'_, CompiledProjectGraph> = project_graph.cast()?; + let native: &Bound<'_, NativeProjectGraph> = project_graph.get().native(py); + let sources: Vec<(String, Option)> = project_sources(project_graph.get().project(py))?; let nodes: PythonSelectionGraph = selection_graph(python_graph)?; let selection: RunSelection = compiler_guard(|| { Ok(run_selection( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs index af037455e5..74cbc632aa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/views.rs @@ -40,7 +40,7 @@ pub(crate) fn compiled_graph_keys<'py>( /// Selector name to key, in project order, without the other graph views. #[pyfunction] -fn lineage_graph_names<'py>( +pub(crate) fn lineage_graph_names<'py>( py: Python<'py>, graph: PyRef<'py, NativeProjectGraph>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index f05a66d68b..f1a5f48c84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,7 +1,5 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. -use crate::bindings::_helpers::native_host::facade::FacadeHost; -use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -68,7 +66,7 @@ pub(crate) fn compiled_model<'py>( inferred = native_field(&proof, "fixed_columns")?; has_star = true_object(py); } - let resource_type: Object<'py> = host::entry(py, FacadeHost::CompiledResourceType)?; + let resource_type: Object<'py> = registry::class(py, RecordId::CompiledResourceType)?; let key_class: Object<'py> = registry::class(py, RecordId::CompiledObjectKey)?; let mut dependency_keys: Vec> = Vec::new(); for dependency in deps.try_iter()? { @@ -262,7 +260,8 @@ fn located<'py>( let diagnostic_class: Object<'py> = registry::class(py, RecordId::CompilerDiagnostic)?; let location_class: Object<'py> = registry::class(py, RecordId::SourceLocation)?; let phase: Object<'py> = registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?; - let model: Object<'py> = host::entry(py, FacadeHost::CompiledResourceType)?.getattr("MODEL")?; + let model: Object<'py> = + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let name: Object<'py> = native_field(&model_file, "file_path")?.getattr("stem")?; let mut built: Vec> = Vec::new(); for row in located.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs index f505a54ff6..a16c115d78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs @@ -8,6 +8,9 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; +use crate::bindings::_helpers::cli::progress_reporters::{ + ConnectionProgressReporter, PlanningProgressReporter, +}; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::janitor::janitor_facts::Warehouse; @@ -17,7 +20,7 @@ use crate::bindings::_helpers::janitor::janitor_planning::{ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; const UNKNOWN_VIEW_CODE: &str = "C503"; @@ -69,18 +72,11 @@ pub(crate) fn compile_janitor_project<'py>( true, )?; let started: Instant = Instant::now(); - let status: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::PlanningProgressReporter)?, - &[ - ("stream", stdout(py)?), - ( - "use_color", - PyBool::new(py, use_color(invocation)?) - .to_owned() - .into_any(), - ), - ], - )?; + let status: Bound<'py, PyAny> = Bound::new( + py, + PlanningProgressReporter::new(py, &stdout(py)?, use_color(invocation)?)?, + )? + .into_any(); let as_target: Bound<'py, PyAny> = native_field(invocation, "as_target")?; let target: Bound<'py, PyAny> = if as_target.is_none() { native_field(invocation, "selected_target")? @@ -139,19 +135,26 @@ pub(crate) fn connect_janitor_warehouse<'py>( compiled: &JanitorCompiled<'py>, ) -> PyResult> { let py: Python<'py> = invocation.py(); - let progress: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ConnectionProgressReporter)?, - &[ - ("adapter_name", compiled.adapter_name.clone()), - ("stream", stdout(py)?), - ( - "use_color", - PyBool::new(py, use_color(invocation)?) - .to_owned() - .into_any(), - ), - ], - )?; + let progress: Bound<'py, PyAny> = Bound::new( + py, + ConnectionProgressReporter::new( + py, + Some(keywords( + py, + &[ + ("adapter_name", compiled.adapter_name.clone()), + ("stream", stdout(py)?), + ( + "use_color", + PyBool::new(py, use_color(invocation)?) + .to_owned() + .into_any(), + ), + ], + )?), + )?, + )? + .into_any(); let started: Instant = Instant::now(); progress.call_method1("on_connection_start", (1,))?; let connected: PyResult> = within( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs index 3b4b82ff5f..463de3a49d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs @@ -33,6 +33,7 @@ use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::build_progress::transient_status::maybe_status; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::lineage::lineage_cache::{ lineage_fingerprint, read_lineage_cache, write_lineage_cache, }; @@ -136,7 +137,7 @@ fn discover<'py>( options.set_item("project_dir", project_dir)?; options.set_item("sql_analysis_enabled_override", analysis_override)?; options.set_item("extract_output_column_locations", false)?; - registry::native(py, NativeAttr::DiscoverProjectInputs)?.call((), Some(&options)) + discover_project_inputs(py, Some(options)) } fn reject_unit_tests( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs index 0afc432355..e8bee6abb6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs @@ -9,7 +9,6 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use sqlbuild_analysis::lineage::models::{LineageColumn, LineageSource}; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const LOGGING_MODULE: &str = "logging"; const LINEAGE_LOGGER: &str = "sqlbuild.lineage"; @@ -35,9 +34,7 @@ pub(crate) struct LineageClasses<'py> { impl<'py> LineageClasses<'py> { pub(crate) fn new(py: Python<'py>) -> PyResult { - let resource_type: Bound<'py, PyAny> = py - .import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")?; + let resource_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledResourceType)?; let nullability: Bound<'py, PyAny> = registry::class(py, RecordId::InferredNullability)?; Ok(Self { qualified_column: registry::class(py, RecordId::QualifiedLineageColumn)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 3cd6139974..6441bd888a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -35,7 +35,7 @@ const GENERIC_DIALECT: &str = "generic"; /// The project's column lineage graph in `mode`, or `None` without SQL analysis. #[pyfunction] -fn build_project_column_lineage<'py>( +pub(crate) fn build_project_column_lineage<'py>( project: &Bound<'py, PyAny>, dialect: Option<&Bound<'py, PyAny>>, mode: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index cec1b2e845..5ef2ef5906 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -17,7 +17,9 @@ use sqlbuild_core::load_command::main::render_load_ready::render_load_ready; use sqlbuild_core::load_command::models::{LoadCounts, LoadResultText}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timestamp}; use crate::bindings::_helpers::load::load_progress::{ LoadCallbackRelay, LoadProgressReporter, load_result_text, @@ -75,9 +77,9 @@ fn invocation<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let select: Bound<'py, PyAny> = native_field(request, "select")?; let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs index 4fba09abbf..462efe8530 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs @@ -19,10 +19,9 @@ use sqlbuild_model_config::templates::models::{ }; use sqlbuild_model_config::templates::types::TemplateHost; +use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::errors::NativeConfigError; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; const TEMPLATE_OPEN_TOKEN: &str = "${"; const ENVIRONMENT_READ: &str = "env"; @@ -241,15 +240,8 @@ impl<'py> TemplateHost for PythonHost<'py> { if value.is_exact_instance_of::() { return self.text_of(value); } - let rendered = - registry::native(self.py, NativeAttr::RenderProjectVarText).and_then(|render| { - let arguments = PyDict::new(self.py); - arguments.set_item("value", value)?; - arguments.set_item("label", label)?; - render.call((), Some(&arguments)) - }); - match rendered { - Ok(text) => self.text_of(&text), + match project_var_text(value, label) { + Ok(text) => Ok(text), Err(error) if error.is_instance_of::(self.py) => Err( TemplateFailure::Invalid(TemplateError::Message(error.value(self.py).to_string())), ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs index 83850b5aa7..3dcabf33db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs @@ -1,8 +1,7 @@ //! Which models `[defaults]`, `[path_defaults]` or a reusable model schema describe. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyString}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_model_config::path_defaults::main::select_path_default::select_path_default; use sqlbuild_model_config::path_defaults::models::PathDefaultChoice; @@ -24,8 +23,7 @@ fn non_empty_text(value: Option>) -> PyResult { } /// The file path of each model whose layered config or model schema gives it a description. -#[pyfunction] -fn externally_described_models<'py>( +pub(crate) fn externally_described_models<'py>( inputs: Bound<'py, PyAny>, model_schemas: Bound<'py, PyDict>, ) -> PyResult>> { @@ -74,7 +72,3 @@ fn externally_described_models<'py>( } Ok(described) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(externally_described_models, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 5ee6943fc6..ab7dc904ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -118,7 +118,7 @@ pub(crate) enum ConfigBuild<'py> { } /// `((target name, run id), (database, schema) or None)` of the active target. -type TargetInputs<'py> = ( +pub(crate) type TargetInputs<'py> = ( (Bound<'py, PyAny>, Bound<'py, PyAny>), Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)>, ); @@ -126,7 +126,7 @@ type TargetInputs<'py> = ( #[pymethods] impl NativeModelConfigBuilder { #[new] - fn new( + pub(crate) fn new( layers: (Bound<'_, PyAny>, Bound<'_, PyDict>, Bound<'_, PyTuple>), sources: (Bound<'_, PyDict>, Bound<'_, PyAny>), target: TargetInputs<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs index fc8b3615f2..f0cc73ea11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/registration.rs @@ -3,15 +3,14 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::model_config::{ - column_migrations, config_errors, config_values, described_models, header_columns, - model_config_builder, model_validator, template_data, unicode_environment, + column_migrations, config_errors, config_values, header_columns, model_config_builder, + model_validator, template_data, unicode_environment, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { column_migrations::register(module)?; config_errors::register(module)?; config_values::register(module)?; - described_models::register(module)?; header_columns::register(module)?; model_config_builder::register(module)?; model_validator::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs index d40ae9e79f..c46ee8fe55 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs @@ -15,7 +15,6 @@ pub(crate) enum FacadeHost { AuditSeverity, ThresholdOperator, AttachedAuditTargetKind, - CompiledResourceType, FunctionLanguage, SqlTestMode, CursorAlgebraError, @@ -74,11 +73,6 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Facade, }, - HostEntry { - name: "CompiledResourceType", - kind: HostKind::Class, - edge: HostEdge::Facade, - }, HostEntry { name: "FunctionLanguage", kind: HostKind::Class, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs index ed3a623375..35517b405f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs @@ -9,6 +9,8 @@ use pyo3::wrap_pyfunction; use crate::bindings::_helpers::build_command::support::keyword_relay; use crate::bindings::_helpers::cli::progress_text::with_verbose_inspection; use crate::bindings::_helpers::command_runtime::invocation::resolve_adapter_connection_context; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -64,12 +66,15 @@ fn resolve_invocation<'py>(request: &Bound<'py, PyAny>) -> PyResult = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("extract_output_column_locations", flag(py, false)), - ], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", flag(py, false)), + ], + )?), )?; let preview: Bound<'py, PyDict> = PyDict::new(py); preview.set_item("discovered_inputs", &discovered)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index b3bfdd4ee5..b5a3ecbde1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -7,6 +7,8 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use pyo3::wrap_pyfunction; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -154,9 +156,9 @@ fn rule_inputs<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = request.py(); - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let select: Bound<'py, PyAny> = native_field(request, "select")?; let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs index f587171c84..5435b9a3fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/actions.rs @@ -4,12 +4,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; const CREATE_TABLE_ACTION: &str = "create_table"; @@ -22,7 +21,7 @@ pub(crate) fn resolve_model_actions<'py>( ) -> PyResult> { let py: Python<'py> = scope.py(); let model_kind: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let action_class: Bound<'py, PyAny> = registry::class(py, RecordId::ResolvedModelAction)?; let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index e301b367cd..60bf12cb77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -48,7 +48,6 @@ use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; const REPLAY_CONFIG_KEY: &str = "replay_on_change"; @@ -236,7 +235,7 @@ pub(crate) fn detect_model_changes<'py>( ) } -fn scope_model_changes<'py>( +pub(crate) fn scope_model_changes<'py>( (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), query_change_tracking: Option, (version_hashes, metadata_jsons): (Bound<'py, PyAny>, Bound<'py, PyAny>), @@ -275,7 +274,7 @@ fn scope_model_changes<'py>( hook_version_hashes: hook_version_hashes.into_any(), }; let model_kind: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let selected: Bound<'py, PyAny> = native_field(scope, "selected_keys")?; let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; let changes: Bound<'py, PyDict> = PyDict::new(py); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index a383c0794a..52f30a7ad8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -92,7 +92,7 @@ fn base_plan_output<'py>( /// Resolve command selection through the canonical build selector phase. #[pyfunction] #[pyo3(signature = (*, project, selection, auto_load_sources = false))] -fn resolve_static_command_scope<'py>( +pub(crate) fn resolve_static_command_scope<'py>( project: &Bound<'py, PyAny>, selection: &Bound<'py, PyAny>, auto_load_sources: bool, @@ -113,7 +113,7 @@ fn resolve_static_command_scope<'py>( /// Resolve canonical locations and source routing without warehouse inventory. #[pyfunction] #[pyo3(signature = (**kwargs))] -fn resolve_static_relation_context<'py>( +pub(crate) fn resolve_static_relation_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index c1d782f835..940c305f91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -2,7 +2,7 @@ use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, within, + KeywordArguments, call_with, within, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -11,7 +11,7 @@ use crate::bindings::_helpers::planner_changes::detect::detect_changes; use crate::bindings::_helpers::planner_entries::test_planning::{ PlanningRequest, SqlTestPlanning, start_planning, }; -use crate::bindings::_helpers::planner_entries::values::{COMPILE_TYPES, native, replaced}; +use crate::bindings::_helpers::planner_entries::values::{native, replaced}; use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::execution_changes::{ @@ -33,6 +33,7 @@ use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; @@ -52,7 +53,7 @@ fn scopes_and_cursor_overrides<'py>( )?; let selected = class_field(&scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; let models_by_name = native_field(&selected, "models_by_name")?; - let model_type = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("MODEL")?; + let model_type = registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut models: Vec> = Vec::new(); for key in native_field(&selected, "selected_keys")?.try_iter()? { let key = key?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index e9f5542fd0..a79330227f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,8 +1,8 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_driver::python_calls::keywords; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::discovery::guidance::microbatch_guidance; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; use crate::bindings::_helpers::planner_changes::detect::detect_model_changes; @@ -10,8 +10,7 @@ use crate::bindings::_helpers::planner_entries::models::{ ModelPlanContext, PlanEntriesRequest, PlanEntryBuildInputs, PlannerRelationsContext, }; use crate::bindings::_helpers::planner_entries::values::{ - COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, - string, + MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, string, }; use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::full_refresh::{ @@ -225,7 +224,7 @@ fn check_binding_diagnostics<'py>( selected_keys: &Bound<'py, PyAny>, ) -> PyResult<()> { let py = project.py(); - let sql_test = imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SQL_TEST")?; + let sql_test = registry::class(py, RecordId::CompiledResourceType)?.getattr("SQL_TEST")?; let mut errors: Vec> = Vec::new(); for diagnostic in native_field(project, "diagnostics")?.try_iter()? { let diagnostic = diagnostic?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs index 9c13e79504..ed2e2d34d0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/test_planning.rs @@ -13,11 +13,13 @@ use pyo3::sync::PyOnceLock; use pyo3::types::PyDict; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::native_threads::exit_signal::signal_interpreter_exit; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_entries::plan_output::plan_selected_sql_tests; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; const TEST_PLANNING_THREAD: &str = "sqlbuild-test-planning"; @@ -158,21 +160,17 @@ pub(crate) struct PlanningRequest { pub(crate) sql_test_selection: Py, } -/// `plan_selected_sql_tests(**request)`, looked up on the module so tests can replace it. +/// Plan the request's selected SQL tests, reporting a panic as a native compiler error. fn plan(py: Python<'_>, request: &PlanningRequest) -> Planned { - call_with( - ®istry::native(py, NativeAttr::PlanSelectedSqlTests)?, - &[ - ("project", request.project.bind(py).clone()), - ("adapter", request.adapter.bind(py).clone()), - ("selected_keys", request.selected_keys.bind(py).clone()), - ( - "sql_test_selection", - request.sql_test_selection.bind(py).clone(), - ), - ], - ) - .map(Bound::unbind) + compiler_guard(|| { + plan_selected_sql_tests( + request.project.bind(py), + request.adapter.bind(py), + request.selected_keys.bind(py), + request.sql_test_selection.bind(py), + ) + .map(Bound::unbind) + }) } /// Plan on a native thread when `enabled`, or resolve at once to no planned tests. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs index 0bdb855ac3..2e0bdc8483 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/values.rs @@ -9,7 +9,6 @@ use pyo3::types::PyString; pub(super) const MICROBATCH_CONCURRENCY_MESSAGE: &str = "batch_concurrency > 1 requires concurrent microbatches; selected model(s): "; -pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; /// `sqlbuild._native.name(**pairs)`. pub(super) fn native<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs index 2dfaca4bd6..2bfa79d714 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/identities.rs @@ -78,16 +78,29 @@ pub(crate) fn build_direct_model_version_identities<'py>( &["seeds", "source_version_hashes", "hook_functions"], ), )?; - let functions: Bound<'py, PyAny> = arguments.value("functions")?; - let scope: Bound<'py, PyAny> = arguments.value("scope")?; - let dialect: Option = arguments.value("dialect")?.extract()?; - let function_hashes: Bound<'py, PyDict> = - function_local_hashes(&functions, dialect.as_deref())?; - let hooks: Bound<'py, PyAny> = arguments.value("hook_functions")?; + direct_model_version_identities( + (&arguments.value("functions")?, &arguments.value("scope")?), + arguments.value("dialect")?.extract()?, + ( + &arguments.value("seeds")?, + &arguments.value("source_version_hashes")?, + &arguments.value("hook_functions")?, + ), + ) +} + +/// Current direct model identities of `(functions, scope)` with optional seeds, source hashes and hooks. +pub(crate) fn direct_model_version_identities<'py>( + (functions, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + dialect: Option, + (seeds, source_hashes, hooks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = functions.py(); + let function_hashes: Bound<'py, PyDict> = function_local_hashes(functions, dialect.as_deref())?; let hook_hashes: Bound<'py, PyDict> = if hooks.is_none() { PyDict::new(py) } else { - hook_version_hashes(&hooks)? + hook_version_hashes(hooks)? }; let mut graph = IdentityGraph::default(); for function in functions.try_iter()? { @@ -106,7 +119,6 @@ pub(crate) fn build_direct_model_version_identities<'py>( } let seed_hashes: Bound<'py, PyDict> = PyDict::new(py); let seed_metadata: Bound<'py, PyDict> = PyDict::new(py); - let seeds: Bound<'py, PyAny> = arguments.value("seeds")?; if !seeds.is_none() { for seed in seeds.try_iter()? { let seed: Bound<'py, PyAny> = seed?; @@ -125,9 +137,8 @@ pub(crate) fn build_direct_model_version_identities<'py>( }, ); } - let source_hashes: Bound<'py, PyAny> = arguments.value("source_version_hashes")?; if source_hashes.is_truthy()? { - for (name, source_hash) in string_pairs(&source_hashes)? { + for (name, source_hash) in string_pairs(source_hashes)? { graph.nodes.insert( key_of(SOURCE_RESOURCE, &name), IdentityNode { @@ -139,8 +150,8 @@ pub(crate) fn build_direct_model_version_identities<'py>( } let model_metadata: Bound<'py, PyDict> = PyDict::new(py); let model_local: Bound<'py, PyDict> = PyDict::new(py); - let models_by_name: Bound<'py, PyAny> = native_field(&scope, "models_by_name")?; - for key in native_field(&scope, "execution_order")?.try_iter()? { + let models_by_name: Bound<'py, PyAny> = native_field(scope, "models_by_name")?; + for key in native_field(scope, "execution_order")?.try_iter()? { let key: Bound<'py, PyAny> = key?; let order_key: GraphKey = graph_key(&key)?; graph.execution_order.push(order_key.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs index 79ffcd78a0..b8e3e1006d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/fingerprint.rs @@ -35,15 +35,27 @@ pub(crate) fn build_migration_fingerprint( let (key, value): (String, String) = item?.extract()?; identities.insert(key, value); } - let sql: String = arguments.value("query_sql")?.extract()?; - let metadata: String = arguments.value("metadata_json")?.extract()?; let dialect: Option = arguments.value("dialect")?.extract()?; - migration_fingerprint::migration_fingerprint( - &sql, - &metadata, + fingerprint( + &arguments.value("query_sql")?.extract::()?, + &arguments.value("metadata_json")?.extract::()?, &identities, + dialect.as_deref(), + ) +} + +/// The migration fingerprint of SQL and metadata through rename identities, `generic` by default. +pub(crate) fn fingerprint( + sql: &str, + metadata: &str, + identities: &BTreeMap, + dialect: Option<&str>, +) -> PyResult> { + migration_fingerprint::migration_fingerprint( + sql, + metadata, + identities, dialect - .as_deref() .filter(|value| !value.is_empty()) .unwrap_or("generic"), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index 4cc53740ba..be23229cb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -1,33 +1,60 @@ //! Expected version identities and stale-warning change detection. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_changes::detect::scope_model_changes; +use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_phases::contexts::class_field; +use crate::bindings::_helpers::planner_phases::models::IdentityWork; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, replace}; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use pyo3::IntoPyObject; +use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; +use pyo3::sync::PyOnceLock; use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyDict, PyDictMethods}; -use std::collections::HashSet; +use pyo3::{Py, PyRefMut, Python}; +use std::collections::{BTreeMap, HashSet}; + +static IDENTITY_WORK: PyOnceLock> = PyOnceLock::new(); + +fn identity_work(py: Python<'_>) -> PyResult> { + Ok(IDENTITY_WORK + .get_or_try_init(py, || Py::new(py, IdentityWork::default()))? + .bind(py) + .borrow_mut()) +} + +/// Report identity builds and per-model stale-warning change detections, optionally resetting them. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn planner_identity_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult<(u64, BTreeMap)> { + let mut work = identity_work(py)?; + let info = (work.identity_builds, work.stale_detections.clone()); + if reset { + *work = IdentityWork::default(); + } + Ok(info) +} fn version_identities<'py>( project: &Bound<'py, PyAny>, scope: Bound<'py, PyAny>, ) -> PyResult> { - native( - project.py(), - NativeAttr::BuildDirectModelVersionIdentities, - &[ - ("functions", native_field(project, "functions")?), - ("seeds", native_field(project, "seeds")?), - ("scope", scope), - ("hook_functions", native_field(project, "hook_functions")?), - ("dialect", native_field(project, "sql_analysis_dialect")?), - ], + identity_work(project.py())?.identity_builds += 1; + direct_model_version_identities( + (&native_field(project, "functions")?, &scope), + native_field(project, "sql_analysis_dialect")?.extract()?, + ( + &native_field(project, "seeds")?, + &project.py().None().into_bound(project.py()), + &native_field(project, "hook_functions")?, + ), ) } @@ -153,32 +180,29 @@ pub(crate) fn detect_stale_warning_changes<'py>( let stale_identities = class_field(identities, |i: &PlannerIdentityContext| { &i.stale_warning_identities })?; - let remaining_changes = native( - py, - NativeAttr::DetectModelChangesInScope, - &[ - ("project", project.clone()), - ( - "scope", - replace( - &stale_scope, - &[("selected_keys", frozenset(py, &remaining)?)], - )?, - ), - ("snapshot", snapshot.clone()), - ( - "query_change_tracking", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ( - "expected_version_hashes", - native_field(&stale_identities, "model_version_hashes")?, - ), - ( - "expected_metadata_jsons", - native_field(&stale_identities, "model_metadata_jsons")?, - ), - ], + { + let mut work = identity_work(py)?; + for key in &remaining { + *work + .stale_detections + .entry(native_field(key, "name")?.extract()?) + .or_default() += 1; + } + } + let remaining_changes = scope_model_changes( + ( + project, + &replace( + &stale_scope, + &[("selected_keys", frozenset(py, &remaining)?)], + )?, + snapshot, + ), + Some(true), + ( + native_field(&stale_identities, "model_version_hashes")?, + native_field(&stale_identities, "model_metadata_jsons")?, + ), )?; let models = PyDict::new(py); for (name, change) in execution_changes @@ -190,7 +214,7 @@ pub(crate) fn detect_stale_warning_changes<'py>( models.set_item(name, change)?; } } - for (name, change) in remaining_changes.downcast::()?.iter() { + for (name, change) in remaining_changes.iter() { models.set_item(name, change)?; } native( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index baecdfe807..dcb5eee1d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -1,5 +1,7 @@ //! Plan output fields gathered across the plan phases before the output is built once. +use std::collections::BTreeMap; + use pyo3::prelude::{Bound, Py, PyAny}; use pyo3::pyclass; @@ -68,3 +70,11 @@ pub(crate) struct PlannerIdentityContext { pub(crate) version_identities: Py, pub(crate) stale_warning_identities: Py, } + +/// Planner identity work since the last reset, for work-count tests. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct IdentityWork { + pub(crate) identity_builds: u64, + pub(crate) stale_detections: BTreeMap, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs index fa2922368e..e1198f52f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/objects.rs @@ -172,12 +172,7 @@ pub(crate) fn model_key<'py>( &[ ( "resource_type", - imported( - py, - "sqlbuild.compiler.compile.types", - "CompiledResourceType", - )? - .getattr("MODEL")?, + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?, ), ("name", name.clone()), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 7004855328..1b3478de46 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -17,12 +17,12 @@ use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ native, replace, sorted_names, strings, text, tuple, }; -use crate::bindings::_helpers::planner_phases::retention::plan_retention; +use crate::bindings::_helpers::planner_phases::retention::{plan_retention, plan_table_types}; use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; @@ -228,15 +228,7 @@ pub(crate) fn add_storage_policies<'py>( ], )?; let selected = class_field(scopes, |s: &PlannerScopeResolution| &s.selected_scope)?; - let table_types = native( - py, - NativeAttr::PlanTableTypes, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("scope", selected.clone()), - ], - )?; + let table_types = plan_table_types(runtime, warehouse, &selected)?; let retention = plan_retention(runtime, warehouse, &selected)?; output.set("model_entries", entries); output.set("migration_entries", migration_entries); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 5092e097f9..5c7f3e8641 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -16,6 +16,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::identities::build_planner_identity_context, module )?)?; + module.add_function(wrap_pyfunction!( + super::identities::planner_identity_work_info, + module + )?)?; module.add_function(wrap_pyfunction!( super::identities::detect_stale_warning_changes_of, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs index 4b6548cff7..f7a0043f37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/execution.rs @@ -205,12 +205,7 @@ fn complete_fixtures<'py>( ("MODEL", "__ref__"), ("SEED", "__seed__"), ] { - let kind = imported( - py, - "sqlbuild.compiler.compile.types", - "CompiledResourceType", - )? - .getattr(kind)?; + let kind = registry::class(py, RecordId::CompiledResourceType)?.getattr(kind)?; groups.push(PyTuple::new( py, [kind, extract_fixtures(scenario, prefix)?.into_any()], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs index 429e968709..41cc14bbb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/loaders.rs @@ -1,13 +1,13 @@ //! Loader dependency mapping and synthetic source boundary construction. -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::spec_resolution::loader_parts; use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::graph::views::edge_dict; use crate::bindings::_helpers::planner_scope::values::{ dependency_names, edges, keys, loader_graph, loader_names, project_graph, }; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; @@ -42,19 +42,13 @@ pub(crate) fn loader_to_source_entry<'py>( .into_pyobject(py)? .into_any(); } - let parts = call_with( - ®istry::native(py, NativeAttr::LoaderDestinationPartsFunction)?, - &[ - ("destination", destination), - ( - "default_database", - native_field(project, "effective_target_database")?, - ), - ( - "default_schema", - native_field(project, "effective_target_schema")?, - ), - ], + let parts = loader_parts( + py, + &destination.extract::()?, + ( + Some(native_field(project, "effective_target_database")?), + Some(native_field(project, "effective_target_schema")?), + ), )?; let kwargs = PyDict::new(py); kwargs.set_item("name", &name)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs index a6f82e5a16..72ac0202d4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scope/scope.rs @@ -2,11 +2,18 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::graph::keys::KeyCache; +use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::views::lineage_graph_names; +use crate::bindings::_helpers::planner_identity::graph::{ + build_downstream_deps, build_execution_edge_origins, build_execution_upstream_deps, + topologically_order_keys, +}; use crate::bindings::_helpers::planner_scope::loaders::expand_selected_loader_dependencies; use crate::bindings::_helpers::planner_scope::values::{edges, keys}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; +use crate::bindings::_helpers::planner_selection::selectors::resolve_graph_selection; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::PyDictMethods; @@ -67,22 +74,14 @@ pub(crate) fn build_planner_scope<'py>( let select = default_tuple(&args.value("select")?); let exclude = default_tuple(&args.value("exclude")?); let auto_load = args.value("auto_load_sources")?.is_truthy()?; - let mut upstream = - registry::native(py, NativeAttr::BuildExecutionUpstreamDeps)?.call1((&project,))?; - let mut downstream = - registry::native(py, NativeAttr::BuildDownstreamDeps)?.call1((&upstream,))?; - let lineage = native_field(&project, "lineage_graph")?; - let all_keys = registry::native(py, NativeAttr::LineageGraphNames)?.call1((&lineage,))?; + let mut upstream = build_execution_upstream_deps(&project)?.into_any(); + let mut downstream = build_downstream_deps(&upstream)?.into_any(); + let lineage: Bound<'py, NativeProjectGraph> = + native_field(&project, "lineage_graph")?.cast_into()?; + let all_keys = lineage_graph_names(py, lineage.borrow())?.into_any(); let mut selected = args.value("selected_keys")?; if selected.is_none() { - selected = call_with( - ®istry::native(py, NativeAttr::ResolveGraphSelection)?, - &[ - ("graph", lineage), - ("select", select.clone()), - ("exclude", exclude), - ], - )?; + selected = resolve_graph_selection(lineage.borrow(), &select, &exclude)?.into_any(); } let names: Vec<(String, (String, String))> = all_keys .call_method0("items")? @@ -118,17 +117,10 @@ pub(crate) fn build_planner_scope<'py>( expand_selected_loader_dependencies(&project, &selected, &upstream, Some(&executable))?; selected = expanded.into_any(); upstream = deps.into_any(); - downstream = registry::native(py, NativeAttr::BuildDownstreamDeps)?.call1((&upstream,))?; + downstream = build_downstream_deps(&upstream)?.into_any(); } - let origins = - registry::native(py, NativeAttr::BuildExecutionEdgeOrigins)?.call1((&project,))?; - let order = call_with( - ®istry::native(py, NativeAttr::TopologicallyOrderKeys)?, - &[ - ("upstream", upstream.clone()), - ("injected_edge_origins", origins), - ], - )?; + let origins = build_execution_edge_origins(&project)?.into_any(); + let order = topologically_order_keys(&upstream, Some(origins))?.into_any(); let models = PyDict::new(py); for model in native_field(&project, "models")?.try_iter()? { let model = model?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs index 863f746c4d..2a77feee21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/selectors.rs @@ -17,7 +17,7 @@ use sqlbuild_analysis::planner_selection::main::unit_test_selector_rejection::un use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::graph::algorithms::closure; use crate::bindings::_helpers::graph::keys::KeyCache; -use crate::bindings::_helpers::graph::project_graph::NativeProjectGraph; +use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::graph::views::{GraphViewMappings, graph_from_view_mappings}; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -148,8 +148,11 @@ pub(crate) fn select_project_graph<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, ) -> PyResult> { - let native: PyRef<'py, NativeProjectGraph> = native_field(graph, "native")?.extract()?; - resolved(native.graph(), (select, exclude)) + let graph: &Bound<'py, CompiledProjectGraph> = graph.cast()?; + resolved( + graph.get().native(graph.py()).get().graph(), + (select, exclude), + ) } /// Model, source, seed and function selectors resolved against compiled graph mappings. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index e8e322e596..b7cb0c1ccf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -142,7 +142,7 @@ fn split_python<'py>( /// Move unit-test selectors out of resource selectors, or reject them for this command. #[pyfunction] #[pyo3(signature = (*, select, exclude, discovered_inputs, accepts_unit_tests))] -fn split_project_unit_test_selectors<'py>( +pub(crate) fn split_project_unit_test_selectors<'py>( select: &Bound<'py, PyAny>, exclude: &Bound<'py, PyAny>, discovered_inputs: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index d30cd41696..23ca89d791 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -203,7 +203,7 @@ pub(crate) fn audit_is_selected( .call1((kind,))? .getattr("value")?; let resource_type = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.call1((target_value,))?; + registry::class(py, RecordId::CompiledResourceType)?.call1((target_value,))?; let key = call_with( ®istry::class(py, RecordId::CompiledObjectKey)?, &[("resource_type", resource_type), ("name", name)], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs index 68778fbf3a..e08ffec28f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audits.rs @@ -28,7 +28,7 @@ fn object_key<'py>(py: Python<'py>, kind: &str, name: &str) -> PyResult> = PyOnceLock::new(); /// Count one batch resolution of SQL-test model chains. pub(crate) fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { @@ -33,8 +34,12 @@ pub(crate) fn count_chain_resolution(py: Python<'_>) -> PyResult<()> { Ok(()) } -fn work_counts(py: Python<'_>) -> PyResult> { - Ok(registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?) +/// The extension-wide fixture read cache and planning work counters. +pub(crate) fn work_counts(py: Python<'_>) -> PyResult> { + Ok(FIXTURE_READS + .get_or_try_init(py, || Py::new(py, FixtureReadCacheHost::default()))? + .bind(py) + .clone()) } /// The planning context of a project, over its models or the given name-to-model map. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 0124530157..ec5d29ca61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -18,13 +18,10 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; -use crate::bindings::_helpers::planner_strategy::models::{ - FixtureReadCacheHost, RelationFixturePlanningContext, -}; +use crate::bindings::_helpers::planner_strategy::fixture_context::work_counts; +use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::planner_strategy::types::FixtureKey; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; use crate::bindings::types::CompilerDetach; /// The columns each fixture must supply for the chain's models, and the fixtures read through a star. @@ -84,8 +81,7 @@ pub(crate) fn required_fixture_columns( stars: model_stars(&model, &schema, adapter)?, }); } - let owner = - registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?; + let owner = work_counts(py)?; let mut owner = owner.borrow_mut(); let cache = std::mem::take(&mut owner.cache); let (cache, result) = catch_compiler_panic(|| { @@ -175,10 +171,6 @@ fn model_stars( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add( - "_fixture_read_cache", - pyo3::Py::new(module.py(), FixtureReadCacheHost::default())?, - )?; module.add_function(wrap_pyfunction!(fixture_read_cache_info, module)?) } @@ -186,8 +178,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { #[pyfunction] #[pyo3(signature = (*, reset = false))] fn fixture_read_cache_info(py: pyo3::Python<'_>, reset: bool) -> PyResult<(u64, usize)> { - let owner = - registry::native(py, NativeAttr::FixtureReadCache)?.cast_into::()?; + let owner = work_counts(py)?; let mut owner = owner.borrow_mut(); if reset { owner.cache = Default::default(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index f7e93f1c97..d1c4b85a8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -4,22 +4,24 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::planner_changes::metadata::without_declared_columns_hash; +use crate::bindings::_helpers::planner_migrations::fingerprint::fingerprint; use crate::bindings::_helpers::planner_migrations::fingerprint::migration_fingerprint_ref_names; use crate::bindings::_helpers::planner_phases::objects::object; -use crate::bindings::_helpers::planner_phases::objects::{optional_text, text}; +use crate::bindings::_helpers::planner_warehouse::models::FingerprintWork; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use pyo3::PyRefMut; use pyo3::exceptions::{PyOSError, PyValueError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyFloat, PyList, PyListMethods, PyTuple}; -use pyo3::{pyclass, pymethods}; +use pyo3::{pyclass, pyfunction, pymethods}; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_entry::fingerprint_cache_entry; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_key::fingerprint_cache_key; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_lookup::fingerprint_cache_lookup; use sqlbuild_analysis::planner_warehouse::main::fingerprint_cache_shared_digest::fingerprint_cache_shared_digest; use sqlbuild_analysis::planner_warehouse::models::FingerprintCacheKey; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; +use std::hash::{DefaultHasher, Hash, Hasher}; const CACHE_DIRECTORY_NAME: &str = "migration-fingerprints-v1"; const CACHE_DATABASE_NAME: &str = "migration-fingerprints.sqlite3"; @@ -31,6 +33,39 @@ const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_ type MemoKey = (String, String, Option, Vec<(String, String)>); +static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); + +fn fingerprint_work(py: Python<'_>) -> PyResult> { + Ok(FINGERPRINT_WORK + .get_or_try_init(py, || Py::new(py, FingerprintWork::default()))? + .bind(py) + .borrow_mut()) +} + +fn count_computation(py: Python<'_>, key: &MemoKey) -> PyResult<()> { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + let mut work = fingerprint_work(py)?; + work.computations += 1; + work.distinct.insert(hasher.finish()); + Ok(()) +} + +/// Report migration fingerprints computed and distinct requests among them, optionally resetting. +#[pyfunction] +#[pyo3(signature = (*, reset = false))] +pub(crate) fn migration_fingerprint_work_info( + py: Python<'_>, + reset: bool, +) -> PyResult<(u64, usize)> { + let mut work = fingerprint_work(py)?; + let info = (work.computations, work.distinct.len()); + if reset { + *work = FingerprintWork::default(); + } + Ok(info) +} + /// Reuse migration fingerprints across rename maps, planning phases and invocations. #[pyclass(module = "sqlbuild._native")] pub(crate) struct MigrationFingerprintCache { @@ -313,20 +348,10 @@ impl MigrationFingerprintCache { let stored = match self.stored(py, &cache_key)? { Some(stored) => stored, None => { - let identities = PyDict::new(py); - for (name, identity) in &key.3 { - pyo3::types::PyDictMethods::set_item(&identities, name, identity)?; - } - let computed: Option = call_with( - ®istry::native(py, NativeAttr::BuildMigrationFingerprint)?, - &[ - ("query_sql", text(py, &key.0)), - ("metadata_json", text(py, &key.1)), - ("ref_identities", identities.into_any()), - ("dialect", optional_text(py, key.2.as_deref())), - ], - )? - .extract()?; + count_computation(py, &key)?; + let identities: BTreeMap = key.3.iter().cloned().collect(); + let computed: Option = + fingerprint(&key.0, &key.1, &identities, key.2.as_deref())?; self.remember(&cache_key, computed.as_deref())?; computed } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs index b48621b451..69d9ef9fe4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod constants; pub(crate) mod cursors; pub(crate) mod fingerprint_cache; +pub(crate) mod models; pub(crate) mod registration; pub(crate) mod semantic_sources; pub(crate) mod snapshot; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs new file mode 100644 index 0000000000..6c19c1fcea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/models.rs @@ -0,0 +1,13 @@ +//! Warehouse planning state shared across planner runs. + +use std::collections::BTreeSet; + +use pyo3::pyclass; + +/// Migration fingerprints computed since the last reset, for work-count tests. +#[pyclass] +#[derive(Debug, Default)] +pub(crate) struct FingerprintWork { + pub(crate) computations: u64, + pub(crate) distinct: BTreeSet, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs index 5d6419d550..939d91e174 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs @@ -5,6 +5,10 @@ use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; + module.add_function(wrap_pyfunction!( + super::fingerprint_cache::migration_fingerprint_work_info, + module + )?)?; module.add_function(wrap_pyfunction!( super::snapshot::gather_warehouse_snapshot, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index fda2adb7e8..e68586513e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -1,8 +1,7 @@ //! What each model, seed, source and table function declares about its columns before analysis. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::model_input_view::{ @@ -22,7 +21,8 @@ const NON_NULL_MEMBER: &str = "NON_NULL"; const UNKNOWN_MEMBER: &str = "UNKNOWN"; /// The three per-relation declaration maps the analysis starts from. -type DeclaredRelations<'py> = (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); +pub(crate) type DeclaredRelations<'py> = + (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); /// `{column.name: column.type or "UNKNOWN"}`, or the declared type unchanged when `exact`. fn column_types<'py>( @@ -73,8 +73,9 @@ fn table_function_name(function: &Bound<'_, PyAny>) -> PyResult { } /// Declared column types, declared nullability and the authoritative binding schemas. -#[pyfunction] -fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn declared_relations<'py>( + inputs: Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = inputs.py(); let nullability: Bound<'py, PyAny> = registry::class(py, RecordId::InferredNullability)?; let types: Bound<'py, PyDict> = PyDict::new(py); @@ -146,7 +147,3 @@ fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult) -> PyResult<()> { - module.add_function(wrap_pyfunction!(declared_relations, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs index 4b6c8aa7b5..160f660c9b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -6,7 +6,7 @@ use crate::bindings::_helpers::project_assembly::models::{ NativeModelFacts, NativeProjectFacts, NativeProjectResources, NativeResourceFailure, }; use crate::bindings::_helpers::project_assembly::{ - declared_relations, native_resources, resources, syntax_checks, syntax_validation, + native_resources, resources, syntax_checks, syntax_validation, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -14,7 +14,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; - declared_relations::register(module)?; native_resources::register(module)?; syntax_validation::register(module)?; syntax_checks::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs index 7503096c23..b8ded37b92 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs @@ -18,7 +18,7 @@ use crate::bindings::_helpers::records::registry; const GENERIC_DIALECT: &str = "generic"; /// `(placeholder defaults, dialect)` a statement parses with. -type ParseOptions = (Option>, Option); +pub(crate) type ParseOptions = (Option>, Option); /// What every statement of one model is parsed with. struct SyntaxContext<'a, 'py> { @@ -145,7 +145,7 @@ fn check_hook_value( /// Validate the SQL of one model's hook value, raising the first parse failure. #[pyfunction] #[pyo3(signature = (value, hook_name, model, parsing=(None, None)))] -fn validate_hook_sql_syntax( +pub(crate) fn validate_hook_sql_syntax( value: Bound<'_, PyAny>, hook_name: &str, model: (String, Bound<'_, PyAny>), @@ -179,7 +179,7 @@ fn validate_hook_sql_syntax( /// Validate that a model's query parses, raising its syntax error. #[pyfunction] #[pyo3(signature = (query_sql, model, parsing=(None, None)))] -fn validate_sql_syntax( +pub(crate) fn validate_sql_syntax( py: Python<'_>, query_sql: &str, model: (String, Bound<'_, PyAny>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index 257e448d25..4bd16132a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -3,6 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::NativeAttr; @@ -23,9 +25,9 @@ pub(crate) fn project_adapter<'py>( Some(path) => path, None => imported(py, "pathlib", "Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let adapter_name: Bound<'py, PyAny> = call_with( ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs index e890555235..5597535659 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs @@ -4,14 +4,11 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::relation_keys::entry_relation_key_row; -/// Module of the compiled resource type enum. -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// `CompilerDiagnostic` positions. const DIAGNOSTIC_PHASE_FIELD: usize = 0; const DIAGNOSTIC_SEVERITY_FIELD: usize = 1; @@ -36,13 +33,6 @@ const TEST_BLOCK_INDEX_FIELD: usize = 10; const TEST_EXPLICIT_NAME_FIELD: usize = 11; const TEST_PARENT_NAME_FIELD: usize = 12; -/// `class(value)`. -fn coerce(py: Python<'_>, value: &Py, (module, class): (&str, &str)) -> PyResult> { - Ok(imported(py, module, class)? - .call1((value.bind(py),))? - .unbind()) -} - /// The native string enum `class(value)`. fn coerce_native(py: Python<'_>, value: &Py, class: RecordId) -> PyResult> { Ok(registry::class(py, class)? @@ -55,7 +45,7 @@ fn coerce_resource_type(py: Python<'_>, value: &Py) -> PyResult if value.is_none(py) { return Ok(value.clone_ref(py)); } - coerce(py, value, (COMPILE_TYPES_MODULE, "CompiledResourceType")) + coerce_native(py, value, RecordId::CompiledResourceType) } /// `CompilerDiagnostic`: enum fields coerced, and its location and path, line and column aligned. @@ -124,11 +114,7 @@ pub(crate) fn finish_resource_typed( py: Python<'_>, mut values: Vec>, ) -> PyResult>> { - values[0] = coerce( - py, - &values[0], - (COMPILE_TYPES_MODULE, "CompiledResourceType"), - )?; + values[0] = coerce_native(py, &values[0], RecordId::CompiledResourceType)?; Ok(values) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index 8025590a3b..cfc3cf2d89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -17,8 +17,7 @@ use crate::bindings::_helpers::record_behaviours::cursors::{ snapshot_watermarks_available, }; use crate::bindings::_helpers::record_behaviours::discovery::{ - constant_logical_type, constant_scalar_type, graph_all_keys, graph_downstream_deps, - graph_lineage_views, graph_path_index, graph_tag_index, graph_upstream_deps, + constant_logical_type, constant_scalar_type, }; use crate::bindings::_helpers::record_behaviours::durations::{ duration_add_to, duration_fixed_seconds, duration_has_calendar_component, @@ -212,11 +211,6 @@ pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ ), ("ConstantDeclaration", "logical_type", constant_logical_type), ("ConstantDeclaration", "scalar_type", constant_scalar_type), - ("ProjectGraph", "upstream_deps", graph_upstream_deps), - ("ProjectGraph", "downstream_deps", graph_downstream_deps), - ("ProjectGraph", "tag_index", graph_tag_index), - ("ProjectGraph", "path_index", graph_path_index), - ("ProjectGraph", "all_keys", graph_all_keys), ("CompilerDiagnostic", "is_error", diagnostic_is_error), ( "ModelInputBuildContext", @@ -267,7 +261,6 @@ pub(crate) const BEHAVIOUR_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ ]; /// Computed attributes of compiler shapes that a record computes once and keeps. pub(crate) const BEHAVIOUR_CACHED_PROPERTIES: &[(&str, &str, ClassProperty)] = &[ - ("ProjectGraph", "lineage_views", graph_lineage_views), ("LoadedMacro", "injects_context", macro_injects_context), ("CompactLineageFacts", "facts", facts_projected), ("CompiledProject", "lineage_graph", project_lineage_graph), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs index 78d83afa8e..bf2008d88c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/discovery.rs @@ -1,11 +1,9 @@ //! Computed attributes of the natively hosted discovery and pipeline records. +use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult}; -use crate::bindings::_helpers::graph::views::lineage_graph_views; -use crate::bindings::_helpers::records::fields::field as native_field; - /// SQL scalar type labels by logical value kind. const SCALAR_TYPE_LABELS: [(&str, &str); 6] = [ ("string", "VARCHAR"), @@ -37,40 +35,3 @@ pub(crate) fn constant_scalar_type(declaration: &Bound<'_, PyAny>) -> PyResult

) -> PyResult> { - let native = native_field(graph, "native")?; - Ok(lineage_graph_views(graph.py(), native.extract()?)?.unbind()) -} - -fn view(graph: &Bound<'_, PyAny>, name: &str) -> PyResult> { - Ok(native_field(graph, "lineage_views")? - .getattr(name)? - .unbind()) -} - -/// `ProjectGraph.upstream_deps`. -pub(crate) fn graph_upstream_deps(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "upstream_deps") -} - -/// `ProjectGraph.downstream_deps`. -pub(crate) fn graph_downstream_deps(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "downstream_deps") -} - -/// `ProjectGraph.tag_index`. -pub(crate) fn graph_tag_index(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "tag_index") -} - -/// `ProjectGraph.path_index`. -pub(crate) fn graph_path_index(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "path_index") -} - -/// `ProjectGraph.all_keys`. -pub(crate) fn graph_all_keys(graph: &Bound<'_, PyAny>) -> PyResult> { - view(graph, "all_keys") -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index afdc0ce45c..f6a44b1f4d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -63,6 +63,7 @@ native_ids! { ColumnLineageConfidence => "ColumnLineageConfidence", ColumnLineageEdge => "ColumnLineageEdge", ColumnLineageMode => "ColumnLineageMode", + CompiledResourceType => "CompiledResourceType", ColumnLineageSource => "ColumnLineageSource", ColumnMigrationDecision => "ColumnMigrationDecision", ColumnMigrationEvent => "ColumnMigrationEvent", @@ -350,10 +351,8 @@ native_ids! { NativeRulesEvaluation => "NativeRulesEvaluation", NativeSqlTestArtifact => "NativeSqlTestArtifact", NativeSqlTestAssembly => "NativeSqlTestAssembly", - NativeSqlTestChainRequest => "NativeSqlTestChainRequest", NativeSqlTestFailure => "NativeSqlTestFailure", NativeSqlTestPlan => "NativeSqlTestPlan", - NativeSqlTestPlanningRequest => "NativeSqlTestPlanningRequest", NestedProgressChildRow => "NestedProgressChildRow", NormalizedDbtProfileConnection => "NormalizedDbtProfileConnection", OldNameView => "OldNameView", @@ -393,7 +392,6 @@ native_ids! { PolyglotAnalysisResult => "PolyglotAnalysisResult", Project => "Project", ProjectConfig => "ProjectConfig", - ProjectGraph => "ProjectGraph", ProjectPath => "ProjectPath", PythonHookEntry => "PythonHookEntry", PythonIdentityDependency => "PythonIdentityDependency", @@ -661,26 +659,17 @@ native_ids! { pub(crate) enum NativeAttr { AnalyzeCompileProject => "analyze_compile_project", ApplyCommandWarehouse => "apply_command_warehouse", - ApplyConfiguredMicrobatchLimits => "apply_configured_microbatch_limits", - ApplyDeferredLocations => "apply_deferred_locations", ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", - ApplyUnitTestSelectors => "apply_unit_test_selectors", - AssembleNativeProjectResources => "assemble_native_project_resources", - AssembleNativeSqlTests => "assemble_native_sql_tests", AssembleProject => "assemble_project", - AssertNoUnresolvedSqlMarkers => "assert_no_unresolved_sql_markers", AttachChecks => "attach_checks", AttachResources => "attach_resources", AttachSqlbuildPlanOutput => "attach_sqlbuild_plan_output", - AuditIsSelected => "audit_is_selected", AuditProgressReporter => "AuditProgressReporter", BindingPositions => "BindingPositions", BuildAuditCommandPlan => "build_audit_command_plan", BuildCompileDbtManifestIndex => "build_compile_dbt_manifest_index", BuildCompileInputs => "build_compile_inputs", - BuildCompiledFunctionFingerprintSql => "build_compiled_function_fingerprint_sql", - BuildCompiledProject => "build_compiled_project", BuildCursorInputRelations => "build_cursor_input_relations", BuildDagJson => "build_dag_json", BuildDbtCombinedGraph => "build_dbt_combined_graph", @@ -688,35 +677,21 @@ native_ids! { BuildDbtInitProjectToml => "build_dbt_init_project_toml", BuildDeclarationScope => "build_declaration_scope", BuildDeferredLocations => "build_deferred_locations", - BuildDirectModelVersionIdentities => "build_direct_model_version_identities", BuildDirectSemanticUses => "build_direct_semantic_uses", BuildDisplayOnlySqlbuildPlan => "build_display_only_sqlbuild_plan", - BuildDownstreamDeps => "build_downstream_deps", - BuildEffectiveConnection => "build_effective_connection", BuildEffectiveConnectionConfig => "build_effective_connection_config", BuildEffectiveRuntimeConfig => "build_effective_runtime_config", - BuildEffectiveSettings => "build_effective_settings", BuildEffectiveSettingsConfig => "build_effective_settings_config", BuildEffectiveTargetConfig => "build_effective_target_config", BuildEffectiveTargetNamespace => "build_effective_target_namespace", - BuildEffectiveVars => "build_effective_vars", - BuildExecutionEdgeOrigins => "build_execution_edge_origins", BuildExecutionPlan => "build_execution_plan", - BuildExecutionUpstreamDeps => "build_execution_upstream_deps", BuildFailedSqlbuildModelNames => "build_failed_sqlbuild_model_names", - BuildFunctionLocalHashes => "build_function_local_hashes", - BuildFunctionLocations => "build_function_locations", BuildIdentity => "BUILD_IDENTITY", BuildIntermediateSourceMap => "build_intermediate_source_map", BuildManifest => "build_manifest", BuildMergedDbtExecutionArgv => "build_merged_dbt_execution_argv", - BuildMigrationFingerprint => "build_migration_fingerprint", - BuildModelInputs => "build_model_inputs", - BuildModelLocations => "build_model_locations", - BuildModelVersionIdentityMetadataJson => "build_model_version_identity_metadata_json", BuildModelWarnings => "build_model_warnings", BuildPhaseTimingTracker => "BuildPhaseTimingTracker", - BuildPlannerIdentityContext => "build_planner_identity_context", BuildPlannerRelationsContext => "build_planner_relations_context", BuildPlannerScope => "build_planner_scope", BuildPlannerSourceFreshnessResult => "build_planner_source_freshness_result", @@ -724,55 +699,38 @@ native_ids! { BuildProjectColumnLineage => "build_project_column_lineage", BuildProjectGraph => "build_project_graph", BuildProjectGraphFromCompiledProject => "build_project_graph_from_compiled_project", - BuildProjectGraphWithAnalysisSelection => "build_project_graph_with_analysis_selection", BuildProjectRelationTargets => "build_project_relation_targets", BuildPythonRelationTargets => "build_python_relation_targets", BuildPythonRunPlanOutputs => "build_python_run_plan_outputs", BuildPythonSqlRunLifecycle => "build_python_sql_run_lifecycle", - BuildRelationCommandPlan => "build_relation_command_plan", - BuildRelationFixtureCompletion => "build_relation_fixture_completion", - BuildRelationFixtureContext => "build_relation_fixture_context", BuildRuleCatalogue => "build_rule_catalogue", BuildRulesRequest => "build_rules_request", BuildScenarioExecutionPlan => "build_scenario_execution_plan", BuildScenarioHashIndex => "build_scenario_hash_index", BuildScenarioRelationMap => "build_scenario_relation_map", BuildSeedCommandPlan => "build_seed_command_plan", - BuildSeedIdentity => "build_seed_identity", - BuildSeedLocations => "build_seed_locations", BuildSelectedAuditEntries => "build_selected_audit_entries", - BuildSourceLoadEntries => "build_source_load_entries", - BuildSourceLoadMap => "build_source_load_map", BuildSourceReadMap => "build_source_read_map", BuildSqlbuildPlanOutput => "build_sqlbuild_plan_output", BuildStaleOutOfSelectionWarnings => "build_stale_out_of_selection_warnings", BuildStaticPipelineResult => "build_static_pipeline_result", BuildTestCommandPlan => "build_test_command_plan", - BuildUpstreamIntermediateSourceMap => "build_upstream_intermediate_source_map", CheckBuildability => "check_buildability", CheckCompileReuse => "check_compile_reuse", CheckCursorTypeConsistency => "check_cursor_type_consistency", - CheckFullRebuildAllowed => "check_full_rebuild_allowed", - CheckSelectedScopeBuildability => "check_selected_scope_buildability", - ClassifyCursorSqlType => "classify_cursor_sql_type", CliStyle => "CliStyle", - CollectCompileDiagnostics => "collect_compile_diagnostics", CommandWarehouseScopeContext => "_command_warehouse_scope_context", CompareContracts => "compare_contracts", CompileDbtInteropProject => "compile_dbt_interop_project", CompileFrontier => "compile_frontier", - CompileInputsView => "compile_inputs_view", CompileProjectPhase => "compile_project_phase", CompileReuseExcludedRootDirectories => "COMPILE_REUSE_EXCLUDED_ROOT_DIRECTORIES", CompileReuseRacyWindowNs => "COMPILE_REUSE_RACY_WINDOW_NS", CompileStaticCommandContext => "compile_static_command_context", CompiledCodeIdentity => "compiled_code_identity", - CompiledModelRecord => "compiled_model_record", CompiledProjectFacts => "compiled_project_facts", CompiledProjectRecord => "compiled_project_record", - CompleteSemanticProject => "complete_semantic_project", ComputeCursorBounds => "compute_cursor_bounds", - ComputeQueryHash => "compute_query_hash", ConnectionProgressReporter => "ConnectionProgressReporter", ContentDigest => "content_digest", ContextStoreToken => "context_store_token", @@ -780,10 +738,8 @@ native_ids! { CostContext => "CostContext", CostOutputPayload => "cost_output_payload", CurrentNativeProgressProjector => "current_native_progress_projector", - CurrentTerminalEventIndex => "current_terminal_event_index", CursorCompare => "cursor_compare", CursorExclusiveToInclusive => "cursor_exclusive_to_inclusive", - CursorInputsWithoutRows => "cursor_inputs_without_rows", CursorObservedPartition => "cursor_observed_partition", CursorParse => "cursor_parse", CursorRender => "cursor_render", @@ -799,25 +755,13 @@ native_ids! { DbtRunner => "DbtRunner", DebugDbtFromProject => "debug_dbt_from_project", DeclarationRows => "declaration_rows", - DeclaredColumnAliases => "declared_column_aliases", - DeclaredRelations => "declared_relations", DescribeConnectionWarehouse => "describe_connection_warehouse", - DescribeFullRebuildCause => "describe_full_rebuild_cause", DetectChanges => "detect_changes", DetectModelChanges => "detect_model_changes", - DetectModelChangesInScope => "detect_model_changes_in_scope", DetectStaleWarningChanges => "detect_stale_warning_changes", DirectorySnapshotScope => "directory_snapshot_scope", - DiscoverEventExporterFunctions => "discover_event_exporter_functions", - DiscoverHookFunctions => "discover_hook_functions", - DiscoverMaterializationFiles => "discover_materialization_files", DiscoverModelFiles => "discover_model_files", DiscoverModelSchemaFiles => "discover_model_schema_files", - DiscoverProjectInputs => "discover_project_inputs", - DiscoverProviderClasses => "discover_provider_classes", - DiscoverPythonHookFiles => "discover_python_hook_files", - DiscoverPythonModuleFiles => "discover_python_module_files", - DiscoverPythonNodeFunctions => "discover_python_node_functions", DiscoverSchemaFiles => "discover_schema_files", DiscoverSourceFiles => "discover_source_files", DiscoverTestFiles => "discover_test_files", @@ -827,13 +771,8 @@ native_ids! { EffectiveConfigConnection => "effective_config_connection", EffectiveConfigSettings => "effective_config_settings", EffectiveConfigVars => "effective_config_vars", - EffectiveTargetConfig => "effective_target_config", - EffectivelyFullRefreshedModelNames => "effectively_full_refreshed_model_names", - EmptyCursorWindow => "empty_cursor_window", - EmptyInputRebuildRefusal => "empty_input_rebuild_refusal", EnforceModelExecutionLimit => "enforce_model_execution_limit", EnforcePlanSafetyPolicies => "enforce_plan_safety_policies", - EvaluateModelContracts => "evaluate_model_contracts", EvaluateRules => "evaluate_rules", EvaluateRulesRequest => "evaluate_rules_request", EventExporterCommandScope => "EventExporterCommandScope", @@ -845,17 +784,10 @@ native_ids! { ExecuteDbtSqlbuildWork => "execute_dbt_sqlbuild_work", ExecuteDbtWithoutStateTracking => "execute_dbt_without_state_tracking", ExecutionEventWriter => "ExecutionEventWriter", - ExpandProjectUpstreamKeys => "expand_project_upstream_keys", - ExpandSqlMacros => "expand_sql_macros", - ExplicitReferenceReport => "explicit_reference_report", ExplicitReferencesHelp => "explicit_references_help", - ExternallyDescribedModels => "externally_described_models", - ExtractSeedColumns => "extract_seed_columns", FinalizeRuleFindingsRows => "finalize_rule_findings_rows", FindPythonStringLocations => "find_python_string_locations", FinishRuleRun => "finish_rule_run", - FixtureColumnInferences => "FixtureColumnInferences", - FixtureReadCache => "_fixture_read_cache", FormatAuditExecutionJson => "format_audit_execution_json", FormatBuildExecutionJson => "format_build_execution_json", FormatBuildFooter => "format_build_footer", @@ -883,27 +815,18 @@ native_ids! { FormatSkippedTypeProofRules => "format_skipped_type_proof_rules", FormatTestExecutionJson => "format_test_execution_json", GatherDeferredRelations => "gather_deferred_relations", - GatherPlannerWarehouseState => "gather_planner_warehouse_state", GatherRedirectedCursorSnapshots => "gather_redirected_cursor_snapshots", - GatherSourceColumns => "gather_source_columns", - GatherSourceInspection => "gather_source_inspection", GetConfigCursorBound => "get_config_cursor_bound", GetConfigStr => "get_config_str", GetMaterializationType => "get_materialization_type", GetSemanticSourceColumns => "get_semantic_source_columns", - HardCodedRelationRemedy => "hard_coded_relation_remedy", IdenticalRenames => "identical_renames", InferFixtureColumnFacts => "infer_fixture_column_facts", InferNativeExpressionSourceShapes => "infer_native_expression_source_shapes", - InferNativeQueryColumnRows => "infer_native_query_column_rows", InferNativeQueryColumns => "infer_native_query_columns", - InputsWith => "inputs_with", InstallRulesSkills => "install_rules_skills", - KnownDeclaredTypes => "known_declared_types", - KnownFunctionNames => "known_function_names", LaunchRecordOffered => "_launch_record_offered", LaunchRecordWithheld => "_launch_record_withheld", - LineageGraphNames => "lineage_graph_names", ListCostRuns => "list_cost_runs", LoadCompiledDbtManifest => "load_compiled_dbt_manifest", LoadDbtManifestIndex => "load_dbt_manifest_index", @@ -917,80 +840,45 @@ native_ids! { MachineOutputMode => "machine_output_mode", MacroStoreToken => "macro_store_token", MaintainSqlbuildSkills => "maintain_sqlbuild_skills", - MatchProjectRelation => "match_project_relation", MatchProjectSelector => "match_project_selector", MatchingDynamicFamilies => "matching_dynamic_families", MaximumAllowedStart => "maximum_allowed_start", - MergeRecomputedModelChanges => "merge_recomputed_model_changes", - MicrobatchGuidance => "microbatch_guidance", - MigrationFingerprintCache => "MigrationFingerprintCache", - MigrationFingerprintRefNames => "migration_fingerprint_ref_names", - MissingUpstreamMessage => "missing_upstream_message", - ModelDefinitionHash => "model_definition_hash", - ModelHeaderFailure => "model_header_failure", - ModelOutputColumns => "model_output_columns", - ModelPlaceholders => "model_placeholders", ModelPythonHookNames => "model_python_hook_names", - ModelSyntaxChecks => "model_syntax_checks", - NativeCompiledModel => "NativeCompiledModel", - NativeCompiledProject => "NativeCompiledProject", - NativeCompiledSeed => "NativeCompiledSeed", NativeDeclarationContexts => "NativeDeclarationContexts", - NativeModelConfigBuilder => "NativeModelConfigBuilder", - NativeModelInput => "NativeModelInput", NativeModelLoop => "NativeModelLoop", NativeModelValidator => "NativeModelValidator", NativeProgressProjector => "NativeProgressProjector", NativeProgressProjectorContext => "_native_progress_projector_context", NestedCommandProgressCallbacks => "NestedCommandProgressCallbacks", NormalizeAnalysisSql => "normalize_analysis_sql", - NormalizeCursorSnapshotGrain => "normalize_cursor_snapshot_grain", NormalizeDbtProfileConnection => "normalize_dbt_profile_connection", OldNameClaimErrors => "old_name_claim_errors", - OldNameReferenceMessage => "old_name_reference_message", - OldNameSchemasToRead => "old_name_schemas_to_read", OldNameViewSkipReason => "old_name_view_skip_reason", ParseDbtLsJsonLines => "parse_dbt_ls_json_lines", ParseHeaderValues => "parse_header_values", - ParseModelHeaders => "parse_model_headers", ParseProjectSelector => "parse_project_selector", - ParseQueryShape => "parse_query_shape", PlanColumnMigrations => "plan_column_migrations", PlanCompiledSqlTestArtifacts => "plan_compiled_sql_test_artifacts", - PlanCompiledSqlTests => "plan_compiled_sql_tests", PlanDbtInteropCommand => "plan_dbt_interop_command", PlanDbtInteropFromProject => "plan_dbt_interop_from_project", PlanHasExecutableWork => "plan_has_executable_work", PlanModel => "plan_model", PlanReferenceRenames => "plan_reference_renames", - PlanRetention => "plan_retention", PlanRuleFixes => "plan_rule_fixes", PlanRuleRun => "plan_rule_run", - PlanSelectedSqlTests => "plan_selected_sql_tests", PlanSqlTests => "plan_sql_tests", PlanSqlTestsNatively => "plan_sql_tests_natively", PlanStaticSqlTests => "plan_static_sql_tests", - PlanTableTypes => "plan_table_types", PlanningProgressReporter => "PlanningProgressReporter", - PrepareModelHeaderTokens => "prepare_model_header_tokens", PreparedCompileArtifacts => "PreparedCompileArtifacts", - ProjectLineageViews => "project_lineage_views", - ProjectRelativePath => "project_relative_path", ProjectWith => "project_with", - PromotionConflictDiagnostics => "promotion_conflict_diagnostics", - ProveNativeDynamicContracts => "prove_native_dynamic_contracts", PublicDeclarationIndexes => "public_declaration_indexes", - PublicModelSchemaIndex => "public_model_schema_index", PythonReadSourceNamesFor => "python_read_source_names_for", ReadCostRun => "read_cost_run", ReadLatestFingerprints => "read_latest_fingerprints", ReadOldNameViewHistories => "read_old_name_view_histories", ReadSelectedSeedFingerprints => "read_selected_seed_fingerprints", ReadsOutsideTargetLineage => "reads_outside_target_lineage", - RebindDeclarationScope => "rebind_declaration_scope", - RecordCompileInputReads => "record_compile_input_reads", - RecordModelAnalyses => "record_model_analyses", - RecordSettingsClassRead => "record_settings_class_read", RecordTemplateReads => "record_template_reads", RejectUnitTestSelectors => "reject_unit_test_selectors", RejectUnparseableHeaderRewrites => "reject_unparseable_header_rewrites", @@ -999,73 +887,44 @@ native_ids! { RenameHints => "rename_hints", RenamesExplainChange => "renames_explain_change", RenderAuditSql => "render_audit_sql", - RenderConstantDeclaration => "render_constant_declaration", RenderDbtExecutionSummaryFooter => "render_dbt_execution_summary_footer", RenderDbtInteropPlan => "render_dbt_interop_plan", RenderDbtNodeResult => "render_dbt_node_result", RenderDiffErrorJson => "render_diff_error_json", RenderProjectVarText => "render_project_var_text", - RenderSourceReadRelation => "render_source_read_relation", RenderSourceRelation => "render_source_relation", - ReplacesIncrementalRelation => "replaces_incremental_relation", ReportCompileDiagnostic => "report_compile_diagnostic", - ResolveAnalysisModelNames => "resolve_analysis_model_names", - ResolveAttachmentKind => "resolve_attachment_kind", - ResolveBoundedCursorOverride => "resolve_bounded_cursor_override", ResolveCloneBoundary => "resolve_clone_boundary", ResolveCompileAnalysisSelection => "resolve_compile_analysis_selection", - ResolveCompiledSqlTestChains => "resolve_compiled_sql_test_chains", - ResolveConfigFilePath => "resolve_config_file_path", - ResolveConfiguredRuleSelection => "resolve_configured_rule_selection", ResolveCostRun => "resolve_cost_run", - ResolveCursorInputRoles => "resolve_cursor_input_roles", - ResolveCursorIntrinsics => "resolve_cursor_intrinsics", ResolveDbtConfig => "resolve_dbt_config", ResolveDbtConnectionConfig => "resolve_dbt_connection_config", ResolveDbtExecutable => "resolve_dbt_executable", ResolveDbtExecutionInvocation => "resolve_dbt_execution_invocation", ResolveDbtInteropPlan => "resolve_dbt_interop_plan", - ResolveDbtRefReferences => "resolve_dbt_ref_references", ResolveDbtReferenceRelation => "resolve_dbt_reference_relation", ResolveDeferredTargetConfig => "resolve_deferred_target_config", ResolveEffectiveAdapterName => "resolve_effective_adapter_name", - ResolveEffectiveCollectionRendering => "resolve_effective_collection_rendering", - ResolveEffectiveRunScope => "resolve_effective_run_scope", ResolveEffectiveScenarioConfig => "resolve_effective_scenario_config", - ResolveEffectiveTimestampGrain => "resolve_effective_timestamp_grain", ResolveExternalSqlReferenceResolver => "resolve_external_sql_reference_resolver", ResolveFunctionSql => "resolve_function_sql", ResolveFutureCursorSafety => "resolve_future_cursor_safety", - ResolveGraphSelection => "resolve_graph_selection", - ResolveLineageReference => "resolve_lineage_reference", - ResolveMicrobatchLookback => "resolve_microbatch_lookback", - ResolveModelActions => "resolve_model_actions", ResolveModelDeferralInputs => "resolve_model_deferral_inputs", - ResolveModelFullRefresh => "resolve_model_full_refresh", - ResolveModelHookEntries => "resolve_model_hook_entries", - ResolveModelPlanAction => "resolve_model_plan_action", ResolveModelSql => "resolve_model_sql", - ResolvePlannerScopes => "resolve_planner_scopes", ResolveProjectConnectionConfig => "resolve_project_connection_config", ResolveProjectSelectorTokens => "resolve_project_selector_tokens", ResolveProjectSelectors => "resolve_project_selectors", - ResolvePythonCheckExcludes => "resolve_python_check_excludes", ResolveRawDbtProfileConnection => "resolve_raw_dbt_profile_connection", ResolveRefReferences => "resolve_ref_references", ResolveRuleIgnoreSelectors => "resolve_rule_ignore_selectors", - ResolveSchemaActions => "resolve_schema_actions", ResolveSkippedViewChain => "resolve_skipped_view_chain", ResolveSourceReferences => "resolve_source_references", ResolveSqlbuildExecutionPlanOutput => "resolve_sqlbuild_execution_plan_output", ResolveSqlbuildModelSelectorNames => "resolve_sqlbuild_model_selector_names", - ResolveStartCursorConfig => "resolve_start_cursor_config", - ResolveStaticCommandScope => "resolve_static_command_scope", ResolveStaticRelationContext => "resolve_static_relation_context", - ResolveTableFunctionReferences => "resolve_table_function_references", ResolveTargetConfig => "resolve_target_config", ResolveTargetConnectionConfig => "resolve_target_connection_config", ResolveTargetName => "resolve_target_name", - ResolveUdfReferences => "resolve_udf_references", RouteDbtInteropArgs => "route_dbt_interop_args", RuleSelected => "rule_selected", RulesBulkCachePath => "rules_bulk_cache_path", @@ -1112,7 +971,6 @@ native_ids! { SchemaValidations => "schema_validations", ScopeMetadataProjection => "scope_metadata_projection", SelectCatalogueRules => "select_catalogue_rules", - SelectLatestFingerprints => "select_latest_fingerprints", SelectProjectGraph => "select_project_graph", SelectedRuleCodes => "selected_rule_codes", SessionOutcomeAnalysis => "session_outcome_analysis", @@ -1121,46 +979,31 @@ native_ids! { SourceBindingDiagnostics => "source_binding_diagnostics", SourceFreshnessInputError => "SourceFreshnessInputError", SourceFreshnessObservationError => "SourceFreshnessObservationError", - SplitProjectUnitTestSelectors => "split_project_unit_test_selectors", SplitSelectorExpansion => "split_selector_expansion", SplitUnitTestSelectors => "split_unit_test_selectors", - SqlCodeScanner => "SqlCodeScanner", - SqlRefKey => "sql_ref_key", SqlTestModelChainNamesByKey => "sql_test_model_chain_names_by_key", SqlTestScanCache => "SqlTestScanCache", SqlValueValidationError => "SqlValueValidationError", - StandalonePivotModels => "standalone_pivot_models", StaticSqlTestPlanningDiagnostics => "static_sql_test_planning_diagnostics", StoredOldNameViewColumns => "stored_old_name_view_columns", TerminalEventIndex => "TerminalEventIndex", TerminalEventIndexContext => "_terminal_event_index_context", TerminalEventIndexScopeFunction => "terminal_event_index_scope", - TopologicallyOrderKeys => "topologically_order_keys", TransientStatusReporter => "TransientStatusReporter", - TransitiveClosureMany => "transitive_closure_many", TypeCursorOverrides => "type_cursor_overrides", - UnreadableCursorInputsError => "unreadable_cursor_inputs_error", UpdateSqlbuildSkills => "update_sqlbuild_skills", - UpstreamLoaderDependencyNames => "upstream_loader_dependency_names", ValidateAuthoredSqlTestCtes => "validate_authored_sql_test_ctes", ValidateCappedWatermarkInputs => "validate_capped_watermark_inputs", ValidateCliWarehouse => "validate_cli_warehouse", ValidateCompileDbtModelNames => "validate_compile_dbt_model_names", ValidateCompileDbtModelReference => "validate_compile_dbt_model_reference", ValidateDbtInitProject => "validate_dbt_init_project", - ValidateHookSqlSyntax => "validate_hook_sql_syntax", - ValidateLocalSqlAnalysis => "validate_local_sql_analysis", ValidatePreviewTarget => "validate_preview_target", - ValidateResourceIdentity => "validate_resource_identity", ValidateSourceCursorInputColumns => "validate_source_cursor_input_columns", - ValidateSqlSyntax => "validate_sql_syntax", WithBindingDiagnostics => "with_binding_diagnostics", - WithDeclaredSourceReads => "with_declared_source_reads", WithHonestModelWriteHashes => "with_honest_model_write_hashes", WithMigrationFingerprints => "with_migration_fingerprints", WithTypeRecovery => "with_type_recovery", - WithoutDeclaredColumnsHash => "without_declared_columns_hash", - WithoutDestinationCursor => "without_destination_cursor", WriteAtomically => "write_atomically", WriteCheckResults => "write_check_results", WriteCompileTarget => "write_compile_target", @@ -1169,11 +1012,9 @@ native_ids! { WriteDbtExecutionSummary => "write_dbt_execution_summary", WriteExecutionHeader => "write_execution_header", WriteExecutionJsonOutput => "write_execution_json_output", - WriteReusableCompile => "write_reusable_compile", WriteRuntimeTarget => "write_runtime_target", WriteSourceFreshnessRecords => "write_source_freshness_records", WriteSqlbuildSkipNotice => "write_sqlbuild_skip_notice", - YamlEntryLine => "yaml_entry_line", } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs index 9e2e4c3b05..66e2939713 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs @@ -14,7 +14,6 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const DATACLASSES_MODULE: &str = "dataclasses"; const MODEL_RESOURCE: &str = "model"; const SOURCE_RESOURCE: &str = "source"; @@ -118,9 +117,7 @@ fn column_infos<'py>( } fn resource_type<'py>(py: Python<'py>, value: &str) -> PyResult> { - py.import(COMPILE_TYPES_MODULE)? - .getattr("CompiledResourceType")? - .call1((value,)) + registry::class(py, RecordId::CompiledResourceType)?.call1((value,)) } /// The evidence a model's authored contract offers, before inspection. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 8e58c7ded7..5111817fb5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -228,7 +228,7 @@ pub(crate) fn macro_output(request: ExpansionRequest<'_>) -> PyResult(request: ExpansionRequest<'py>) -> PyResult> { +pub(crate) fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult> { let py: Python<'py> = request.sql.py(); let sql: Bound<'py, PyString> = request.sql.clone(); let Some(output) = macro_output(request)? else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs index abb88195cf..c0cabbca1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs @@ -115,7 +115,7 @@ fn write(stream: &Bound<'_, PyAny>, line: &str) -> PyResult<()> { impl ExecutionEventWriter { #[new] #[pyo3(signature = (*, path = None))] - fn new(py: Python<'_>, path: Option>) -> PyResult { + pub(crate) fn new(py: Python<'_>, path: Option>) -> PyResult { let configured: Bound<'_, PyAny> = py .import("os")? .getattr("environ")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index f648ff9397..58fccc2ad9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -9,12 +9,14 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ ExpansionSources, SqlExpansionInputs, present, sql_expansion_inputs, }; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; const COMPILE_FIRST_MESSAGE: &str = "compiler-integrated Rules check the SQL your project \ @@ -76,13 +78,16 @@ fn expansion_inputs(sources: LintExpansionSources<'_>) -> PyResult discovered, None => { let no: Bound<'_, PyAny> = PyBool::new(py, false).to_owned().into_any(); - call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("sql_analysis_enabled_override", no.clone()), - ("extract_output_column_locations", no), - ], + discover_project_inputs( + py, + Some(keywords( + py, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", no.clone()), + ("extract_output_column_locations", no), + ], + )?), )? } }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 8c71f4a136..126272ba37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -33,7 +33,7 @@ pub(crate) fn plan_rule_run<'py>( let py: Python<'py> = options.py(); let graph: Bound<'py, PyAny> = options.get("graph")?; let project: Bound<'py, PyAny> = field(&graph, "project")?; - freed_memory_released(py); + freed_memory_released(py)?; let effective_config: Bound<'py, PyAny> = resolved_ignore_selectors(&options.get("config")?, &project)?; let project_dir: Bound<'py, PyAny> = options.get("project_dir")?.call_method0("resolve")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index ab494cc30c..3b3961fc8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -3,6 +3,7 @@ use std::collections::BTreeSet; use std::path::PathBuf; +use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; @@ -145,16 +146,19 @@ fn rules_bulk_cache_path(project_dir: PathBuf, file_name: &str) -> PathBuf { } /// Hand freed glibc heap pages back to the OS; a no-op elsewhere or without `malloc_trim`. -pub(crate) fn freed_memory_released(py: Python<'_>) { +pub(crate) fn freed_memory_released(py: Python<'_>) -> PyResult<()> { if !cfg!(target_os = "linux") { - return; + return Ok(()); } let trim = || -> PyResult<()> { let process: Bound<'_, PyAny> = imported(py, CTYPES_MODULE, "CDLL")?.call1((py.None(),))?; let _ = native_field(&process, "malloc_trim")?.call1((0,))?; Ok(()) }; - let _ = trim(); + match trim() { + Err(error) if !error.is_instance_of::(py) => Err(error), + _ => Ok(()), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs index bfc075e488..3f5f1bb1fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -21,6 +21,8 @@ use sqlbuild_core::scenario_command::models::{ }; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::discovery::guidance::sql_analysis_guidance; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -114,9 +116,9 @@ pub(crate) fn discover_project<'py>( directory if !directory.is_none() => directory, _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[("project_dir", project_dir.clone())], + let discovered: Bound<'py, PyAny> = discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), )?; let selectors: Bound<'py, PyAny> = native_field(request, "selectors")?; let exclude: Bound<'py, PyAny> = native_field(request, "exclude")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 271d0ccd10..7ea3883b93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{ use pyo3::types::{PyBool, PyCFunction, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; @@ -20,7 +21,7 @@ use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::scopes::scope_cache::{ project_fingerprint, read_scope_index_cache, write_scope_index_cache, @@ -57,8 +58,7 @@ fn project_adapter<'py>( let discover_keywords: Bound<'py, PyDict> = PyDict::new(py); discover_keywords.set_item("project_dir", project_dir)?; discover_keywords.set_item("extract_output_column_locations", false)?; - let discovered: Bound<'py, PyAny> = registry::native(py, NativeAttr::DiscoverProjectInputs)? - .call((), Some(&discover_keywords))?; + let discovered: Bound<'py, PyAny> = discover_project_inputs(py, Some(discover_keywords))?; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( &native_field(&discovered, "project_config")?, &native_field(&discovered, "local_config")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs index b944fb5140..b89ec1db02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -29,17 +29,41 @@ fn complete_semantic_project<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("complete_semantic_project", request)?; - let project: Bound<'py, PyAny> = request.get("project")?; - let profile: Bound<'py, PyAny> = request.get("profile")?; - let binding_results: Bound<'py, PyAny> = request.get("binding_results")?; - let resource_sql_analysis: bool = request.get("resource_sql_analysis")?.extract()?; - let session: Option> = - match request.optional("session")? { - Some(session) if !session.is_none() => Some(session.extract()?), - _ => None, - }; + completed_semantic_project(SemanticCompletion { + project: request.get("project")?, + profile: request.get("profile")?, + binding_results: request.get("binding_results")?, + resource_sql_analysis: request.get("resource_sql_analysis")?.extract()?, + session: request.optional("session")?, + }) +} + +/// The semantic completion stage's inputs. +pub(crate) struct SemanticCompletion<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) profile: Bound<'py, PyAny>, + pub(crate) binding_results: Bound<'py, PyAny>, + pub(crate) resource_sql_analysis: bool, + pub(crate) session: Option>, +} + +/// The completed project; an internal failure raises `NativeCompilerError`. +pub(crate) fn completed_semantic_project<'py>( + stage: SemanticCompletion<'py>, +) -> PyResult> { + let SemanticCompletion { + project, + profile, + binding_results, + resource_sql_analysis, + session, + } = stage; + let session: Option> = match session { + Some(session) if !session.is_none() => Some(session.extract()?), + _ => None, + }; let catalog: PyRef<'py, ProjectCatalog> = project_catalog(&project)?; - let classes = StageClasses::new(request.py())?; + let classes = StageClasses::new(project.py())?; let shared = StageShared { catalog: &catalog.inner, session: session.as_deref(), @@ -59,7 +83,7 @@ fn complete_semantic_project<'py>( &recovered, &[( "diagnostics", - &PyTuple::new(request.py(), diagnostics)?.into_any(), + &PyTuple::new(project.py(), diagnostics)?.into_any(), )], )?; completed_diagnostics(&classes, &combined, &shared) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index 8396178b1b..4d3645ec91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -41,7 +41,7 @@ impl<'py> StageClasses<'py> { pub(crate) fn new(py: Python<'py>) -> PyResult { let types = py.import(COMPILE_TYPES_MODULE)?; let severity = registry::class(types.py(), RecordId::DiagnosticSeverity)?; - let resource = native_field(&types, "CompiledResourceType")?; + let resource = registry::class(py, RecordId::CompiledResourceType)?; let kinds = registry::class(py, RecordId::SqlReferenceKind)?; Ok(Self { diagnostic: registry::class(py, RecordId::CompilerDiagnostic)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs index ba956e8688..5d8cccd3ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/declarations.rs @@ -2,12 +2,8 @@ use std::collections::BTreeSet; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; - -use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::records::fields::field as native_field; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; const UNKNOWN: &str = "UNKNOWN"; @@ -50,26 +46,3 @@ pub(crate) fn declared_types( types.remove(UNKNOWN); Ok(types.into_iter().collect()) } - -/// `known_function_names` as a sorted tuple. -#[pyfunction] -fn known_function_names<'py>(functions: &Bound<'py, PyAny>) -> PyResult> { - PyTuple::new(functions.py(), function_names(functions)?) -} - -/// `known_declared_types(*, functions, column_types)` as a sorted tuple. -#[pyfunction] -#[pyo3(signature = (**request))] -fn known_declared_types<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("known_declared_types", request)?; - let types: Vec = - declared_types(&request.get("functions")?, &request.get("column_types")?)?; - PyTuple::new(request.py(), types) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(known_function_names, module)?)?; - module.add_function(wrap_pyfunction!(known_declared_types, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs index 8d3deb3d24..5116b8f3a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/registration.rs @@ -2,11 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_shapes::{coverage, declarations, shapes, source_bindings}; +use crate::bindings::_helpers::semantic_shapes::{coverage, shapes, source_bindings}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { shapes::register(module)?; coverage::register(module)?; - source_bindings::register(module)?; - declarations::register(module) + source_bindings::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs index d1c3d30b2c..fe32823a87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs @@ -22,8 +22,6 @@ use crate::bindings::_helpers::semantic_shapes::shapes::{ model_inputs, project_shapes, published_shape, }; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - const BINDING_UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; const DATACLASSES_MODULE: &str = "dataclasses"; const GRAPHLIB_MODULE: &str = "graphlib"; @@ -303,11 +301,11 @@ fn validated_diagnostics<'py>( expansions.set_item(native_field(&path, "stem")?, expansion)?; } let skipped: Object<'py> = PyString::new(py, BINDING_UNKNOWN_TABLE_INTERNAL_CODE).into_any(); - let types = py.import(COMPILE_TYPES_MODULE)?; let diagnostic_class: Object<'py> = registry::class(py, RecordId::CompilerDiagnostic)?; let severity_class: Object<'py> = registry::class(py, RecordId::DiagnosticSeverity)?; let phase: Object<'py> = registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?; - let model_type: Object<'py> = native_field(&types, "CompiledResourceType")?.getattr("MODEL")?; + let model_type: Object<'py> = + registry::class(py, RecordId::CompiledResourceType)?.getattr("MODEL")?; let mut diagnostics: Vec> = Vec::new(); for ((model, request), result) in models.iter().zip(&requests).zip(results.try_iter()?) { let result: Object<'py> = result?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index bac9ffcbf3..c7cb9d5ec8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -12,7 +12,6 @@ use crate::bindings::_helpers::sql_test_glue::assembly::{ AssemblyRequest, AssemblyRow, DiagnosticRow, FactsRow, assemble_compiled_sql_tests, }; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// Fields a direct-logic compiled payload copies from its input payload, in order. const DIRECT_PAYLOAD_FIELDS: [&str; 5] = [ "mode", @@ -61,10 +60,9 @@ struct Classes<'py> { impl<'py> Classes<'py> { fn new(py: Python<'py>) -> PyResult { - let types: Bound<'py, PyModule> = py.import(COMPILE_TYPES_MODULE)?; Ok(Self { direct_input: registry::class(py, RecordId::CompileDirectLogicSqlTestInputPayload)?, - resource_type: native_field(&types, "CompiledResourceType")?, + resource_type: registry::class(py, RecordId::CompiledResourceType)?, phase: registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?, severity: registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?, assembly: registry::class(py, RecordId::NativeSqlTestAssembly)?, @@ -293,7 +291,7 @@ fn assembly<'py>( /// One assembly per test input, with the error its assembly raises, if any. #[pyfunction] #[pyo3(signature = (*, inputs))] -fn assemble_native_sql_tests<'py>( +pub(crate) fn assemble_native_sql_tests<'py>( py: Python<'py>, inputs: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs index f23145f868..019c313c3b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs @@ -3,7 +3,6 @@ pub(super) const UDF: &str = "udf"; pub(super) const TABLE_FN: &str = "table_fn"; pub(super) const ERROR_SEVERITY: &str = "error"; -pub(super) const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; pub(super) const FIXTURE_PREFIXES: [(&str, &str); 4] = [ ("model", "__ref__"), ("source", "__source__"), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs index ac00c76f44..01856fb93e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/fixture_validation.rs @@ -9,7 +9,6 @@ use sqlbuild_analysis::planner_fixtures::main::{ }; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; use crate::bindings::_helpers::planner_strategy::fixture_completion::complete_fixtures; use crate::bindings::_helpers::planner_strategy::fixture_sql::{ @@ -18,7 +17,9 @@ use crate::bindings::_helpers::planner_strategy::fixture_sql::{ use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::planner_strategy::types::FixtureKey; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::sql_test_glue::constants::{COMPILE_TYPES, FIXTURE_PREFIXES}; +use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::constants::FIXTURE_PREFIXES; /// Report every statically provable fixture defect or return the completed fixture SQL groups. pub(super) fn validated_fixtures<'py>( @@ -30,7 +31,7 @@ pub(super) fn validated_fixtures<'py>( let py = test.py(); let (chain, [refs, sources, seeds, expected]) = request; let adapter = inputs.1; - let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; + let resource_types = registry::class(py, RecordId::CompiledResourceType)?; let model = native_field(&resource_types, "MODEL")?; let kinds = [ model.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index f36ea6e975..2ff1cb2cb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1,6 +1,6 @@ pub(crate) mod assembly; pub(crate) mod authored_ctes; -mod compiled_tests; +pub(crate) mod compiled_tests; pub(crate) mod constants; mod cursor_window; mod fixture_validation; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index a7b8394942..38ee42ab5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -7,13 +7,15 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; -use crate::bindings::_helpers::sql_test_glue::planning::resolve_compiled_sql_test_chains; +use crate::bindings::_helpers::sql_test_glue::planning::{ + ChainRequest, PlanningRequest, plan_compiled_sql_tests, resolve_compiled_sql_test_chains, +}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; -use pyo3::types::{PyBool, PyDict, PyTuple}; +use pyo3::types::{PyDict, PyTuple}; pub(super) fn chains<'py>( project: &Bound<'py, PyAny>, @@ -23,23 +25,16 @@ pub(super) fn chains<'py>( if !tests.is_truthy()? { return Ok(PyTuple::empty(py).into_any()); } - let request = call_with( - ®istry::class(py, RecordId::NativeSqlTestChainRequest)?, - &[ - ("models", native_field(project, "models")?), - ("tests", tests.clone()), - ( - "lexical_syntax", - project - .getattr("sql_lexical_syntax")? - .getattr("native_mapping")?, - ), - ], - )?; - let native = request - .extract() - .and_then(|request| resolve_compiled_sql_test_chains(py, request)) - .and_then(|chains| object(py, chains)); + let request = ChainRequest { + models: native_field(project, "models")?.extract()?, + tests: tests.extract()?, + lexical_syntax: project + .getattr("sql_lexical_syntax")? + .getattr("native_mapping")? + .extract()?, + }; + let native = + resolve_compiled_sql_test_chains(py, request).and_then(|chains| object(py, chains)); let native = match native { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { @@ -80,57 +75,34 @@ pub(crate) fn planned<'py>( let model_sql = super::plan_rendering::models(project, adapter)?; let functions = super::plan_rendering::functions(project, adapter)?; let overrides = super::plan_rendering::overrides(project, tests, adapter)?; - let request = call_with( - ®istry::class(py, RecordId::NativeSqlTestPlanningRequest)?, - &[ - ("models", native_field(project, "models")?), - ("rendered_model_sql", model_sql.into_any()), - ("functions", functions.into_any()), - ("tests", tests.clone()), - ("rendered_test_overrides", overrides.into_any()), - ("sql_analysis_enabled", options.sql_analysis_enabled.clone()), - ( - "sql_analysis_dialect", - adapter.call_method0("sql_analysis_dialect")?, - ), - ( - "set_difference_operator", - adapter.call_method0("render_set_difference_operator")?, - ), - ( - "requires_derived_table_aliases", - adapter.call_method0("requires_derived_table_aliases")?, - ), - ( - "lexical_syntax", - project - .getattr("sql_lexical_syntax")? - .getattr("native_mapping")?, - ), - ( - "render_sql", - PyBool::new(py, artifact || options.render_sql) - .to_owned() - .into_any(), - ), - ( - "include_plan", - PyBool::new(py, !artifact && options.include_plan) - .to_owned() - .into_any(), - ), - ], - )?; - let native = registry::native(py, NativeAttr::PlanCompiledSqlTests)?.call1((request,)); - let native = match native { + let request = PlanningRequest { + models: native_field(project, "models")?.extract()?, + rendered_model_sql: model_sql.extract()?, + functions: functions.extract()?, + tests: tests.extract()?, + rendered_test_overrides: overrides.extract()?, + sql_analysis_enabled: options.sql_analysis_enabled.extract()?, + sql_analysis_dialect: adapter.call_method0("sql_analysis_dialect")?.extract()?, + set_difference_operator: adapter + .call_method0("render_set_difference_operator")? + .extract()?, + requires_derived_table_aliases: adapter + .call_method0("requires_derived_table_aliases")? + .extract()?, + lexical_syntax: project + .getattr("sql_lexical_syntax")? + .getattr("native_mapping")? + .extract()?, + render_sql: artifact || options.render_sql, + include_plan: !artifact && options.include_plan, + }; + let (rows, planning_ns, rendering_ns) = match plan_compiled_sql_tests(py, request) { Ok(value) => value, Err(error) if error.is_instance_of::(py) => { return Err(super::plan_errors::planning_error(py, error, tests)?); } Err(error) => return Err(error), }; - let planning_ns: u64 = native.get_item(1)?.extract()?; - let rendering_ns: u64 = native.get_item(2)?.extract()?; let end: u64 = imported(py, "time", "perf_counter_ns")? .call0()? .extract()?; @@ -148,8 +120,8 @@ pub(crate) fn planned<'py>( add_compile_timing(phase, elapsed); } let mut plans: Vec> = Vec::new(); - for row in native.get_item(0)?.try_iter()? { - let plan = super::plan_rows::plan(&row?)?; + for row in rows { + let plan = super::plan_rows::plan(py, row)?; if artifact { let sql = native_field(&plan, "sql")?; if sql.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs index e2c7fb12fd..b59b1d4c69 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_entries.rs @@ -3,10 +3,8 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_strategy::constants::MODEL_REFERENCE_KIND; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; -use crate::bindings::_helpers::sql_test_glue::constants::COMPILE_TYPES; use crate::bindings::_helpers::sql_test_glue::constants::{TABLE_FN, UDF}; use crate::bindings::_helpers::sql_test_glue::plan_values::model_payload; use sqlbuild_analysis::compiler::main::missing_expected_columns::missing_expected_columns; @@ -149,7 +147,7 @@ pub(super) fn expected_diagnostics( return Ok(result); }; let py = test.py(); - let resource_types = imported(py, COMPILE_TYPES, "CompiledResourceType")?; + let resource_types = registry::class(py, RecordId::CompiledResourceType)?; for step in native_field(plan, "chain")?.try_iter()? { let step = step?; let name = native_field(&step, "model_name")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs index 9976e742c1..a4791201f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs @@ -3,70 +3,68 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use crate::bindings::_helpers::sql_test_glue::planning::{AssertionRow, ChainRow, PlanRow}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyTuple; -fn step<'py>(row: &Bound<'py, PyAny>, chain: bool) -> PyResult> { - let py = row.py(); - let kwargs = PyDict::new(py); - let fields: &[&str] = if chain { +fn chain_step<'py>(py: Python<'py>, row: ChainRow) -> PyResult> { + let (model, sql, expected_sql, lifted, body, expected_columns, expected_lifted) = row; + call_with( + ®istry::class(py, RecordId::ChainStep)?, &[ - "model_name", - "resolved_sql", - "expected_cte_sql", - "lifted_ctes", - "comparison_body_sql", - "expected_columns", - "expected_lifted_ctes", - ] - } else { - &["name", "resolved_sql", "lifted_ctes", "comparison_body_sql"] - }; - for (index, name) in fields.iter().enumerate() { - let value = row.get_item(index)?; - kwargs.set_item( - *name, - if ["lifted_ctes", "expected_columns", "expected_lifted_ctes"].contains(name) - && !value.is_none() - { - tuple(&value)? - } else { - value - }, - )?; - } - registry::class( - py, - if chain { - RecordId::ChainStep - } else { - RecordId::SqlTestAssertionStep - }, - )? - .call((), Some(&kwargs)) + ("model_name", model.into_bound_py_any(py)?), + ("resolved_sql", sql.into_bound_py_any(py)?), + ("expected_cte_sql", expected_sql.into_bound_py_any(py)?), + ("lifted_ctes", PyTuple::new(py, lifted)?.into_any()), + ("comparison_body_sql", body.into_bound_py_any(py)?), + ( + "expected_columns", + match expected_columns { + Some(columns) => PyTuple::new(py, columns)?.into_any(), + None => py.None().into_bound(py), + }, + ), + ( + "expected_lifted_ctes", + PyTuple::new(py, expected_lifted)?.into_any(), + ), + ], + ) +} + +fn assertion_step<'py>(py: Python<'py>, row: AssertionRow) -> PyResult> { + let (name, sql, lifted, body) = row; + call_with( + ®istry::class(py, RecordId::SqlTestAssertionStep)?, + &[ + ("name", name.into_bound_py_any(py)?), + ("resolved_sql", sql.into_bound_py_any(py)?), + ("lifted_ctes", PyTuple::new(py, lifted)?.into_any()), + ("comparison_body_sql", body.into_bound_py_any(py)?), + ], + ) } -pub(super) fn plan<'py>(row: &Bound<'py, PyAny>) -> PyResult> { - let py = row.py(); - let mut chain: Vec> = Vec::new(); - for value in row.get_item(1)?.try_iter()? { - chain.push(step(&value?, true)?); +pub(super) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { + let (sql, chain_rows, assertion_rows, model_names, warning_rows, error_messages) = row; + let mut chain: Vec> = Vec::new(); + for value in chain_rows { + chain.push(chain_step(py, value)?); } - let mut assertions: Vec> = Vec::new(); - for value in row.get_item(2)?.try_iter()? { - assertions.push(step(&value?, false)?); + let mut assertions: Vec> = Vec::new(); + for value in assertion_rows { + assertions.push(assertion_step(py, value)?); } - let mut warnings: Vec> = Vec::new(); + let mut warnings: Vec> = Vec::new(); let severity = registry::class(py, RecordId::WarningSeverity)?; - for value in row.get_item(4)?.try_iter()? { - let value = value?; + for (model_name, level, message) in warning_rows { warnings.push(call_with( ®istry::class(py, RecordId::PlanWarning)?, &[ - ("model_name", value.get_item(0)?), - ("severity", severity.call1((value.get_item(1)?,))?), - ("message", value.get_item(2)?), + ("model_name", model_name.into_bound_py_any(py)?), + ("severity", severity.call1((level,))?), + ("message", message.into_bound_py_any(py)?), ], )?); } @@ -75,10 +73,13 @@ pub(super) fn plan<'py>(row: &Bound<'py, PyAny>) -> PyResult> &[ ("chain", PyTuple::new(py, chain)?.into_any()), ("assertions", PyTuple::new(py, assertions)?.into_any()), - ("model_names", tuple(&row.get_item(3)?)?), + ("model_names", PyTuple::new(py, model_names)?.into_any()), ("warnings", PyTuple::new(py, warnings)?.into_any()), - ("error_messages", tuple(&row.get_item(5)?)?), - ("sql", row.get_item(0)?), + ( + "error_messages", + PyTuple::new(py, error_messages)?.into_any(), + ), + ("sql", sql.into_bound_py_any(py)?), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index 39a41e690e..7dd187e01d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -2,8 +2,7 @@ use std::collections::{BTreeMap, HashMap}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_analysis::compiler::main::sql_test_chains::resolve_sql_test_chains; use sqlbuild_analysis::compiler::main::sql_test_glue::plan_sql_test_batch; use sqlbuild_analysis::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; @@ -22,9 +21,9 @@ use crate::bindings::types::CompilerDetach; const MODEL_RESOURCE_TYPE: &str = "model"; const PLANNING_WORKERS: usize = 4; -type CtePair = (String, String); +pub(crate) type CtePair = (String, String); /// `(model, SQL, expected SQL, lifted CTEs, body, expected columns, expected lifted CTEs)`. -type ChainRow = ( +pub(crate) type ChainRow = ( String, String, Option, @@ -34,11 +33,11 @@ type ChainRow = ( Vec, ); /// `(name, resolved SQL, lifted CTEs, comparison body)`. -type AssertionRow = (String, String, Vec, Option); +pub(crate) type AssertionRow = (String, String, Vec, Option); /// `(model, severity, message)`. -type WarningRow = (Option, &'static str, String); +pub(crate) type WarningRow = (Option, &'static str, String); /// `(SQL, chain, assertions, model names, warnings, distinct error messages)`. -type PlanRow = ( +pub(crate) type PlanRow = ( Option, Vec, Vec, @@ -47,50 +46,32 @@ type PlanRow = ( Vec, ); -/// The Python planning request: compiled models and tests, with the adapter's facts. -#[derive(FromPyObject)] +/// The planning request: compiled models and tests, with the adapter's facts. pub(crate) struct PlanningRequest<'py> { - #[pyo3(attribute)] - models: Vec>, + pub(crate) models: Vec>, /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. - #[pyo3(attribute)] - rendered_model_sql: HashMap, - #[pyo3(attribute)] - functions: Vec<(String, String, String, String, String)>, - #[pyo3(attribute)] - tests: Vec>, + pub(crate) rendered_model_sql: HashMap, + pub(crate) functions: Vec<(String, String, String, String, String)>, + pub(crate) tests: Vec>, /// Cursor-rendered model query overrides by test index, replacing the test's own. - #[pyo3(attribute)] - rendered_test_overrides: HashMap>, - #[pyo3(attribute)] - sql_analysis_enabled: bool, - #[pyo3(attribute)] - sql_analysis_dialect: Option, - #[pyo3(attribute)] - set_difference_operator: String, - #[pyo3(attribute)] - requires_derived_table_aliases: bool, - #[pyo3(attribute)] - lexical_syntax: LexicalSyntaxInput, - #[pyo3(attribute)] - render_sql: bool, - #[pyo3(attribute)] - include_plan: bool, + pub(crate) rendered_test_overrides: HashMap>, + pub(crate) sql_analysis_enabled: bool, + pub(crate) sql_analysis_dialect: Option, + pub(crate) set_difference_operator: String, + pub(crate) requires_derived_table_aliases: bool, + pub(crate) lexical_syntax: LexicalSyntaxInput, + pub(crate) render_sql: bool, + pub(crate) include_plan: bool, } -/// The Python chain request: compiled models and the tests whose chains to resolve. -#[derive(FromPyObject)] +/// The chain request: compiled models and the tests whose chains to resolve. pub(crate) struct ChainRequest<'py> { - #[pyo3(attribute)] - models: Vec>, - #[pyo3(attribute)] - tests: Vec>, - #[pyo3(attribute)] - lexical_syntax: LexicalSyntaxInput, + pub(crate) models: Vec>, + pub(crate) tests: Vec>, + pub(crate) lexical_syntax: LexicalSyntaxInput, } /// Each test's ordered unmocked model chain; direct-logic tests have none. -#[pyfunction] pub(crate) fn resolve_compiled_sql_test_chains( py: Python<'_>, request: ChainRequest<'_>, @@ -118,7 +99,6 @@ pub(crate) fn resolve_compiled_sql_test_chains( } /// Plan SQL tests in one batch and return each plan with its distinct error messages. -#[pyfunction] pub(crate) fn plan_compiled_sql_tests( py: Python<'_>, request: PlanningRequest<'_>, @@ -298,9 +278,3 @@ fn assertion_row(step: SqlTestAssertionStep) -> AssertionRow { step.comparison_body_sql, ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_compiled_sql_tests, module)?)?; - module.add_function(wrap_pyfunction!(resolve_compiled_sql_test_chains, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 2e414fa4ec..9b9f6f8a0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; -use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests, planning}; +use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( @@ -23,6 +23,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; authored_ctes::register(module)?; - planning::register(module)?; compiled_tests::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index 2f7cec3bcf..c66b211658 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -7,14 +7,17 @@ use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::sqltext::authored_values::optional_authored_value_to_python; use crate::bindings::types::CompilerDetach; -type ParsedModelHeader = ( +pub(crate) type ParsedModelHeader = ( Option>, Option>, Option, ); #[pyfunction] -fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult> { +pub(crate) fn parse_model_headers( + py: Python<'_>, + headers: Vec, +) -> PyResult> { let parsed = py .compiler_detach(|| { sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch(&headers) @@ -72,7 +75,10 @@ pub(crate) fn location_rows( /// Simple top-level SELECT output names and their authored locations. #[pyfunction] -fn model_output_columns(contents: &str, implicit_aliases: bool) -> PyResult> { +pub(crate) fn model_output_columns( + contents: &str, + implicit_aliases: bool, +) -> PyResult> { compiler_guard(|| { Ok(location_rows( sqlbuild_sqltext::compiler::main::model_output_columns::model_output_columns( diff --git a/python-boundary.toml b/python-boundary.toml index 84f3282619..5857186522 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -745,7 +745,7 @@ note = "Shape rule: data exceptions only (SqlValueRenderingError), no logic; con [[stays]] path = "compiler/compile/types.py" rule = 1 -note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, CompiledResourceType, AuditInput, ...), no logic; consumed by executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." +note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectView, CompiledModelView, AuditInput, ...), no logic; consumed by executor/pipeline/_helpers/testing.py, executor/scenario/_helpers/local/execution.py and project audit factories that annotate AuditInput." [[stays]] path = "rule_engine/types.py" diff --git a/scripts/cold_compile_performance/_helpers/varied_statistics.py b/scripts/cold_compile_performance/_helpers/varied_statistics.py index 2068451e3d..7bb09e1fb2 100644 --- a/scripts/cold_compile_performance/_helpers/varied_statistics.py +++ b/scripts/cold_compile_performance/_helpers/varied_statistics.py @@ -8,7 +8,7 @@ VARIED_WINDOW_NODE_FRAGMENT, ) from scripts.cold_compile_performance.models import VariedProjectStatistics -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledResourceType from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql _UDF_REFERENCE: re.Pattern[str] = re.compile(r'__udf\("([a-zA-Z0-9_]+)"\)') diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a5be1ba378..8cbcaa2e99 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -53,7 +53,6 @@ from sqlbuild.compiler.compile.types import ( CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, ExternalSqlReferenceResolver, @@ -311,31 +310,6 @@ type ProofRow = tuple[ type QueryColumnsMode = Literal["batch", "reanalysis", "parse", "legacy"] -type CtePairRow = tuple[str, str] - -type NativeChainStepRow = tuple[ - str, - str, - str | None, - list[CtePairRow], - str | None, - list[str] | None, - list[CtePairRow], -] - -type NativeAssertionStepRow = tuple[str, str, list[CtePairRow], str | None] - -type NativePlanWarningRow = tuple[str | None, str, str] - -type NativeSqlTestPlanRow = tuple[ - str | None, - list[NativeChainStepRow], - list[NativeAssertionStepRow], - list[str], - list[NativePlanWarningRow], - list[str], -] - type SqlTestAssemblyFailureKind = Literal["input", "internal", "decimal_overflow"] type ProjectProvider = Provider @@ -3931,6 +3905,9 @@ def simplify_fixture_null_projections(*, sql: str, relation: Any) -> str: ... def is_empty_fixture_query(sql: str) -> bool: ... def fixture_read_cache_info(*, reset: bool = False) -> tuple[int, int]: ... def fixture_planning_work_info(*, reset: bool = False) -> tuple[int, int]: ... +def planner_identity_work_info(*, reset: bool = False) -> tuple[int, dict[str, int]]: ... +def compile_stage_work_info(*, reset: bool = False) -> dict[str, int]: ... +def migration_fingerprint_work_info(*, reset: bool = False) -> tuple[int, int]: ... def build_relation_fixture_context( *, project: CompiledProjectView, models_by_name: dict[str, CompiledModelView] | None = None ) -> RelationFixturePlanningContext: ... @@ -4104,6 +4081,16 @@ def assemble_project( analysis_cache_dir: Path | None = None, analysis_model_names: frozenset[str] | None = None, ) -> CompiledProjectView: ... +def assembly_stage_inputs( + stage: Literal["expression_shapes", "model_facts", "semantic_completion"], + *, + inputs: CompileInputsView, + inference_profile: ExpressionInferenceProfile | None = None, + skip_column_inference: bool = False, + column_lineage_mode: ColumnLineageMode = ..., + analysis_cache_dir: Path | None = None, + analysis_model_names: frozenset[str] | None = None, +) -> dict[str, Any] | None: ... def build_project_graph_with_analysis_selection( *, discovered_inputs: DiscoveredProjectInputs, @@ -4300,16 +4287,6 @@ def validate_sql_syntax( parsing: tuple[dict[str, str] | None, str | None] = (None, None), ) -> None: ... def hook_sql_statements(value: object) -> list[str]: ... -def declared_relations( - inputs: CompileInputsView, -) -> tuple[ - dict[str, dict[str, str]], - dict[str, dict[str, InferredNullability]], - dict[str, dict[str, str]], -]: ... -def externally_described_models( - inputs: DiscoveredProjectInputs, model_schemas: dict[str, ModelSchemaDeclaration] -) -> list[Path]: ... def analyze_resolved_column_reads( *, query_sql: str, references: Sequence[CompileSqlReference], dialect: str | None ) -> tuple[CompiledLineageSourceFact, ...]: ... @@ -5427,10 +5404,6 @@ def semantic_coverage(project: Any, /) -> dict[str, tuple[str, ...]]: ... def source_binding_diagnostics( *, project: Any, columns: Mapping[str, Any], profile: Any, selected_keys: frozenset[Any] ) -> tuple[Any, ...]: ... -def known_function_names(functions: Sequence[Any], /) -> tuple[str, ...]: ... -def known_declared_types( - *, functions: Sequence[Any], column_types: Mapping[str, Mapping[str, str]] -) -> tuple[str, ...]: ... def complete_semantic_project( *, project: Any, @@ -6020,10 +5993,6 @@ def render_janitor_completion( ) -> str: ... # Native analysis: SQL test planning glue. -def plan_compiled_sql_tests( - request: NativeSqlTestPlanningRequest, / -) -> tuple[list[NativeSqlTestPlanRow], int, int]: ... -def resolve_compiled_sql_test_chains(request: NativeSqlTestChainRequest, /) -> list[list[str]]: ... def assemble_native_sql_tests( *, inputs: CompileInputsView ) -> tuple[NativeSqlTestAssembly, ...]: ... @@ -10242,50 +10211,6 @@ class ManualLocation(NativeRecord): reason: str, ) -> None: ... -class NativeSqlTestPlanningRequest(NativeRecord): - """Compiled models and tests to plan, with the adapter facts native planning cannot derive.""" - - models: tuple[CompiledModelView, ...] - rendered_model_sql: dict[int, str] - functions: tuple[tuple[str, str, str, str, str], ...] - tests: tuple[CompiledSqlTest, ...] - rendered_test_overrides: dict[int, dict[str, str]] - sql_analysis_enabled: bool - sql_analysis_dialect: str | None - set_difference_operator: str - requires_derived_table_aliases: bool - lexical_syntax: dict[str, object] - render_sql: bool - include_plan: bool - def __init__( - self, - models: tuple[CompiledModelView, ...], - rendered_model_sql: dict[int, str], - functions: tuple[tuple[str, str, str, str, str], ...], - tests: tuple[CompiledSqlTest, ...], - rendered_test_overrides: dict[int, dict[str, str]], - sql_analysis_enabled: bool, - sql_analysis_dialect: str | None, - set_difference_operator: str, - requires_derived_table_aliases: bool, - lexical_syntax: dict[str, object], - render_sql: bool, - include_plan: bool, - ) -> None: ... - -class NativeSqlTestChainRequest(NativeRecord): - """Compiled models and the tests whose unmocked model chains to resolve.""" - - models: tuple[CompiledModelView, ...] - tests: tuple[CompiledSqlTest, ...] - lexical_syntax: dict[str, object] - def __init__( - self, - models: tuple[CompiledModelView, ...], - tests: tuple[CompiledSqlTest, ...], - lexical_syntax: dict[str, object], - ) -> None: ... - class NativeSqlTestFailure(NativeRecord): """The error one test's assembly raises: its kind and message.""" @@ -10555,6 +10480,17 @@ class DiagnosticSeverity(StrEnum): WARNING = "warning" INFO = "info" +class CompiledResourceType(StrEnum): + MODEL = "model" + SOURCE = "source" + SEED = "seed" + UDF = "udf" + TABLE_FN = "table_fn" + DBT_REF = "dbt_ref" + AUDIT = "audit" + SQL_TEST = "sql_test" + SQL_SCENARIO = "sql_scenario" + class InferredNullability(StrEnum): """Conservative nullability state inferred for an output column.""" @@ -16532,6 +16468,7 @@ class DiscoveredLoaderFunction(NativeRecord): description: str | None connection_mode: LoaderConnectionMode provider_usages: tuple[DiscoveredProviderUsage, ...] + identity: PythonNodeIdentity | None def __init__( self, file_path: Path, @@ -16548,6 +16485,7 @@ class DiscoveredLoaderFunction(NativeRecord): description: str | None = ..., connection_mode: LoaderConnectionMode = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + identity: PythonNodeIdentity | None = ..., ) -> None: ... class DiscoveredTaskFunction(NativeRecord): @@ -16564,6 +16502,7 @@ class DiscoveredTaskFunction(NativeRecord): meta: dict[str, object] | None retry: RetryPolicy | None provider_usages: tuple[DiscoveredProviderUsage, ...] + identity: PythonNodeIdentity | None def __init__( self, file_path: Path, @@ -16577,6 +16516,7 @@ class DiscoveredTaskFunction(NativeRecord): meta: dict[str, object] | None = ..., retry: RetryPolicy | None = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + identity: PythonNodeIdentity | None = ..., ) -> None: ... class DiscoveredAssetFunction(NativeRecord): @@ -16595,6 +16535,7 @@ class DiscoveredAssetFunction(NativeRecord): column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None retry: RetryPolicy | None provider_usages: tuple[DiscoveredProviderUsage, ...] + identity: PythonNodeIdentity | None def __init__( self, file_path: Path, @@ -16610,6 +16551,7 @@ class DiscoveredAssetFunction(NativeRecord): column_lineage: dict[str, tuple[ColumnLineageRef, ...]] | None = ..., retry: RetryPolicy | None = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + identity: PythonNodeIdentity | None = ..., ) -> None: ... class DiscoveredCheckFunction(NativeRecord): @@ -16626,6 +16568,7 @@ class DiscoveredCheckFunction(NativeRecord): description: str | None meta: dict[str, object] | None provider_usages: tuple[DiscoveredProviderUsage, ...] + identity: PythonNodeIdentity | None def __init__( self, file_path: Path, @@ -16639,6 +16582,7 @@ class DiscoveredCheckFunction(NativeRecord): description: str | None = ..., meta: dict[str, object] | None = ..., provider_usages: tuple[DiscoveredProviderUsage, ...] = ..., + identity: PythonNodeIdentity | None = ..., ) -> None: ... class DiscoveredHookFunction(NativeRecord): @@ -16879,17 +16823,20 @@ class DiscoveredRuntimeExtensions(NativeRecord): lifecycle_shutdown_timeout_seconds: int | None = ..., ) -> None: ... -class ProjectGraph(NativeRecord): +class ProjectGraph: """Static compiled project graph held natively; dict views are built only when read.""" - project: CompiledProjectView - native: NativeProjectGraph def __init__( self, + *, project: CompiledProjectView, native: NativeProjectGraph, ) -> None: ... @property + def project(self) -> CompiledProjectView: ... + @property + def native(self) -> NativeProjectGraph: ... + @property def lineage_views(self) -> LineageGraphViews: ... @property def upstream_deps(self) -> dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]]: ... diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 1464e00c2b..b7574a6e37 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -711,7 +711,6 @@ "AuditSeverity": ("sqlbuild.compiler.auditing.types", "AuditSeverity"), "ThresholdOperator": ("sqlbuild.compiler.auditing.types", "ThresholdOperator"), "AttachedAuditTargetKind": ("sqlbuild.compiler.compile.types", "AttachedAuditTargetKind"), - "CompiledResourceType": ("sqlbuild.compiler.compile.types", "CompiledResourceType"), "FunctionLanguage": ("sqlbuild.compiler.compile.types", "FunctionLanguage"), "SqlTestMode": ("sqlbuild.compiler.compile.types", "SqlTestMode"), "CursorAlgebraError": ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"), diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index d71a52e47d..1e53f628e7 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -103,7 +103,6 @@ from sqlbuild.compiler.compile._helpers.render.user_macros import ( macro_invocation_context as macro_invocation_context, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind as AttachedAuditTargetKind -from sqlbuild.compiler.compile.types import CompiledResourceType as CompiledResourceType from sqlbuild.compiler.compile.types import FunctionLanguage as FunctionLanguage from sqlbuild.compiler.compile.types import SqlTestMode as SqlTestMode from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index f1eac540c4..f2f1b9d938 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -379,18 +379,6 @@ class AttachedAuditTargetKind(StrEnum): SEED = "seed" -class CompiledResourceType(StrEnum): - MODEL = "model" - SOURCE = "source" - SEED = "seed" - UDF = "udf" - TABLE_FN = "table_fn" - DBT_REF = "dbt_ref" - AUDIT = "audit" - SQL_TEST = "sql_test" - SQL_SCENARIO = "sql_scenario" - - class FunctionLanguage(StrEnum): SQL = "sql" PYTHON = "python" diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index 2548cd8384..b43fa5aa2d 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -4,10 +4,15 @@ from typing import Any -from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, PlanOutput, audit_resource_id +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + CompiledResourceType, + PlanOutput, + audit_resource_id, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.auditing.types import AuditOutcome, AuditRunScope -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, ) diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 2d8fad6bcb..8eb61acfeb 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -18,6 +18,7 @@ from sqlbuild._native import ( AuditPlanEntry, CompiledObjectKey, + CompiledResourceType, CostContext, DiscoveredLoaderFunction, FunctionPlanEntry, @@ -38,7 +39,6 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.cost.models import StatementExecutionTelemetry from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index f618ca5b23..a28023fb1d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -826,6 +826,7 @@ class NativeAnalysisSeamTestCase: description: str expected_native_returns: dict[str, list[object]] + expected_stage_work: dict[str, int] @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py index 1161eb5b16..930da556bc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fast_lineage/test_native_fast_lineage.py @@ -116,13 +116,6 @@ def _panicked(*arguments): native.build_project_column_lineage = _panicked """ -_RICH_COMPILE: tuple[str, ...] = ( - "compile", - "--json", - "--no-cache", - "--lineage-mode", - "rich", -) @pytest.mark.parametrize( @@ -247,13 +240,6 @@ def test_given_project_when_tracing_rich_lineage_then_it_is_traced( @pytest.mark.parametrize( "test_case", [ - NativeRichLineageFailureCliTestCase( - description="compile --lineage-mode rich", - files=_RICH_FILES, - command=_RICH_COMPILE, - expected_returncode=1, - expected_error_line=f"_native.NativeCompilerError: {_NATIVE_PANIC_ERROR}", - ), NativeRichLineageFailureCliTestCase( description="lineage --mode rich", files=_RICH_FILES, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index f02bbb8997..71b02a24a6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -17,7 +17,6 @@ import time import zipfile from bisect import bisect_left -from collections import Counter from collections.abc import Callable, Iterator from contextlib import contextmanager, redirect_stdout from dataclasses import replace @@ -44,7 +43,7 @@ from sqlbuild._native import ContractValidationResult, compiler_cache_directory from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter -from sqlbuild.adapter.contract.types import TypeDialect +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from sqlbuild.observability import EventDispatcher, LifecycleEvent @@ -3350,16 +3349,16 @@ def write_before_reuse_store( ) -> None: """Let another writer change target/ once, after a compile wrote it and before it stores.""" - original: Callable[..., None] = native_module.write_reusable_compile pending: list[Callable[[Path], None]] = [write] - def write_then_store(**kwargs: Any) -> None: - for action in tuple(pending): - action(project_dir) - pending.clear() - original(**kwargs) + class WrittenThenChanged(native_module.PreparedCompileArtifacts): + def __exit__(self, *exception: object) -> None: + super().__exit__(*exception) + for action in tuple(pending): + action(project_dir) + pending.clear() - monkeypatch.setattr(native_module, "write_reusable_compile", write_then_store) + monkeypatch.setattr(native_module, "PreparedCompileArtifacts", WrittenThenChanged) def rewrite_compiled_model_unchanged(project_dir: Path) -> None: @@ -4807,11 +4806,6 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( (native_module, "infer_native_expression_source_shapes"), - (native_module, "assemble_native_sql_tests"), - (native_module, "complete_semantic_project"), - (native_module, "evaluate_model_contracts"), - (native_module, "promotion_conflict_diagnostics"), - (native_module, "build_project_column_lineage"), (plan_artifacts_module, "plan_compiled_sql_test_artifacts"), ) @@ -4853,6 +4847,7 @@ def seam_recording_compile( }.items(): patch.setenv(name, value) seams: dict[str, list[object]] = record_analysis_seams(monkeypatch=patch) + _ = native_module.compile_stage_work_info(reset=True) run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) return run, seams @@ -4874,35 +4869,31 @@ def type_system_compile( """Compile in this process; return the run and what native types answered.""" normalized: list[bool] = [] - statuses: Counter[str] = Counter() normalize: Callable[[str, str], object] = native_module.normalize_type - evaluate_contracts: Callable[..., ContractValidationResult] = ( - native_module.evaluate_model_contracts - ) def recorded(type_sql: str, dialect: str) -> object: result: object = normalize(type_sql, dialect) normalized.append(result is not None) return result - def recorded_contracts( - *, project: CompiledProjectView, dialect: TypeDialect | str | None - ) -> ContractValidationResult: - result: ContractValidationResult = evaluate_contracts(project=project, dialect=dialect) - statuses.update(contract_statuses(project=project, result=result)) - return result - type_normalization.normalize_type.cache_clear() with monkeypatch.context() as patch: patch.setattr(native_module, "normalize_type", recorded) - patch.setattr(native_module, "evaluate_model_contracts", recorded_contracts) for name, value in {REUSE_DISABLE_ENV_VAR: "1"}.items(): patch.setenv(name, value) run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) type_normalization.normalize_type.cache_clear() + adapter: DuckDbAdapter = DuckDbAdapter() + project: CompiledProjectView = native_module.build_project_graph( + discovered_inputs=native_module.discover_project_inputs(project_dir=project_dir), + adapter=adapter, + ).project + result: ContractValidationResult = native_module.evaluate_model_contracts( + project=project, dialect=adapter.sql_analysis_dialect() + ) return run, NativeTypeAnswers( normalized=normalized, - typed_comparisons=statuses["typed_comparisons"], + typed_comparisons=contract_statuses(project=project, result=result)["typed_comparisons"], ) @@ -4975,6 +4966,8 @@ def compile_with_binding_error(project_dir: Path) -> dict[str, Any]: "import time\n\n\n" "def slow_total(amount: str) -> str:\n" ' """Return the amount, pausing for the macro tests that only SQL lint expands."""\n' + " if amount == 'model_amount':\n" + " time.sleep(2.0)\n" " if amount == 'slow_amount':\n" " with open(os.environ['SLOW_MACRO_MARKER'], 'a', encoding='utf-8') as marker:\n" " marker.write('expanding\\n')\n" @@ -4994,7 +4987,8 @@ def slow_lint_model_files(*, model_count: int) -> dict[str, str]: ), "models/slow_total.sql": ( 'MODEL (description "Slow total");\n' - 'SELECT @slow_total("amount") AS amount FROM (SELECT 5 AS amount) AS input_values\n' + 'SELECT @slow_total("model_amount") AS amount\n' + "FROM (SELECT 5 AS model_amount) AS input_values\n" ), **{ f"tests/slow_total_{index}.sql": ( @@ -5010,3 +5004,9 @@ def slow_lint_model_files(*, model_count: int) -> dict[str, str]: for index in range(model_count) }, } + + +def restore_default_interrupt() -> None: + """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" + + _ = signal.signal(signal.SIGINT, signal.SIG_DFL) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py index 8ca339f922..77d262f4b3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py @@ -19,6 +19,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( compile_json_payload, diagnostic_location_keys, + restore_default_interrupt, slow_lint_model_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project @@ -132,6 +133,7 @@ def test_given_slow_background_lint_when_interrupted_then_compile_stops_promptly "SLOW_MACRO_MARKER": str(marker), "SLOW_MACRO_SECONDS": str(test_case.macro_seconds), }, + preexec_fn=restore_default_interrupt, ) deadline: float = time.monotonic() + 60.0 while not marker.exists() and time.monotonic() < deadline and process.poll() is None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index f8dc62f435..64e5273d6b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -3,16 +3,13 @@ from __future__ import annotations from pathlib import Path -from unittest.mock import ANY, Mock +from unittest.mock import ANY import pytest import sqlbuild._native as native_module from sqlbuild._native import ( - ContractValidationResult, NativeSqlTestArtifact, - NativeSqlTestAssembly, - ProjectColumnLineage, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, @@ -29,14 +26,18 @@ NativeAnalysisSeamTestCase( description="models_contracts_lineage_and_sql_tests", expected_native_returns={ - "infer_native_expression_source_shapes": [ANY, ANY, ANY, ANY], - "assemble_native_sql_tests": [ANY], - "complete_semantic_project": [ANY], - "evaluate_model_contracts": [ANY], - "promotion_conflict_diagnostics": [()], - "build_project_column_lineage": [ANY], + "infer_native_expression_source_shapes": [ANY, ANY, ANY], "plan_compiled_sql_test_artifacts": [ANY], }, + expected_stage_work={ + "evaluate_model_contracts": 1, + "build_project_column_lineage": 1, + "infer_native_expression_source_shapes": 1, + "analyze_native_model_sql": 1, + "complete_semantic_project": 1, + "assemble_native_sql_tests": 1, + "promotion_conflict_diagnostics": 1, + }, ) ], ids=lambda case: case.description, @@ -49,8 +50,6 @@ def test_given_project_when_compiling_then_native_seams_answer( ) -> None: prepared_project: Path = tmp_path / "orders" prepare_compile_reuse_project(project_dir=prepared_project) - native_sql_test_planning: Mock = Mock(wraps=native_module.plan_compiled_sql_tests) - monkeypatch.setattr(native_module, "plan_compiled_sql_tests", native_sql_test_planning) native_run, native_seams = seam_recording_compile( project_dir=prepared_project, monkeypatch=monkeypatch, capsys=capsys ) @@ -58,26 +57,16 @@ def test_given_project_when_compiling_then_native_seams_answer( assert native_run.returncode == 0, native_run.stderr assert native_run.compiled assert native_seams == test_case.expected_native_returns - planned_counts: list[int] = [] + stage_work: dict[str, int] = native_module.compile_stage_work_info() + assert { + stage: stage_work.get(stage, 0) for stage in test_case.expected_stage_work + } == test_case.expected_stage_work for seams in (native_seams,): - assert isinstance(seams["build_project_column_lineage"][0], ProjectColumnLineage) assert isinstance(seams["infer_native_expression_source_shapes"][0], tuple) - assembled_tests: object = seams["assemble_native_sql_tests"][0] - assert isinstance(assembled_tests, tuple) - assert assembled_tests - assert all(isinstance(assembled, NativeSqlTestAssembly) for assembled in assembled_tests) planned_artifacts: object = seams["plan_compiled_sql_test_artifacts"][0] assert isinstance(planned_artifacts, tuple) assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) - planned_counts.append(len(planned_artifacts)) - completed: object = seams["complete_semantic_project"][0] - assert isinstance(completed, native_module.NativeProjectView) - assert any(model.inferred_columns for model in completed.models) - assert isinstance(seams["evaluate_model_contracts"][0], ContractValidationResult) - assert [ - len(call.args[0].tests) for call in native_sql_test_planning.call_args_list - ] == planned_counts if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 2090faf3ca..6828de306d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from collections.abc import Callable, Mapping +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path from textwrap import dedent @@ -13,8 +13,7 @@ import pytest from _pytest.capture import CaptureResult -import sqlbuild._native as fingerprint_cache_module -from sqlbuild._native import FINGERPRINT_TABLE_NAME +from sqlbuild._native import FINGERPRINT_TABLE_NAME, migration_fingerprint_work_info from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.models import MigrationStagePlan from sqlbuild.adapter.contract.types import MigrationTransfer @@ -750,34 +749,16 @@ def state_tables(*, project_dir: Path, schema: str) -> tuple[str, ...]: ) -type FingerprintComputation = tuple[str, str, tuple[tuple[str, str], ...], str | None] +def fingerprint_work() -> tuple[int, int]: + """Return migration fingerprints computed and distinct requests since the last reset.""" + return migration_fingerprint_work_info() -def record_fingerprint_computations( - monkeypatch: pytest.MonkeyPatch, -) -> list[FingerprintComputation]: - """Record every migration fingerprint the planner actually computes.""" - computed: list[FingerprintComputation] = [] - original: Callable[..., str | None] = fingerprint_cache_module.build_migration_fingerprint +def reset_fingerprint_work() -> None: + """Forget migration fingerprint work recorded so far in this process.""" - def recording( - *, - query_sql: str, - metadata_json: str, - ref_identities: Mapping[str, str], - dialect: str | None, - ) -> str | None: - computed.append((query_sql, metadata_json, tuple(sorted(ref_identities.items())), dialect)) - return original( - query_sql=query_sql, - metadata_json=metadata_json, - ref_identities=ref_identities, - dialect=dialect, - ) - - monkeypatch.setattr(fingerprint_cache_module, "build_migration_fingerprint", recording) - return computed + migration_fingerprint_work_info(reset=True) def models_with_stored_migration_fingerprints(project_dir: Path) -> tuple[str, ...]: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_fingerprint_work.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_fingerprint_work.py index 5053547d82..0e1f4b1ed9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_fingerprint_work.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_fingerprint_work.py @@ -11,15 +11,15 @@ ) from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( CliRun, - FingerprintComputation, build_each, + fingerprint_work, load_raw_orders, models_with_stored_migration_fingerprints, original_order_models, plan_json, planned_migrations, - record_fingerprint_computations, renamed_order_models, + reset_fingerprint_work, run_sqb, write_project, ) @@ -58,17 +58,17 @@ def test_given_project_state_when_planning_then_fingerprints_only_migration_cand test_case: MigrationFingerprintWorkTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) build_each(project_dir=tmp_path, model_sets=test_case.prior_models, capsys=capsys) write_project(project_dir=tmp_path, models=test_case.models()) - computed: list[FingerprintComputation] = record_fingerprint_computations(monkeypatch) + reset_fingerprint_work() migrations: int = len(planned_migrations(plan_json(project_dir=tmp_path, capsys=capsys))) assert migrations == test_case.expected_migrations - assert len(computed) == len(set(computed)) == test_case.expected_computations + computations, distinct = fingerprint_work() + assert computations == distinct == test_case.expected_computations @pytest.mark.parametrize( @@ -117,19 +117,19 @@ def test_given_project_state_when_building_then_stores_fingerprints_computed_onc test_case: MigrationFingerprintWorkTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: load_raw_orders(project_dir=tmp_path, first_day=1, last_day=5) build_each(project_dir=tmp_path, model_sets=test_case.prior_models, capsys=capsys) write_project(project_dir=tmp_path, models=test_case.models()) migrations: int = len(planned_migrations(plan_json(project_dir=tmp_path, capsys=capsys))) - computed: list[FingerprintComputation] = record_fingerprint_computations(monkeypatch) + reset_fingerprint_work() result: CliRun = run_sqb(project_dir=tmp_path, args=test_case.command, capsys=capsys) assert result.exit_code == 0, result.output assert migrations == test_case.expected_migrations - assert len(computed) == len(set(computed)) == test_case.expected_computations + computations, distinct = fingerprint_work() + assert computations == distinct == test_case.expected_computations assert models_with_stored_migration_fingerprints(tmp_path) == (test_case.expected_stored_models) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 95eefdc761..d0d62ffc72 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -17,11 +17,8 @@ import sqlbuild.compiler.compile._helpers.render.user_macros as macros_module from sqlbuild._native import ( CustomHostObserver, - DeclarationScopeBuild, DiscoveredMacroFile, - DiscoveredProjectInputs, LoadedMacro, - SqlLexicalSyntax, rules_bulk_cache_path, ) from sqlbuild._native import run_cli as main @@ -803,7 +800,6 @@ def test_given_warm_scoped_macro_sql_test_when_compiling_then_rules_reuse_compil test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', @@ -824,31 +820,13 @@ def test_given_warm_scoped_macro_sql_test_when_compiling_then_rules_reuse_compil ) macro_finding: str = f"tests/unit/test_macro_orders.sql:{test_case.expected_code}" cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) - scope_builds: list[Path] = [] - original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope - - def counting_build( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], - sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, - ) -> DeclarationScopeBuild: - scope_builds.append(tmp_path) - return original_build( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, - ) - - monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) + _ = native_module.compile_stage_work_info(reset=True) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) assert macro_finding in cold assert warm == cold - assert len(scope_builds) == 1 + assert native_module.compile_stage_work_info()["build_declaration_scope"] == 1 @pytest.mark.parametrize( @@ -879,38 +857,21 @@ def test_given_cold_scoped_macro_sql_test_when_compiling_then_early_lint_uses_pr "SELECT 1\n", encoding="utf-8", ) - scope_builds: list[Path] = [] macro_loads: list[dict[str, LoadedMacro]] = [] - original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope original_load: Callable[..., dict[str, LoadedMacro]] = macros_module.load_project_macros - def counting_build( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], - sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, - ) -> DeclarationScopeBuild: - scope_builds.append(tmp_path) - return original_build( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, - ) - def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, LoadedMacro]: loaded: dict[str, LoadedMacro] = original_load(macro_files) macro_loads.append(loaded) return loaded - monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) monkeypatch.setattr(macros_module, "load_project_macros", recording_load) + _ = native_module.compile_stage_work_info(reset=True) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) assert f"tests/unit/test_macro_orders.sql:{test_case.expected_code}" in cold - assert len(scope_builds) == 1 + assert native_module.compile_stage_work_info()["build_declaration_scope"] == 1 assert len(macro_loads) == 2 assert macro_loads[0]["row_limit"].function is not macro_loads[1]["row_limit"].function diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 38befd7710..a4f2c1993f 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -10,7 +10,7 @@ from itertools import chain from operator import attrgetter from pathlib import Path -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Literal, cast import pytest @@ -47,7 +47,7 @@ from tests.integration.src.sqlbuild.compiler.helpers import GoldenEntry, golden_entry if TYPE_CHECKING: - from sqlbuild._native import CompileInputsView, ModelInput, ProofRow + from sqlbuild._native import CompileInputsView, ModelInput LINEAGE_NATIVE: str = "native" @@ -395,32 +395,22 @@ def pivot_project_files() -> dict[str, str]: } -@dataclass +@dataclass(frozen=True) class NativePivotProofs: - """Every native pivot proof assembly took, and how many a finished session proved.""" + """How many standalone pivot proofs assembly ran, and how many a session proved.""" - proofs: list[DynamicColumnContractProof | None] = field(default_factory=list) - session_proofs: int = 0 + proofs: int + session_proofs: int -def native_pivot_proofs(*, monkeypatch: pytest.MonkeyPatch) -> NativePivotProofs: - """Record every batch of native pivot proofs assembly takes.""" +def native_pivot_proofs() -> NativePivotProofs: + """The standalone pivot proofs assembly ran since the counters were last reset.""" - recorded_proofs: NativePivotProofs = NativePivotProofs() - native_proofs: Callable[..., tuple[ProofRow | None, ...]] = ( - native_module.prove_native_dynamic_contracts + work: dict[str, int] = native_module.compile_stage_work_info(reset=True) + return NativePivotProofs( + proofs=work.get("pivot_proofs", 0), session_proofs=work.get("session_pivot_proofs", 0) ) - def recorded(**keywords: Any) -> tuple[ProofRow | None, ...]: - proofs: tuple[ProofRow | None, ...] = native_proofs(**keywords) - recorded_proofs.proofs.extend(map(contract_proof, proofs)) - proven: int = sum(proof is not None for proof in proofs) - recorded_proofs.session_proofs += proven * (keywords["session"] is not None) - return proofs - - monkeypatch.setattr(native_module, "prove_native_dynamic_contracts", recorded) - return recorded_proofs - def generated_analysis_files(*, rng: random.Random, model_count: int) -> dict[str, str]: """Sources (typed, inferred and untyped), a pivot and models drawing on every template.""" @@ -548,6 +538,9 @@ class NativeAnalysisRuns: sessions: list[Any] = field(default_factory=list) +type AssemblyStage = Literal["expression_shapes", "model_facts", "semantic_completion"] + + def analyse_natively( *, inputs: CompileInputsView, @@ -555,7 +548,6 @@ def analyse_natively( lineage_mode: ColumnLineageMode, runs: NativeAnalysisRuns, cache_root: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: """Assemble `inputs` with a cold, then a warm analysis cache, then uncached, and compare them. @@ -565,67 +557,67 @@ def analyse_natively( """ views: list[object] = [] - sinks: list[NativeAnalysisRuns] = [runs] - profiles: list[ExpressionInferenceProfile] = [] - proved: Callable[..., tuple[ProofRow | None, ...]] = ( - native_module.prove_native_dynamic_contracts - ) def recorded_shapes( - *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile - ) -> tuple[dict[str, str] | None, ...]: + sink: NativeAnalysisRuns, + *, + expressions: tuple[str, ...], + profile: ExpressionInferenceProfile, + ) -> None: catalog: Any = cast(Any, profile.binding_catalog) native: tuple[dict[str, str] | None, ...] = infer_native_expression_source_shapes( expressions=expressions, profile=profile ) - profiles.append(profile) - sinks[-1].expression_shapes += sum(shape is not None for shape in native) - sinks[-1].golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) - return native - - def recorded_analyses(**keywords: Any) -> tuple[ProofRow | None, ...]: - for profile in profiles[-1:]: - _record_analyses( - inputs=inputs, - catalog=profile.binding_catalog, - proof_inputs=(keywords["tables"], keywords["session"]), - views=views, - runs=sinks[-1], - ) - profiles.clear() - return proved(**keywords) - - completed: Callable[..., CompiledProjectView] = native_module.complete_semantic_project - - def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> CompiledProjectView: - sinks[-1].golden.extend( + sink.expression_shapes += sum(shape is not None for shape in native) + sink.golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) + + def recorded_analyses( + sink: NativeAnalysisRuns, + *, + profile: ExpressionInferenceProfile, + session: Any, + tables: NativePivotTables, + ) -> None: + _record_analyses( + inputs=inputs, + catalog=profile.binding_catalog, + proof_inputs=(tables, session), + views=views, + runs=sink, + ) + + def recorded_proofs( + sink: NativeAnalysisRuns, *, project: CompiledProjectView, **keywords: Any + ) -> CompiledProjectView: + sink.golden.extend( golden_entry("proof", model.dynamic_column_contract) for model in project.models ) - with monkeypatch.context() as completing: - completing.setattr( - native_module, - "infer_native_expression_source_shapes", - infer_native_expression_source_shapes, - ) - return completed(project=project, **keywords) - - with monkeypatch.context() as patch: - patch.setattr(native_module, "complete_semantic_project", recorded_proofs) - patch.setattr(native_module, "infer_native_expression_source_shapes", recorded_shapes) - patch.setattr(native_module, "prove_native_dynamic_contracts", recorded_analyses) - for cache_dir, sink in ( - (cache_root, NativeAnalysisRuns()), - (cache_root, NativeAnalysisRuns()), - (None, runs), - ): - sinks.append(sink) - with suppress(CompileInputError): - _ = native_module.assemble_project( - inputs=inputs, - inference_profile=replace(inference_profile), - column_lineage_mode=lineage_mode, - analysis_cache_dir=cache_dir, - ) + return native_module.complete_semantic_project(project=project, **keywords) + + def stage_inputs(stage: AssemblyStage, cache_dir: Path | None) -> list[dict[str, Any]]: + found: dict[str, Any] | None = native_module.assembly_stage_inputs( + stage, + inputs=inputs, + inference_profile=replace(inference_profile), + column_lineage_mode=lineage_mode, + analysis_cache_dir=cache_dir, + ) + return list(filter(None, [found])) + + for cache_dir, sink in ( + (cache_root, NativeAnalysisRuns()), + (cache_root, NativeAnalysisRuns()), + (None, runs), + ): + with suppress(CompileInputError): + shapes: list[dict[str, Any]] = stage_inputs("expression_shapes", cache_dir) + _ = [recorded_shapes(sink, **found) for found in shapes] + facts: list[dict[str, Any]] = stage_inputs("model_facts", cache_dir) + _ = [recorded_analyses(sink, **found) for found in facts[: len(shapes)]] + _ = [ + recorded_proofs(sink, **found) + for found in stage_inputs("semantic_completion", cache_dir) + ] for index, view in enumerate(views[:-1]): _append(runs, f"model analyses, cached run {index}", views[-1], view) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index ca5f413ed6..33ddbde8b1 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -199,7 +199,6 @@ def test_given_generated_projects_when_analysing_natively_then_cached_and_proven lineage_mode=test_case.lineage_mode, runs=runs, cache_root=tmp_path / f"cache_{index}_{profile_index}", - monkeypatch=monkeypatch, ) assert ( @@ -254,7 +253,6 @@ def test_given_equal_model_queries_when_analysing_natively_then_shares_and_cache lineage_mode=ColumnLineageMode.FAST, runs=runs, cache_root=tmp_path / f"cache_{dialect}", - monkeypatch=monkeypatch, ) for dialect in test_case.dialects ] @@ -312,7 +310,6 @@ def test_given_cyclic_models_when_analysing_natively_then_session_answers_and_ca lineage_mode=lineage_mode, runs=runs, cache_root=tmp_path / f"cache_{dialect}_{lineage_mode.value}", - monkeypatch=monkeypatch, ) for dialect, lineage_mode in product(test_case.dialects, test_case.lineage_modes) ] @@ -402,29 +399,28 @@ def test_given_cyclic_project_when_compiling_then_native_output_is_complete_and_ ids=lambda case: case.description, ) def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_without_the_wheel( - test_case: StandalonePivotProofTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: StandalonePivotProofTestCase, tmp_path: Path ) -> None: inputs: CompileInputsView = compile_inputs( project_dir=tmp_path / "project", files=pivot_project_files() ) - recorded: NativePivotProofs = native_pivot_proofs(monkeypatch=monkeypatch) + _ = native_pivot_proofs() project: CompiledProjectView = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), analysis_model_names=test_case.analysed_models, ) + recorded: NativePivotProofs = native_pivot_proofs() assert ( - all(proof is not None for proof in recorded.proofs), - len(recorded.proofs), + recorded.proofs, recorded.session_proofs, { model.name: getattr(model.dynamic_column_contract, "output_proven", None) for model in project.models }, ) == ( - True, test_case.expected_native_proofs, test_case.expected_session_proofs, test_case.expected_proven_by_model, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index b076b00c34..4055ecad6e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -13,14 +13,6 @@ class RetainedRulesRowsTestCase: expected_codes: tuple[str, ...] -@dataclass(frozen=True) -class UnretainedRulesRowsTestCase: - """A compile whose model facts were not retained, so rules cannot read them.""" - - description: str - expected_error: str - - @dataclass(frozen=True) class ZeroMaterializationTestCase: """An ordinary compile that must not construct the legacy Python project representation.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 72f66866e1..e45320423a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -6,10 +6,9 @@ import shutil from collections import Counter from collections.abc import Callable, Iterator -from dataclasses import replace from pathlib import Path from types import ModuleType -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING import pytest @@ -46,7 +45,7 @@ from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: - from sqlbuild._native import CompileInputsView, ModelInput + from sqlbuild._native import CompileInputsView TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( @@ -83,30 +82,6 @@ def retained_only(project: CompiledProjectView) -> _native.NativeCompiledProject monkeypatch.setattr(module, "compiled_project_facts", retained_only) -def skip_model_retention(*, monkeypatch: pytest.MonkeyPatch) -> None: - """Run the native model loop into a throwaway project instead of the compile's own. - - The compile's own project still runs the loop over no models, so it keeps the loop that - attaches its other resources. - """ - - build_inputs: Callable[..., tuple[ModelInput, ...]] = cast( - "Callable[..., tuple[ModelInput, ...]]", _native.build_model_inputs - ) - - def discarded(**kwargs: Any) -> tuple[ModelInput, ...]: - discovered: DiscoveredProjectInputs = kwargs["discovered_inputs"] - without_models: dict[str, Any] = { - **kwargs, - "discovered_inputs": replace(discovered, model_files=()), - } - _ = build_inputs(**without_models) - throwaway: dict[str, Any] = {**kwargs, "native_project": _native.NativeCompiledProject()} - return build_inputs(**throwaway) - - monkeypatch.setattr(_native, "build_model_inputs", discarded) - - LEGACY_PROJECT_TYPES: tuple[str, ...] = ( "CompileProjectInputs", "CompileModelInput", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 2c9d07b14f..0dd0ca946b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -27,7 +27,6 @@ ModelPathScopeErrorTestCase, RetainedRulesRowsTestCase, SharedInferredColumnsTestCase, - UnretainedRulesRowsTestCase, ZeroMaterializationTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( @@ -48,7 +47,6 @@ expected_summary, forbid_hand_built_projects, inputs_summary, - skip_model_retention, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import ORDERS_PROJECT_FILES, write_files @@ -122,32 +120,6 @@ def test_given_compiled_project_when_evaluating_rules_then_retained_model_facts_ assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes -@pytest.mark.parametrize( - "test_case", - [ - UnretainedRulesRowsTestCase( - description="models the compile-input stage did not retain", - expected_error="compiled model 'orders' has no retained compile facts", - ) - ], - ids=lambda case: case.description, -) -def test_given_unretained_model_facts_when_evaluating_rules_then_compile_reports_it( - test_case: UnretainedRulesRowsTestCase, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, -) -> None: - write_files(project_dir=tmp_path, files=TYPE_PROOF_FILES) - skip_model_retention(monkeypatch=monkeypatch) - - exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) - output: str = "".join(capsys.readouterr()) - - assert exit_code == 1 - assert test_case.expected_error in output - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/graph/helpers.py b/tests/integration/src/sqlbuild/compiler/graph/helpers.py index bb647d0aac..77d4d681fc 100644 --- a/tests/integration/src/sqlbuild/compiler/graph/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/graph/helpers.py @@ -4,10 +4,6 @@ from pathlib import Path -import pytest - -import sqlbuild._native as native_module - ORDERS_PROJECT_FILES: dict[str, str] = { "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', "models/staging/stg_orders.sql": ( @@ -30,23 +26,6 @@ def write_files(*, project_dir: Path, files: dict[str, str]) -> None: path.write_text(text, encoding="utf-8") -def record_compile_owners( - *, monkeypatch: pytest.MonkeyPatch -) -> list[native_module.NativeCompiledProject]: - """Record every native compiled project the compile creates, to read its graph builds.""" - - owners: list[native_module.NativeCompiledProject] = [] - owner_type: type[native_module.NativeCompiledProject] = native_module.NativeCompiledProject - - def recorded() -> native_module.NativeCompiledProject: - owner: native_module.NativeCompiledProject = owner_type() - owners.append(owner) - return owner - - monkeypatch.setattr(native_module, "NativeCompiledProject", recorded) - return owners - - def compiled_models(*, stdout: str) -> dict[str, bool]: """Whether the compile report lists each project model.""" diff --git a/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py b/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py index 3efb4be77f..416390bb84 100644 --- a/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py +++ b/tests/integration/src/sqlbuild/compiler/graph/test_native_project_graph.py @@ -14,7 +14,6 @@ MODEL_NAMES, ORDERS_PROJECT_FILES, compiled_models, - record_compile_owners, write_files, ) @@ -53,12 +52,9 @@ def test_given_compile_selection_when_compiling_then_native_graph_selects_models test_case: NativeGraphSelectionTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: write_files(project_dir=tmp_path, files=ORDERS_PROJECT_FILES) - owners: list[native_module.NativeCompiledProject] = record_compile_owners( - monkeypatch=monkeypatch - ) + _ = native_module.compile_stage_work_info(reset=True) exit_code: int = main( ["--project-dir", str(tmp_path), "--no-color", "compile", "--no-cache", *test_case.select] @@ -68,7 +64,10 @@ def test_given_compile_selection_when_compiling_then_native_graph_selects_models assert compiled_models(stdout=capsys.readouterr().out) == { name: name in test_case.expected_models for name in sorted(MODEL_NAMES) } - assert sum(owner.graph_builds() for owner in owners) == test_case.expected_graph_builds + assert ( + native_module.compile_stage_work_info()["lineage_graph_builds"] + == test_case.expected_graph_builds + ) @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 9487e17ec9..e5f8e00ff5 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -1,13 +1,10 @@ from __future__ import annotations -import functools import json from collections.abc import Callable from pathlib import Path from typing import TYPE_CHECKING, Any -import pytest - import sqlbuild._native as native_module from sqlbuild._native import ( CollectionRendering, @@ -24,7 +21,6 @@ semantic_shapes, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView @@ -215,32 +211,11 @@ def apply_warehouse_setup_sql(*, project_dir: Path, statements: tuple[str, ...]) connection.close() -def traced_shape_batches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, ...]]: - """Record each shape request's expressions the binding catalog's shape cache misses.""" - - batches: list[tuple[str, ...]] = [] - original: Callable[..., Any] = native_module.infer_native_expression_source_shapes - monkeypatch.setattr( - native_module, - "infer_native_expression_source_shapes", - functools.partial(_traced_shapes, original=original, batches=batches), - ) - return batches - +def shape_batches(*, reset: bool = False) -> tuple[int, int]: + """Native shape batches the binding catalog's shape cache missed, and expressions they held.""" -def _traced_shapes( - *, - original: Callable[..., Any], - batches: list[tuple[str, ...]], - expressions: tuple[str, ...], - profile: ExpressionInferenceProfile, -) -> tuple[dict[str, str] | None, ...]: - cache: dict[str, object] = getattr(profile.binding_catalog, "expression_shapes", {}) - pending: tuple[str, ...] = tuple( - dict.fromkeys(filter(lambda item: item not in cache, expressions)) - ) - batches.extend((pending,) * bool(pending)) - return original(expressions=expressions, profile=profile) + work: dict[str, int] = native_module.compile_stage_work_info(reset=reset) + return work.get("expression_shape_batches", 0), work.get("expression_shapes_inferred", 0) _OFFLINE_POSTGRES_CONNECTION: dict[str, object] = { diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py index 314244e82d..5326d7e785 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -18,7 +18,7 @@ ExpressionMemoCase, RestoredShapeCase, ) -from tests.integration.src.sqlbuild.compiler.pipeline.helpers import traced_shape_batches +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import shape_batches @pytest.mark.parametrize( @@ -30,7 +30,6 @@ def test_given_equal_source_expressions_when_compiling_then_infers_once_and_inva test_case: ExpressionMemoCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' @@ -44,17 +43,17 @@ def test_given_equal_source_expressions_when_compiling_then_infers_once_and_inva source.write_text( f"sources:\n - name: raw_orders\n description: Test source raw_orders.\n expression: {test_case.expression}\n - name: raw_customers\n description: Test source raw_customers.\n expression: {test_case.expression}\n" ) - calls: list[tuple[str, ...]] = traced_shape_batches(monkeypatch) + _ = shape_batches(reset=True) args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] assert main(args) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert payload["diagnostics"] == [] - assert len(calls) == test_case.expected_analysis_calls + assert shape_batches()[0] == test_case.expected_analysis_calls source.write_text(source.read_text().replace("AS id", "AS quantity")) assert main(args) == 1 changed: dict[str, Any] = json.loads(capsys.readouterr().out) assert [item["code"] for item in changed["diagnostics"]] == ["B002"] - assert len(calls) == test_case.expected_analysis_calls * 2 + assert shape_batches()[0] == test_case.expected_analysis_calls * 2 @pytest.mark.parametrize( @@ -73,7 +72,6 @@ def test_given_distinct_source_expressions_when_compiling_then_infers_them_in_on test_case: ExpressionBatchCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' @@ -88,11 +86,14 @@ def test_given_distinct_source_expressions_when_compiling_then_infers_them_in_on f" - name: raw_orders\n description: Test source raw_orders.\n expression: {test_case.orders_expression}\n" f" - name: raw_customers\n description: Test source raw_customers.\n expression: {test_case.customers_expression}\n" ) - batches: list[tuple[str, ...]] = traced_shape_batches(monkeypatch) + _ = shape_batches(reset=True) assert main(["--project-dir", str(tmp_path), "compile", "--json"]) == 0 payload: dict[str, Any] = json.loads(capsys.readouterr().out) assert payload["diagnostics"] == [] - assert tuple(batches) == test_case.expected_batches + assert shape_batches() == ( + len(test_case.expected_batches), + sum(len(batch) for batch in test_case.expected_batches), + ) @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py index bb8df9c6ab..d185c972c0 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/fixtures/test_type_scope.py @@ -4,6 +4,7 @@ import pytest from sqlbuild._native import ( + CompiledResourceType, RelationFixtureCompletion, RelationFixturePlanningContext, build_compiled_project, @@ -12,7 +13,7 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.planner._helpers.fixtures._test_types import ( FixtureTypeScopeTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index 40739c6815..ebfacda47b 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -14,6 +14,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompileModelConfig, @@ -37,7 +38,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 8a7992d288..9936f3fb86 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -14,6 +14,7 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, CompileSqlReference, CursorOverridePair, @@ -32,7 +33,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 4a542a445f..418c09cced 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -10,6 +10,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSqlTest, CompileModelConfig, CompileSqlTestCte, @@ -20,7 +21,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.integration.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( ExecuteChainTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py index f174224260..41d31d176e 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_execute_chain.py @@ -11,6 +11,7 @@ CompiledDirectLogicSqlTestPayload, CompiledObjectKey, CompiledProject, + CompiledResourceType, CompiledSqlTest, CompileSqlTestCte, DiscoveredSqlTestBlock, @@ -20,7 +21,7 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py index 6c223c9ada..dd4fe0765c 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_audit_entry.py @@ -10,13 +10,14 @@ CompiledAudit, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredAuditBlock, DiscoveredAuditFile, SourceEntry, plan_audit, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledAuditView from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ExecuteAuditTestCase, ) diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index 98f64d28f0..f60d8ca0e2 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -6,6 +6,7 @@ import pytest from sqlbuild._native import ( + CompiledResourceType, DiscoveredProjectInputs, RelationFixtureCompletion, RelationFixturePlanningContext, @@ -18,7 +19,7 @@ discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( FixturePlanningIntegrationTestCase, FixtureSchemaIntegrationTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py index 475af48f3e..873ec2eb22 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_cursor_reads.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CursorSnapshotScope, ModelCursorSnapshot, PlannerInputError, @@ -15,7 +16,7 @@ ) from sqlbuild.adapter.relations.main.open_inspection_catalog import open_inspection_catalog from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( ConcurrentCursorReadsTestCase, EligibleMaxFailureTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index b82f927619..5488189925 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorOverrides, CursorSnapshotScope, Fingerprint, @@ -29,7 +30,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.integration.src.sqlbuild.compiler.planner._helpers._test_types import ( GatherCursorSnapshotTestCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 803a661c0b..3067f5ba66 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -18,6 +18,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompileModelConfig, @@ -54,7 +55,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, FunctionLanguage, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index b005a94aba..addaa79155 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -1,13 +1,10 @@ from __future__ import annotations -from collections import Counter from dataclasses import replace from typing import Any -from unittest.mock import Mock import pytest -import sqlbuild._native as identity_planning from sqlbuild._native import ( BackfillAction, CursorBounds, @@ -29,6 +26,7 @@ WarningSeverity, compiled_model_record, format_plan, + planner_identity_work_info, project_with, ) from sqlbuild.adapter.contract.types import BuiltinAdapter @@ -84,7 +82,6 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d test_case: PlannerChangeDetectionWorkTestCase, adapter: DuckDbAdapter, connection: Any, - monkeypatch: pytest.MonkeyPatch, ) -> None: connection.execute("CREATE TABLE staging.stg_orders AS SELECT 1 AS order_id") project: CompiledProjectView = build_project_from_test_case( @@ -114,14 +111,7 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d query_change_tracking=test_case.query_change_tracking, ), ) - stale_detector: Mock = Mock(wraps=identity_planning.detect_model_changes_in_scope) - monkeypatch.setattr(identity_planning, "detect_model_changes_in_scope", stale_detector) - identity_builder: Mock = Mock(wraps=identity_planning.build_direct_model_version_identities) - monkeypatch.setattr( - identity_planning, - "build_direct_model_version_identities", - identity_builder, - ) + planner_identity_work_info(reset=True) plan: PlanOutput = build_execution_plan_from_kwargs( project=project, @@ -131,11 +121,9 @@ def test_given_selected_model_when_planning_then_each_required_model_is_change_d selection_diagnostics=test_case.selection_diagnostics, ) - stale_calls: Counter[str] = Counter() - for call in stale_detector.call_args_list: - stale_calls.update(key.name for key in call.kwargs["scope"].selected_keys) - assert dict(stale_calls) == test_case.expected_stale_detection_calls - assert identity_builder.call_count == test_case.expected_identity_builds + identity_builds, stale_detections = planner_identity_work_info() + assert stale_detections == test_case.expected_stale_detection_calls + assert identity_builds == test_case.expected_identity_builds assert len(plan.warnings) == test_case.expected_warning_count diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 54f83d4884..a0488f1fa2 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -10,19 +10,16 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, cast -import pytest - import sqlbuild._native as native_module from sqlbuild._native import ( CompilerDiagnostic, - build_project_graph, - discover_project_inputs, project_with, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( + compile_inputs, generated_analysis_files, ) @@ -188,29 +185,16 @@ def captured_semantic_inputs( *, project_dir: Path, files: dict[str, str], - monkeypatch: pytest.MonkeyPatch, ) -> SemanticInputs: """Compile and keep the semantic completion stage's inputs.""" - for relative_path, contents in files.items(): - path: Path = project_dir / relative_path - path.parent.mkdir(parents=True, exist_ok=True) - _ = path.write_text(contents, encoding="utf-8") - captured: list[SemanticInputs] = [] - - complete: Callable[..., CompiledProjectView] = native_module.complete_semantic_project - - def capture(**arguments: Any) -> CompiledProjectView: - captured.append(SemanticInputs(**arguments)) - return complete(**arguments) - - with monkeypatch.context() as patch: - patch.setattr(native_module, "complete_semantic_project", capture) - _ = build_project_graph( - discovered_inputs=discover_project_inputs(project_dir=project_dir), - adapter=DuckDbAdapter(), - ) - return captured[0] + stage: dict[str, Any] | None = native_module.assembly_stage_inputs( + "semantic_completion", + inputs=compile_inputs(project_dir=project_dir, files=files), + inference_profile=DuckDbAdapter().expression_inference_profile(), + ) + assert stage is not None + return SemanticInputs(**stage) def with_dialect(inputs: SemanticInputs, dialect: str | None) -> SemanticInputs: diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py index dd452c37c5..de7023ba50 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py @@ -68,7 +68,6 @@ def test_given_generated_projects_when_checking_metadata_natively_then_matches_recorded_python( test_case: GeneratedMetadataParityTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) families: Counter[str] = Counter() @@ -82,7 +81,6 @@ def test_given_generated_projects_when_checking_metadata_natively_then_matches_r rng=rng, non_ascii=test_case.non_ascii, ), - monkeypatch=monkeypatch, ) views: dict[str, object] = { str(dialect): semantic_outcome( diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 37a7d81982..1f2c4f7dcb 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -77,7 +77,6 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_recorded_python( test_case: GeneratedSemanticParityTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) statuses: Counter[str] = Counter() @@ -90,7 +89,6 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ files=generated_semantic_files( rng=rng, model_count=test_case.model_count, require_analysis=index % 2 == 0 ), - monkeypatch=monkeypatch, ) views: dict[str, object] = {} for dialect in test_case.dialects: @@ -156,7 +154,6 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ def test_given_formerly_deferred_projects_when_completing_natively_then_matches_recorded_python( test_case: FormerlyDeferredSemanticTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) golden: list[GoldenEntry] = [] @@ -169,7 +166,6 @@ def test_given_formerly_deferred_projects_when_completing_natively_then_matches_ captured: SemanticInputs = captured_semantic_inputs( project_dir=project_dir, files=files_with_non_ascii(files=files, rng=rng, non_ascii=test_case.non_ascii), - monkeypatch=monkeypatch, ) views: dict[str, object] = {} for dialect in test_case.dialects: @@ -206,12 +202,10 @@ def test_given_formerly_deferred_projects_when_completing_natively_then_matches_ def test_given_project_without_catalog_when_completing_then_raises_native_compiler_error( test_case: MissingCatalogTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: captured: SemanticInputs = captured_semantic_inputs( project_dir=tmp_path / "project", files=generated_semantic_files(rng=random.Random(5), model_count=8, require_analysis=False), - monkeypatch=monkeypatch, ) with pytest.raises(native_module.NativeCompilerError) as raised: @@ -253,7 +247,6 @@ def test_given_project_without_catalog_when_completing_then_raises_native_compil def test_given_compiles_when_completing_from_the_session_then_matches_payload_and_python( test_case: SessionCompletionTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) captured: list[SemanticInputs] = [ @@ -262,7 +255,6 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an files=session_corpus_files( corpus=test_case.corpus, rng=rng, model_count=test_case.model_count ), - monkeypatch=monkeypatch, ) for index in range(test_case.count) ] diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index c835c7305b..953eb66629 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -10,13 +10,14 @@ from pathlib import Path from typing import TYPE_CHECKING, cast -import pytest - import sqlbuild._native as native_module from sqlbuild._native import ( + CollectionRendering, + CompileAdapterContext, CompiledSqlTest, NativeSqlTestAssembly, NativeSqlTestFailure, + build_compile_inputs, build_project_graph, discover_project_inputs, ) @@ -438,6 +439,14 @@ def _helper_mock_test(*, rng: random.Random, index: int) -> str: return f'TEST (name "helper_mock_{index}");\n\nWITH\n{comment}{ctes}\nSELECT 1\n' +_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( + value_renderer=DuckDbAdapter(), + collection_rendering=CollectionRendering.VALUE_LIST, + python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, +) + + def write_project(*, project_dir: Path, files: dict[str, str]) -> None: """Write a project's files below `project_dir`.""" @@ -461,29 +470,28 @@ def _compiled_tests_and_diagnostics(*, project_dir: Path) -> tuple[object, objec return project.sql_tests, project.diagnostics -def record_native_assemblies(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count tests compile takes from the native assembly, with diagnostics, cases or failures. +def native_assembly_answers(*, project_dir: Path) -> Counter[str]: + """Count tests the native assembly answers for a project, with diagnostics, cases or failures. A test whose assembly raises counts under `failed_`. """ + inputs: CompileInputsView = build_compile_inputs( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter_context=_ADAPTER_CONTEXT, + run_id="integration_run", + ) + assemblies: tuple[NativeSqlTestAssembly, ...] = native_module.assemble_native_sql_tests( + inputs=inputs + ) answers: Counter[str] = Counter() - seam: Callable[..., tuple[NativeSqlTestAssembly, ...]] = native_module.assemble_native_sql_tests - - def counted_assemblies(*, inputs: CompileInputsView) -> tuple[NativeSqlTestAssembly, ...]: - assemblies: tuple[NativeSqlTestAssembly, ...] = seam(inputs=inputs) - failures: list[NativeSqlTestFailure | None] = [item.failure for item in assemblies] - answers.update(f"failed_{failure.kind}" for failure in filter(None, failures)) - tests: list[CompiledSqlTest | None] = [item.test for item in assemblies] - native: list[CompiledSqlTest] = list(filter(None, tests)) - answers["native_assembled"] += len(native) - answers["native_with_diagnostics"] += sum(bool(item.diagnostics) for item in assemblies) - answers["native_case_fingerprints"] += sum( - test.case_fingerprint is not None for test in native - ) - return assemblies - - monkeypatch.setattr(native_module, "assemble_native_sql_tests", counted_assemblies) + failures: list[NativeSqlTestFailure | None] = [item.failure for item in assemblies] + answers.update(f"failed_{failure.kind}" for failure in filter(None, failures)) + tests: list[CompiledSqlTest | None] = [item.test for item in assemblies] + native: list[CompiledSqlTest] = list(filter(None, tests)) + answers["native_assembled"] += len(native) + answers["native_with_diagnostics"] += sum(bool(item.diagnostics) for item in assemblies) + answers["native_case_fingerprints"] += sum(test.case_fingerprint is not None for test in native) return answers diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_assembly.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_assembly.py index a553eeefa3..ff8dbab312 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_assembly.py @@ -13,6 +13,7 @@ import pytest +import sqlbuild._native as native_module from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, @@ -39,8 +40,8 @@ assembly_outcome, edge_sql_test_files, generated_sql_test_assembly_files, + native_assembly_answers, outcome_kind, - record_native_assemblies, write_project, ) @@ -72,10 +73,10 @@ def test_given_generated_sql_tests_when_assembling_natively_then_compiled_tests_match_python( test_case: GeneratedSqlTestAssemblyParityTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) - native_assemblies: Counter[str] = record_native_assemblies(monkeypatch=monkeypatch) + native_assemblies: Counter[str] = Counter() + _ = native_module.compile_stage_work_info(reset=True) entries: list[GoldenEntry] = [] outcomes: Counter[str] = Counter() for index in range(test_case.count): @@ -87,6 +88,7 @@ def test_given_generated_sql_tests_when_assembling_natively_then_compiled_tests_ ), ) actual: PlanningCallOutcome = assembly_outcome(project_dir=project_dir) + native_assemblies.update(native_assembly_answers(project_dir=project_dir)) entries.append(assembly_golden_entry(outcome=actual, project_dir=project_dir)) outcomes[outcome_kind(actual)] += 1 @@ -95,7 +97,8 @@ def test_given_generated_sql_tests_when_assembling_natively_then_compiled_tests_ read_golden(golden_name("sql_test_assembly", test_case.description)), entries ), dict(native_assemblies), - ) == ([], test_case.expected_native_assemblies), outcomes + native_module.compile_stage_work_info()["assemble_native_sql_tests"], + ) == ([], test_case.expected_native_assemblies, test_case.count), outcomes @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py index 3426afc360..09608926ef 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/helpers.py @@ -215,8 +215,7 @@ def extract_upstream_model_deps( ) -> dict[str, frozenset[str]]: """Extract upstream model dependency names for each model in the plan.""" - from sqlbuild._native import CompiledObjectKey - from sqlbuild.compiler.compile.types import CompiledResourceType + from sqlbuild._native import CompiledObjectKey, CompiledResourceType dependencies_by_resource_type: dict[object, dict[str, frozenset[str]]] = { resource_type: {} for resource_type in CompiledResourceType diff --git a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py index 4982244da4..bf06f2c529 100644 --- a/tests/integration/src/sqlbuild/executor/build/custom/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/custom/helpers.py @@ -10,6 +10,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -23,7 +24,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.custom.models import ( MaterializationContext, diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index 3f84e5983c..10c2bcbc94 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -4,11 +4,13 @@ import json from collections.abc import Callable +from dataclasses import replace from pathlib import Path from textwrap import dedent from typing import Any import duckdb +import pytest from sqlbuild._native import ( CompilePipelineOptions, @@ -23,6 +25,7 @@ from sqlbuild._native import run_cli as main from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildExecutionResult, @@ -474,20 +477,19 @@ def verify_warehouse_state( assert cursor.fetchone() is None, f"Relation {relation} should not exist but was found" -def drift_for_models( - *, model_names: tuple[str, ...] -) -> Callable[..., tuple[TableTypePlanEntry, ...]]: - """Return a planner stand-in reporting permanent-desired drift for the named models.""" +def drift_for_models(*, model_names: tuple[str, ...], monkeypatch: pytest.MonkeyPatch) -> None: + """Hand the build scheduler a plan reporting permanent-desired drift for the named models.""" - def plan_table_types( - *, runtime: Any, warehouse: Any, scope: Any - ) -> tuple[TableTypePlanEntry, ...]: - del warehouse, scope - models_by_name: dict[str, Any] = {model.name: model for model in runtime.project.models} - return tuple( + original: Callable[..., None] = BuildScheduler.__init__ + + def drifted_init(self: BuildScheduler, *, plan: PlanOutput, **kwargs: Any) -> None: + destinations: dict[str, Any] = { + entry.name: entry.destination for entry in plan.model_entries + } + entries: tuple[TableTypePlanEntry, ...] = tuple( TableTypePlanEntry( model_name=name, - destination=models_by_name[name].destination, + destination=destinations[name], copy_name=f"__sqb_type_swap__{name}", desired_type="permanent", actual_type="transient", @@ -497,8 +499,9 @@ def plan_table_types( ) for name in model_names ) + original(self, plan=replace(plan, table_type_entries=entries), **kwargs) - return plan_table_types + monkeypatch.setattr(BuildScheduler, "__init__", drifted_init) def recording_table_type_conversion(*, conversions: list[tuple[str, int]]) -> Callable[..., None]: diff --git a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py index 08999a5dab..15a8a68b07 100644 --- a/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py +++ b/tests/integration/src/sqlbuild/executor/build/test_table_type_routing.py @@ -8,7 +8,6 @@ import duckdb import pytest -import sqlbuild._native as planner_assembly_module import sqlbuild.executor.build.classes.build_scheduler as build_scheduler_module from tests.integration.src.sqlbuild.executor.build._test_types import TableTypeRoutingTestCase from tests.integration.src.sqlbuild.executor.build.helpers import ( @@ -77,11 +76,7 @@ def test_given_table_type_drift_when_building_then_only_carried_over_tables_conv with duckdb.connect(str(tmp_path / "demo.duckdb")) as setup: setup.execute(_NEW_ORDER_SQL) conversions: list[tuple[str, int]] = [] - monkeypatch.setattr( - planner_assembly_module, - "plan_table_types", - drift_for_models(model_names=("customer_totals", "orders")), - ) + drift_for_models(model_names=("customer_totals", "orders"), monkeypatch=monkeypatch) monkeypatch.setattr( build_scheduler_module, "apply_table_type_conversion", @@ -125,11 +120,7 @@ def test_given_failing_table_type_conversion_when_building_then_model_fails_with with duckdb.connect(str(tmp_path / "demo.duckdb")) as setup: setup.execute(_NEW_ORDER_SQL) conversions: list[tuple[str, int]] = [] - monkeypatch.setattr( - planner_assembly_module, - "plan_table_types", - drift_for_models(model_names=("customer_totals", "orders")), - ) + drift_for_models(model_names=("customer_totals", "orders"), monkeypatch=monkeypatch) monkeypatch.setattr( build_scheduler_module, "apply_table_type_conversion", diff --git a/tests/integration/src/sqlbuild/executor/run/helpers.py b/tests/integration/src/sqlbuild/executor/run/helpers.py index 58662db772..208a979a2a 100644 --- a/tests/integration/src/sqlbuild/executor/run/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/helpers.py @@ -11,6 +11,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -24,7 +25,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.main._execute import execute_table_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py index 4a0f56e9b0..e87555169d 100644 --- a/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/incremental/helpers.py @@ -11,6 +11,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, CursorInputRelation, DiscoveredHookFunction, @@ -28,7 +29,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.incremental import execute_incremental_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py index 7a19c2c803..ce1b74633e 100644 --- a/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/microbatch/helpers.py @@ -13,6 +13,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, CursorInputRelation, DiscoveredHookFunction, @@ -28,7 +29,6 @@ AuditAttachmentKind, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.microbatch import execute_microbatch_entry from sqlbuild.executor.run._helpers.validation.cursor_bounds import resolve_runtime_cursor_bounds from sqlbuild.executor.run.models import ( diff --git a/tests/integration/src/sqlbuild/executor/run/view/helpers.py b/tests/integration/src/sqlbuild/executor/run/view/helpers.py index 25b4a0e41c..3800e8aa8a 100644 --- a/tests/integration/src/sqlbuild/executor/run/view/helpers.py +++ b/tests/integration/src/sqlbuild/executor/run/view/helpers.py @@ -10,6 +10,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -22,7 +23,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.view import execute_view_entry from sqlbuild.executor.run.models import ( HookContext, diff --git a/tests/integration/src/sqlbuild/executor/scenario/helpers.py b/tests/integration/src/sqlbuild/executor/scenario/helpers.py index d8e679f0c1..afe96ebcd5 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/helpers.py +++ b/tests/integration/src/sqlbuild/executor/scenario/helpers.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -20,7 +21,6 @@ ScenarioRelationMap, ScenarioRelationPlan, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.models import HookContext SCENARIO_NAME: str = "revenue__customer_refund" diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py index 1c5a05a9fc..997653f119 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_fixture_lifecycle.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, PythonHookEntry, ScenarioExecutionPlan, @@ -17,7 +18,6 @@ from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.models import SeedExecutionResult from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scenario._helpers.execution.model_execution import execute_scenario_models diff --git a/tests/integration/src/sqlbuild/executor/testing/helpers.py b/tests/integration/src/sqlbuild/executor/testing/helpers.py index 6fcc3ecd2c..e896068c20 100644 --- a/tests/integration/src/sqlbuild/executor/testing/helpers.py +++ b/tests/integration/src/sqlbuild/executor/testing/helpers.py @@ -9,12 +9,12 @@ from sqlbuild._native import ( ChainStep, CompiledObjectKey, + CompiledResourceType, CompilePipelineResult, SqlTestPlanEntry, render_sql_test_comparison_sql, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.testing.main._execute import execute_sql_test from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index e8940a0343..c79c282750 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -5,6 +5,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, @@ -17,7 +18,6 @@ ) from sqlbuild.adapter.contract.models import RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope -from sqlbuild.compiler.compile.types import CompiledResourceType def build_snapshot_full_refresh_entry( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index 9a86d50d88..19883313a2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -86,9 +86,8 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh project_dir=tmp_path, record=replace(record, cost=replace(record.cost, status=test_case.initial_status)), ) - monkeypatch.setattr( - "sqlbuild._native.discover_project_inputs", - lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), + _ = (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "shop"\nadapter = "snowflake"\n', encoding="utf-8" ) monkeypatch.setattr( "sqlbuild._native.resolve_effective_adapter_name", diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py index 5c431c8e83..b045f5f8c8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/helpers.py @@ -7,6 +7,7 @@ ChainStep, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CostContext, MaterializationType, ModelPlanEntry, @@ -19,7 +20,6 @@ ) from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.cost.models import CostResourceContext diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py index 1f72c96b35..45ce23fc8c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1,7 +1,6 @@ from __future__ import annotations -from sqlbuild._native import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledObjectKey, CompiledResourceType def build_compiled_object_key( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py index 3b87f9458a..e0a17839fc 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/build/test_defer_clone.py @@ -8,11 +8,12 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, PlannerScope, resolve_defer_clone_selection, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.cli.commands.main.build._test_types import ( DeferCloneBoundaryTestCase, DeferCloneModelCountTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index e0e8a179d9..17410ca9a1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, FunctionArgument, FunctionPlanEntry, @@ -40,7 +41,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, FunctionLanguage, ) from sqlbuild.executor.build.models import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 5ec96f130e..1d901e9c4d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -498,14 +498,7 @@ def test_given_lineage_mode_when_compiling_then_lineage_runs_in_that_mode( capsys: pytest.CaptureFixture[str], ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) - modes: list[str] = [] - build_lineage: Callable[..., object] = discovery_entry.build_project_column_lineage - - def recorded_lineage(*args: Any) -> object: - modes.append(ColumnLineageMode(args[2]).value) - return build_lineage(*args) - - monkeypatch.setattr(discovery_entry, "build_project_column_lineage", recorded_lineage) + _ = discovery_entry.compile_stage_work_info(reset=True) exit_code: int = run_compile( CompileCommandRequest( @@ -518,7 +511,9 @@ def recorded_lineage(*args: Any) -> object: _ = capsys.readouterr() assert exit_code == 0 - assert tuple(modes) == test_case.expected_lineage_mode_values + assert discovery_entry.compile_stage_work_info().get("build_project_column_lineage", 0) == len( + test_case.expected_lineage_mode_values + ) @pytest.mark.parametrize( @@ -626,16 +621,12 @@ def test_given_collection_state_when_compiling_then_cycles_are_paused_and_state_ ) -> None: project_dir: Path = prepare_static_compile_project(tmp_path) enabled_during: list[bool] = [] - discover: Callable[..., Any] = discovery_entry.discover_project_inputs - def observed_discover(**kwargs: Any) -> Any: + def observed_adapter(*args: Any, **kwargs: Any) -> NoConnectDuckDbAdapter: enabled_during.append(gc.isenabled()) - return discover(**kwargs) + return NoConnectDuckDbAdapter() - monkeypatch.setattr( - runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter() - ) - monkeypatch.setattr(discovery_entry, "discover_project_inputs", observed_discover) + monkeypatch.setattr(runtime_adapters, "resolve_adapter", observed_adapter) set_collection: dict[bool, Callable[[], None]] = {True: gc.enable, False: gc.disable} was_enabled: bool = gc.isenabled() set_collection[test_case.collection_enabled_before]() diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index c25a082aee..69c9c917f2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompileLineageMode, CompileModelConfig, ProjectColumnLineage, @@ -19,7 +20,7 @@ format_compile_text, project_with, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileJsonExecutionLayersTestCase, CompileJsonLargeIntegerTestCase, @@ -64,7 +65,9 @@ def test_given_large_hook_integer_when_formatting_json_then_preserves_value( } ), ) - graph = replace(graph, project=project_with(project=graph.project, models=(model,))) + graph = ProjectGraph( + project=project_with(project=graph.project, models=(model,)), native=graph.native + ) payload: dict[str, object] = json.loads( format_compile_json( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 908f900469..a212eaa5e2 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -12,6 +12,7 @@ import sqlbuild._native as plan_artifacts_module from sqlbuild._native import ( ChainStep, + CompiledResourceType, CompiledSqlTest, DiagnosticPhase, PlanOutput, @@ -28,7 +29,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py index dbcc9a35a7..1b54c4b844 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py @@ -6,6 +6,7 @@ @dataclass(frozen=True) class BuildProcessReportTestCase: description: str + provider_source: str error_type: type[BaseException] verbose: bool expected_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 18a48bbc06..6b1431bc9e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -12,27 +12,28 @@ from sqlbuild import _native as native from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlScenario, - DiscoveredProjectInputs, DiscoveredSqlScenarioFile, - LocalConfig, - ProjectConfig, ) from sqlbuild.adapter.discovery.main import resolve_adapter as adapters_module -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module from sqlbuild.executor.pipeline.main import run as executor_run_module from tests.unit.src.sqlbuild.presentation.helpers import use_plain_terminal -def fail_build_discovery(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) -> Mock: - """Make project discovery raise and record the debug resource reporter calls.""" - - def discover(**_kwargs: object) -> DiscoveredProjectInputs: - raise error +def fail_build_discovery( + *, monkeypatch: pytest.MonkeyPatch, project_dir: Path, provider_source: str +) -> Mock: + """Make project discovery fail in a provider module and record debug resource reporting.""" + providers: Path = project_dir / "providers" + providers.mkdir(parents=True) + _ = (project_dir / "sqlbuild_project.toml").write_text( + 'name = "shop"\nadapter = "duckdb"\n', encoding="utf-8" + ) + _ = (providers / "failing.py").write_text(provider_source, encoding="utf-8") reporter: Mock = Mock(return_value=nullcontext()) - monkeypatch.setattr(native, "discover_project_inputs", discover) monkeypatch.setattr(resource_reporting_module, "process_resource_reporting", reporter) return reporter @@ -53,16 +54,15 @@ def configure_native_scenario_command( monkeypatch: pytest.MonkeyPatch, stream: ScenarioProgressStream, compile_project: Any, + project_dir: Path, ) -> None: - """Isolate the native scenario command from discovery, adapters and the warehouse.""" + """Isolate the native scenario command from adapters and the warehouse in an empty project.""" - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig(name="project", adapter="duckdb"), - local_config=LocalConfig(), + _ = (project_dir / "sqlbuild_project.toml").write_text( + 'name = "project"\nadapter = "duckdb"\n', encoding="utf-8" ) monkeypatch.setattr(sys, "stdout", stream) use_plain_terminal(monkeypatch) - monkeypatch.setattr(native, "discover_project_inputs", lambda **_: discovered_inputs) monkeypatch.setattr(native, "resolve_project_connection_config", lambda **_: {}) monkeypatch.setattr(native, "run_compile_only_pipeline", compile_project) monkeypatch.setattr(adapters_module, "resolve_adapter", lambda **_: Mock()) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py index 429bd1358e..b3284c2102 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py @@ -7,7 +7,12 @@ import pytest -from sqlbuild._native import BuildCommandRequest, CliUserError, run_build_command +from sqlbuild._native import ( + BuildCommandRequest, + CliUserError, + ProviderDiscoveryError, + run_build_command, +) from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( BuildDurationLimitTestCase, BuildProcessReportTestCase, @@ -20,13 +25,15 @@ [ BuildProcessReportTestCase( description="failed debug build reports resources and partial timings", - error_type=RuntimeError, + provider_source='raise RuntimeError("provider import failed")\n', + error_type=ProviderDiscoveryError, verbose=False, expected_fragments=("Phase timings", "total"), expected_absent_fragments=("compile", "planning", "execution"), ), BuildProcessReportTestCase( description="interrupted debug build reports resources and partial timings", + provider_source="raise KeyboardInterrupt\n", error_type=KeyboardInterrupt, verbose=True, expected_fragments=("Phase timings", "total"), @@ -41,7 +48,9 @@ def test_given_failed_debug_build_when_unwinding_then_reports_resources_and_part capsys: pytest.CaptureFixture[str], tmp_path: Path, ) -> None: - reporter: Mock = fail_build_discovery(monkeypatch=monkeypatch, error=test_case.error_type()) + reporter: Mock = fail_build_discovery( + monkeypatch=monkeypatch, project_dir=tmp_path, provider_source=test_case.provider_source + ) with pytest.raises(test_case.error_type): run_build_command( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py index 340918201b..4adb978cb5 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py @@ -59,7 +59,10 @@ def compile_project(**kwargs: Any) -> CompilePipelineResult: return pipeline_result configure_native_scenario_command( - monkeypatch=monkeypatch, stream=stream, compile_project=compile_project + monkeypatch=monkeypatch, + stream=stream, + compile_project=compile_project, + project_dir=tmp_path, ) events: list[LifecycleEvent] = [] dispatcher: EventDispatcher = EventDispatcher() @@ -119,7 +122,10 @@ def fail_compile(**kwargs: Any) -> CompilePipelineResult: raise RuntimeError("compile failed") configure_native_scenario_command( - monkeypatch=monkeypatch, stream=stream, compile_project=fail_compile + monkeypatch=monkeypatch, + stream=stream, + compile_project=fail_compile, + project_dir=tmp_path, ) events: list[LifecycleEvent] = [] dispatcher: EventDispatcher = EventDispatcher() diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py index b9554f7905..6c7eb7748f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/helpers.py @@ -9,6 +9,7 @@ ChainStep, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, @@ -23,7 +24,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py index de19758d14..fa879d11af 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/helpers.py @@ -10,6 +10,7 @@ ColumnRenameHint, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, CursorInputRelation, DiscoveredProviderUsage, @@ -32,7 +33,7 @@ TableTypePlanEntry, WarningSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage +from sqlbuild.compiler.compile.types import FunctionLanguage def build_model_entry( diff --git a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py index ba2277d229..6412cc52f2 100644 --- a/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py +++ b/tests/unit/src/sqlbuild/cli/progress/classes/test_build_sql_test_projection.py @@ -11,13 +11,13 @@ BuildProgressCallbacks, ChainStep, CompiledObjectKey, + CompiledResourceType, NativeProgressProjector, PlanOutput, SqlTestPlanEntry, TerminalEventIndex, terminal_event_index_scope, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run.models import ModelExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult, StepResult diff --git a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py index 5d7f7b34ca..5d835ec7c9 100644 --- a/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/auditing/main/helpers.py @@ -11,6 +11,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, SchemaAuditInstance, SourceEntry, parse_audit_instances, @@ -22,7 +23,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType def build_render_adapter(adapter_name: str | None) -> BaseAdapter | None: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 5f7b1a7378..a48010ff22 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -18,6 +18,7 @@ CompiledModelSqlTestPayload, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompiledSqlTest, CompilerDiagnostic, CompileSqlReference, @@ -67,7 +68,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) if TYPE_CHECKING: diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index cf62a2121f..7f0c985e35 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, DiscoveredProjectInputs, SqlHookEntry, assemble_project, @@ -18,7 +19,6 @@ from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledProjectView, - CompiledResourceType, CompiledSeedView, ) from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py index 8ce9866e21..4af195671d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_parameterized_sql_tests.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlTest, SqlLogicalType, SqlValue, @@ -21,7 +22,6 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( CompiledProjectView, - CompiledResourceType, TypedSqlValueRenderer, ) from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py index 7c3ffd22c2..43ee0aa579 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_required_columns.py @@ -6,11 +6,11 @@ from sqlbuild._native import ( CompiledLineageSourceFact, + CompiledResourceType, CompileSqlReference, SqlReferenceKind, analyze_resolved_column_reads, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( LongUnionRequiredColumnsTestCase, RequiredExternalColumnsTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 4d4345dc32..0803b86221 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -7,6 +7,7 @@ ColumnTransformKind, CompiledLineageColumnFact, CompiledLineageSourceFact, + CompiledResourceType, CompileSqlReference, InferredColumn, InferredNullability, @@ -17,7 +18,6 @@ substitute_placeholder_defaults, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( InferColumnsTestCase, PolyglotAnalysisTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py index 3aed31fcf7..f9da29641b 100644 --- a/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py +++ b/tests/unit/src/sqlbuild/compiler/contract_adoption/_helpers/test_repository_edits.py @@ -1,6 +1,7 @@ import pytest from sqlbuild._native import ( + CompiledResourceType, ContractEvidence, SchemaDynamicColumnFamily, compare_contract_evidence, @@ -8,7 +9,7 @@ ) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.contract_adoption._helpers._test_types import ( DynamicContractAdoptionTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py index 0275c794c9..2f9e482c04 100644 --- a/tests/unit/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/contracts/helpers.py @@ -9,6 +9,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DynamicColumnContractProof, InferredColumn, @@ -20,7 +21,7 @@ SettingsConfig, SourceLocation, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView def make_contract_project( diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index f6a4f748c2..2dba9df49e 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -9,6 +9,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompiledSqlScenario, @@ -35,7 +36,6 @@ from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompiledProjectView, - CompiledResourceType, FunctionLanguage, ) from tests.unit.src.sqlbuild.compiler.graph.helpers import project_graph_from_indexes diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 14303e8a88..606fd31189 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -319,8 +319,8 @@ def test_given_qualified_loader_destination_when_building_dag_then_parses_loader test_case: DagLoaderDestinationTestCase, ) -> None: graph: ProjectGraph = build_dag_artifact_test_graph() - graph = replace( - graph, + graph = ProjectGraph( + native=graph.native, project=project_with( project=graph.project, effective_target_database="default_db", diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 1b4112bbc6..4f4499d076 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -871,12 +871,6 @@ class ExpectedCountTestCase: expected_count: int -@dataclass(frozen=True) -class ExpectedMessageTestCase: - description: str - expected_message: str - - @dataclass(frozen=True) class LoadTargetRetentionPoliciesTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index bf6107ed94..8c45a1122d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -4,7 +4,6 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( AuthoredSqlSet, AuthoredSqlValueCall, @@ -21,7 +20,6 @@ DeferredModelOutputLocationTestCase, ExpectedBooleanTestCase, ExpectedCountTestCase, - ExpectedMessageTestCase, ModelHeaderColumnLocationTestCase, ModelOutputColumnLocationTestCase, ParseModelSqlErrorTestCase, @@ -69,29 +67,6 @@ def test_given_unique_model_headers_when_discovering_then_header_values_and_loca ) -@pytest.mark.parametrize( - "test_case", - [ - ExpectedMessageTestCase( - description="native worker error remains authoritative", - expected_message="MODEL header worker pool construction failed", - ) - ], - ids=lambda case: case.description, -) -def test_given_native_pool_construction_error_when_preparing_headers_then_error_is_authoritative( - monkeypatch: pytest.MonkeyPatch, - test_case: ExpectedMessageTestCase, -) -> None: - def rejecting_tokenize(_headers: list[str]) -> object: - raise ValueError("MODEL header worker pool construction failed") - - monkeypatch.setattr(native_module, "parse_model_headers", rejecting_tokenize) - - with pytest.raises(ValueError, match=test_case.expected_message): - native_module.prepare_model_header_tokens(["pool_error_marker value"]) - - @pytest.mark.parametrize( "test_case", [ExpectedBooleanTestCase(description="cached maps remain fresh", expected_result=True)], diff --git a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py index 4d55aeb1a8..11e965d1e6 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/helpers.py @@ -9,6 +9,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompileModelConfig, DiscoveredSchemaFile, @@ -22,7 +23,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, ) diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 3337fea7b2..00b3ccd5b2 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -11,6 +11,7 @@ ColumnTransformKind, CompiledLineageColumnFact, CompiledLineageSourceFact, + CompiledResourceType, ModelColumnLineage, ProjectColumnLineage, compiled_model_record, @@ -18,7 +19,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.compiler.lineage._test_types import ( ColumnLineageAnalyzerTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py index 5d68f33411..f25daaf0b7 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/helpers.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompileModelConfig, @@ -43,7 +44,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index 0271fefc1c..2bdbe2429c 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -10,13 +10,13 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, Fingerprint, ) from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py index 85eb362f4b..4a5832323b 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/target_validation/helpers.py @@ -10,10 +10,11 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView class ConservativeCustomAdapter(BaseAdapter): diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py index 1c39fe8ed9..db6c3178e5 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/_test_types.py @@ -35,7 +35,9 @@ class PlanWorkTestCase: @dataclass(frozen=True) class PipelinePhaseTimingTestCase: description: str - clock_values: tuple[float, ...] + compile_seconds: float + connection_seconds: float + rules_seconds: float expected_compile_seconds: float expected_planning_seconds: float expected_total_seconds: float diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py index 4437f1965b..bf24e9cfa0 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/helpers.py @@ -7,6 +7,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSource, DiscoveredSourceFile, MaterializationType, @@ -17,7 +18,9 @@ SourceEntry, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView class RelationTargetTestAdapter(BaseAdapter): @@ -87,3 +90,53 @@ def build_plan_output_with_model(name: str = "orders") -> PlanOutput: model_entries=(entry,), selected_keys=frozenset((entry.key,)), ) + + +class ManualClock: + """A monotonic clock that only moves when a test advances it.""" + + def __init__(self) -> None: + self.now: float = 0.0 + + def __call__(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class SlowConnectionDuckDbAdapter(DuckDbAdapter): + """An in-memory DuckDB adapter whose connections, compile profile and rules dialect take + `connection_seconds`, `compile_seconds` and `rules_seconds` on a manual clock. + + The compile profile takes its time on its first call; the dialect on its second, which the + rules gate makes after the compile's own dialect read.""" + + def __init__( + self, + *, + clock: ManualClock, + connection_seconds: float, + compile_seconds: float = 0.0, + rules_seconds: float = 0.0, + ) -> None: + super().__init__() + self._clock: ManualClock = clock + self._connection_seconds: float = connection_seconds + self._compile_seconds: float = compile_seconds + self._dialect_seconds: list[float] = [0.0, rules_seconds] + + def expression_inference_profile(self) -> ExpressionInferenceProfile: + self._clock.advance(self._compile_seconds) + self._compile_seconds = 0.0 + return super().expression_inference_profile() + + def sql_analysis_dialect(self) -> str | None: + self._clock.advance(sum(self._dialect_seconds[:1])) + del self._dialect_seconds[:1] + return super().sql_analysis_dialect() + + def connect(self, config: dict[str, Any]) -> Any: + del config + self._clock.advance(self._connection_seconds) + return super().connect({}) diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py index 0a41ca55d2..ff3ee0d109 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/main/test_compile_phase_timings.py @@ -4,24 +4,23 @@ import time from pathlib import Path -from unittest.mock import Mock import pytest import sqlbuild._native as _native from sqlbuild._native import ( BuildPhaseTimingTracker, - CompiledProject, CompilePipelineOptions, CompilePipelineResult, PartialBuildPhaseTimings, - PlanOutput, - RulesRunResult, + discover_project_inputs, ) -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.pipeline._helpers import materializations from sqlbuild.runtime.contracts.models import ConnectionHooks from tests.unit.src.sqlbuild.compiler.pipeline.main._test_types import PipelinePhaseTimingTestCase +from tests.unit.src.sqlbuild.compiler.pipeline.main.helpers import ( + ManualClock, + SlowConnectionDuckDbAdapter, +) @pytest.mark.parametrize( @@ -29,14 +28,18 @@ [ PipelinePhaseTimingTestCase( description="slow planning connection is excluded from direct compilation", - clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 13.0, 13.0), + compile_seconds=2.0, + connection_seconds=10.0, + rules_seconds=0.0, expected_compile_seconds=2.0, expected_planning_seconds=10.0, - expected_total_seconds=13.0, + expected_total_seconds=12.0, ), PipelinePhaseTimingTestCase( description="rules wall time is reported and excluded from compile and planning", - clock_values=(0.0, 0.0, 2.0, 2.0, 3.0, 3.0, 3.0, 13.0, 13.0), + compile_seconds=2.0, + connection_seconds=10.0, + rules_seconds=1.0, expected_compile_seconds=2.0, expected_planning_seconds=10.0, expected_total_seconds=13.0, @@ -54,42 +57,12 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - monotonic: Mock = Mock(side_effect=test_case.clock_values) - project: CompiledProjectView = CompiledProject( - run_id="timing-run", - effective_target_name="test", - effective_connection={}, - effective_vars={}, - ) - pipeline_result: CompilePipelineResult = CompilePipelineResult( - project=project, plan_output=PlanOutput() - ) - adapter: Mock = Mock() - monkeypatch.setattr(time, "monotonic", monotonic) - monkeypatch.setattr(_native, "build_compiled_project", Mock(return_value=project)) - monkeypatch.setattr(_native, "build_execution_plan", Mock(return_value=PlanOutput())) - monkeypatch.setattr(materializations, "load_custom_materializations", Mock(return_value={})) - monkeypatch.setattr( - _native, - "build_python_run_plan_outputs", - Mock( - return_value=Mock( - plan_output=pipeline_result.plan_output, - selected_python_node_names=frozenset(), - python_plan_entries=(), - ) - ), - ) - (tmp_path / "sqlbuild_project.toml").write_text('name = "timing"\n', encoding="utf-8") - monkeypatch.setattr(_native, "resolve_configured_rule_selection", Mock(return_value=None)) - monkeypatch.setattr( - _native, - "evaluate_rules", - Mock( - return_value=RulesRunResult(findings=(), evaluated_models=0, built_in_ms=0, custom_ms=0) - ), + clock: ManualClock = ManualClock() + monkeypatch.setattr(time, "monotonic", clock) + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "timing"\nadapter = "duckdb"\n', encoding="utf-8" ) - tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker(monotonic=monotonic) + tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker(monotonic=clock) progress: list[str] = [] progress_hooks: dict[bool, ConnectionHooks] = { True: ConnectionHooks(on_progress=progress.append), @@ -98,8 +71,13 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di with tracker.scope(): result: CompilePipelineResult = _native.run_compile_pipeline( - discovered_inputs=Mock(project_dir=tmp_path), - adapter=adapter, + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=SlowConnectionDuckDbAdapter( + clock=clock, + connection_seconds=test_case.connection_seconds, + compile_seconds=test_case.compile_seconds, + rules_seconds=test_case.rules_seconds, + ), options=CompilePipelineOptions(connection_config={}), hooks=progress_hooks[test_case.report_progress], ) @@ -108,7 +86,7 @@ def test_given_slow_planning_connection_when_compiling_then_direct_phases_are_di assert result.compile_seconds == test_case.expected_compile_seconds assert result.planning_seconds == test_case.expected_planning_seconds assert timings.total_seconds == test_case.expected_total_seconds - assert (result.compile_seconds or 0) + (result.planning_seconds or 0) < ( + assert (result.compile_seconds or 0) + (result.planning_seconds or 0) <= ( timings.total_seconds or 0 ) assert set(test_case.expected_progress_fragments) <= set(progress) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py index b8e04c4b3a..0110eb2454 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_helpers.py @@ -14,6 +14,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, Fingerprint, FunctionArgument, @@ -30,7 +31,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, FunctionLanguage, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 7e8ae4ca2b..f6dd035585 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -24,6 +24,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompiledSqlScenario, @@ -77,7 +78,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSeedView, CompiledSourceView, SqlTestMode, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py index cd0c4d1278..3adc1d2436 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/migrations/test_column_migration_candidates.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DeferralInputs, Fingerprint, @@ -25,7 +26,7 @@ plan_column_migrations, ) from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.planner._helpers.migrations._test_types import ( UnfingerprintableCandidateTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py index aa4ed448c9..84ff684bb7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/helpers.py @@ -9,6 +9,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, LocalConfig, PlannerRelationsContext, @@ -28,7 +29,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py index 101fbd5829..2abbe1a717 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/helpers.py @@ -8,12 +8,13 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, ModelPlanContext, SourceEntry, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView def build_cursor_intrinsic_model(*, config_values: dict[str, object]) -> CompiledModelView: diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py index 28b25f0c39..1c5a9349c5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/resolve/test_refs.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, DbtCompileReferenceResolver, DbtManifestIndex, @@ -19,7 +20,6 @@ ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.resolve._test_types import ( ApplyDeferredTargetsTestCase, RefResolutionTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index cfe2db5d8b..f1d3459bcc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -14,6 +14,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSqlTest, CompileModelConfig, CompileSqlTestCte, @@ -42,7 +43,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index b79cb1a6ec..4f1b63bd88 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSqlTest, CompileSqlTestCte, DiscoveredSqlTestBlock, @@ -39,7 +40,7 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( AssertionChainCteErrorTestCase, NativePlanningDifferentialTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 826bc5e16d..7f41f096e5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -9,6 +9,7 @@ from sqlbuild import _native as fixture_completion from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlTest, CompileSqlReference, PlanWarning, @@ -25,7 +26,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteFixturePlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py index de3d39f92b..69561d341c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_microbatch_limit.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileInputError, CursorBounds, CursorInputRelation, @@ -22,7 +23,6 @@ apply_microbatch_limit, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( ConfiguredMicrobatchPolicyTestCase, EffectiveMicrobatchLimitPlanningTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py index f78652a500..d0f0444e57 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py @@ -2,8 +2,11 @@ import pytest -from sqlbuild._native import CompiledObjectKey, build_downstream_model_name_closure -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import ( + CompiledObjectKey, + CompiledResourceType, + build_downstream_model_name_closure, +) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ModelClosureTestCase SOURCE_ORDERS: CompiledObjectKey = CompiledObjectKey( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py index 878a03936b..2f865bed7e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_plan_entry_cursor_overrides.py @@ -15,6 +15,7 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, CursorBounds, CursorInputRelation, @@ -24,7 +25,7 @@ compute_plan_cursor_bounds, ) from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( AuthoritativeCursorOverrideTestCase, MicrobatchCursorEndPlanTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py index 060193c5c8..08c1098c9c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_graph.py @@ -6,12 +6,12 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlScenario, ScenarioGraphPlan, WarningSeverity, plan_scenario_graph, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( PlanScenarioGraphErrorTestCase, PlanScenarioGraphTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index b727f99bde..00c40992b2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -11,6 +11,7 @@ ChangeKind, CompiledObjectKey, CompiledProject, + CompiledResourceType, DirectModelVersionIdentities, DirectSourceFreshnessPlanningResult, Fingerprint, @@ -30,7 +31,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CompleteModelChangesWorkTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py index 3c1c5c141e..6069d383cb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_columns.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledProject, + CompiledResourceType, CursorInputRelation, PlannerInputError, PlannerRelationsContext, @@ -28,7 +29,6 @@ CompiledFunctionView, CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( CursorRuntimeOwnershipTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index fa112e1b48..c07609435c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -12,6 +12,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DiscoveredLoaderFunction, SourceEntry, @@ -28,7 +29,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py index 9172b003b0..a564cb4f92 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/helpers.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledProject, + CompiledResourceType, CompiledSource, DiscoveredSourceFile, PlannerRelationsContext, @@ -18,7 +19,7 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.types import CompiledResourceType, CompiledSourceView +from sqlbuild.compiler.compile.types import CompiledSourceView def column_names(columns: tuple[ColumnInfo, ...]) -> tuple[str, ...]: diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py index ca26c95c10..a504096ade 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/helpers.py @@ -5,7 +5,7 @@ import importlib.metadata import json import sqlite3 -from collections.abc import Mapping +from collections.abc import Callable from pathlib import Path import pytest @@ -13,7 +13,6 @@ import sqlbuild._native as native_module from sqlbuild._native import ( FixtureColumnInference, - build_migration_fingerprint, infer_fixture_column_facts, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -31,30 +30,6 @@ DATABASE_GLOB: str = "migration-fingerprints-v*/migration-fingerprints.sqlite3" -def count_fingerprint_computations(monkeypatch: pytest.MonkeyPatch) -> list[int]: - """Count every fingerprint computed instead of reused.""" - - computations: list[int] = [] - - def counting( - *, - query_sql: str, - metadata_json: str, - ref_identities: Mapping[str, str], - dialect: str | None, - ) -> str | None: - computations.append(1) - return build_migration_fingerprint( - query_sql=query_sql, - metadata_json=metadata_json, - ref_identities=ref_identities, - dialect=dialect, - ) - - monkeypatch.setattr(native_module, "build_migration_fingerprint", counting) - return computations - - def keep_cache(root: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Leave the persisted cache untouched.""" @@ -129,3 +104,14 @@ def counting( monkeypatch.setattr(native_module, "infer_fixture_column_facts", counting) return inferences + + +class SqlTestsProject: + """A project whose SQL tests are read through a callback each time planning asks for them.""" + + def __init__(self, read_tests: Callable[[], tuple[object, ...]]) -> None: + self._read_tests: Callable[[], tuple[object, ...]] = read_tests + + @property + def sql_tests(self) -> tuple[object, ...]: + return self._read_tests() diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py index 021e1b6932..16eac03704 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_migration_fingerprint_cache.py @@ -3,13 +3,15 @@ from __future__ import annotations import json -from collections.abc import Mapping from pathlib import Path import pytest -import sqlbuild._native as native_module -from sqlbuild._native import MigrationFingerprintCache, build_migration_fingerprint +from sqlbuild._native import ( + MigrationFingerprintCache, + build_migration_fingerprint, + migration_fingerprint_work_info, +) from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( MigrationFingerprintCacheTestCase, PersistedMigrationFingerprintTestCase, @@ -19,7 +21,6 @@ ORDERS_SQL, PLAIN_METADATA, corrupt_database, - count_fingerprint_computations, keep_cache, replace_digests_with_non_ascii, tamper_entries, @@ -84,26 +85,9 @@ ids=lambda case: case.description, ) def test_given_fingerprint_requests_when_cached_then_matches_direct_build_with_minimal_work( - test_case: MigrationFingerprintCacheTestCase, monkeypatch: pytest.MonkeyPatch + test_case: MigrationFingerprintCacheTestCase, ) -> None: - computations: list[int] = [] - - def counting( - *, - query_sql: str, - metadata_json: str, - ref_identities: Mapping[str, str], - dialect: str | None, - ) -> str | None: - computations.append(1) - return build_migration_fingerprint( - query_sql=query_sql, - metadata_json=metadata_json, - ref_identities=ref_identities, - dialect=dialect, - ) - - monkeypatch.setattr(native_module, "build_migration_fingerprint", counting) + migration_fingerprint_work_info(reset=True) cache: MigrationFingerprintCache = MigrationFingerprintCache() results: list[str | None] = [ @@ -126,7 +110,7 @@ def counting( for metadata, identities, dialect in test_case.requests ] assert all(result is not None for result in results) - assert len(computations) == test_case.expected_computations + assert migration_fingerprint_work_info()[0] == test_case.expected_computations @pytest.mark.parametrize( @@ -175,7 +159,7 @@ def test_given_persisted_fingerprints_when_planning_again_then_reuses_only_valid ] first.persist() test_case.between_runs(tmp_path, monkeypatch) - computations: list[int] = count_fingerprint_computations(monkeypatch) + migration_fingerprint_work_info(reset=True) second: MigrationFingerprintCache = MigrationFingerprintCache(root=tmp_path) results: list[str | None] = [ @@ -188,7 +172,7 @@ def test_given_persisted_fingerprints_when_planning_again_then_reuses_only_valid assert results == expected assert all(result is not None for result in results) - assert len(computations) == test_case.expected_second_run_computations + assert migration_fingerprint_work_info()[0] == test_case.expected_second_run_computations if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py index 91f8d27399..6a71ddcb5e 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning.py @@ -4,78 +4,33 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, PlannedSqlTests, PlannerInputError, - PlanWarning, SqlTestPlanning, SqlTestSelection, - WarningSeverity, start_sql_test_planning, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner.classes._test_types import ( BackgroundSqlTestPlanningTestCase, ) +from tests.unit.src.sqlbuild.compiler.planner.classes.helpers import SqlTestsProject _KEYS: frozenset[CompiledObjectKey] = frozenset( {CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders")} ) -_PLANNED: PlannedSqlTests = PlannedSqlTests( - selected_keys=_KEYS, - warnings=( - PlanWarning(model_name="orders", severity=WarningSeverity.WARNING, message="planned"), - ), -) @pytest.mark.parametrize( "test_case", [ BackgroundSqlTestPlanningTestCase( - description="enabled planning returns the background result", + description="enabled planning reads the project's SQL tests once", enabled=True, expected_planner_calls=1, ), - ], - ids=lambda case: case.description, -) -def test_given_enabled_planning_when_joining_then_returns_planned_tests( - test_case: BackgroundSqlTestPlanningTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - calls: list[frozenset[CompiledObjectKey]] = [] - - def plan( - *, - project: object, - adapter: object, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - ) -> PlannedSqlTests: - del project, adapter, sql_test_selection - calls.append(selected_keys) - return _PLANNED - - monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - - planning: SqlTestPlanning = start_sql_test_planning( - sql_test_selection=SqlTestSelection(), - project=object(), # ty: ignore[invalid-argument-type] - adapter=object(), # ty: ignore[invalid-argument-type] - selected_keys=_KEYS, - enabled=test_case.enabled, - ) - result: PlannedSqlTests = planning.result() - - assert result == _PLANNED - assert calls == [_KEYS] * test_case.expected_planner_calls - - -@pytest.mark.parametrize( - "test_case", - [ BackgroundSqlTestPlanningTestCase( description="disabled planning never plans and returns no tests", enabled=False, @@ -84,27 +39,18 @@ def plan( ], ids=lambda case: case.description, ) -def test_given_disabled_planning_when_joining_then_returns_empty_tests( - test_case: BackgroundSqlTestPlanningTestCase, monkeypatch: pytest.MonkeyPatch +def test_given_planning_when_joining_then_returns_planned_tests( + test_case: BackgroundSqlTestPlanningTestCase, ) -> None: - calls: list[frozenset[CompiledObjectKey]] = [] + reads: list[int] = [] - def plan( - *, - project: object, - adapter: object, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - ) -> PlannedSqlTests: - del project, adapter, sql_test_selection - calls.append(selected_keys) - return _PLANNED - - monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) + def read_tests() -> tuple[object, ...]: + reads.append(1) + return () planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), - project=object(), # ty: ignore[invalid-argument-type] + project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, @@ -112,7 +58,7 @@ def plan( result: PlannedSqlTests = planning.result() assert result == PlannedSqlTests(selected_keys=_KEYS) - assert len(calls) == test_case.expected_planner_calls + assert len(reads) == test_case.expected_planner_calls @pytest.mark.parametrize( @@ -128,23 +74,14 @@ def plan( ids=lambda case: case.description, ) def test_given_failing_planning_when_joining_then_raises_original_error( - test_case: BackgroundSqlTestPlanningTestCase, monkeypatch: pytest.MonkeyPatch + test_case: BackgroundSqlTestPlanningTestCase, ) -> None: - def plan( - *, - project: object, - adapter: object, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - ) -> PlannedSqlTests: - del project, adapter, selected_keys, sql_test_selection + def read_tests() -> tuple[object, ...]: raise PlannerInputError(str(test_case.expected_error)) - monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) - planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), - project=object(), # ty: ignore[invalid-argument-type] + project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, @@ -165,28 +102,21 @@ def plan( ids=lambda case: case.description, ) def test_given_running_planning_when_listing_threads_then_no_python_worker_blocks_exit( - test_case: BackgroundSqlTestPlanningTestCase, monkeypatch: pytest.MonkeyPatch + test_case: BackgroundSqlTestPlanningTestCase, ) -> None: started: threading.Event = threading.Event() release: threading.Event = threading.Event() + reads: list[int] = [] - def plan( - *, - project: object, - adapter: object, - selected_keys: frozenset[CompiledObjectKey], - sql_test_selection: SqlTestSelection, - ) -> PlannedSqlTests: - del project, adapter, sql_test_selection + def read_tests() -> tuple[object, ...]: + reads.append(1) started.set() release.wait(timeout=5) - return PlannedSqlTests(selected_keys=selected_keys) - - monkeypatch.setattr(native_module, "plan_selected_sql_tests", plan) + return () planning: SqlTestPlanning = start_sql_test_planning( sql_test_selection=SqlTestSelection(), - project=object(), # ty: ignore[invalid-argument-type] + project=SqlTestsProject(read_tests), # ty: ignore[invalid-argument-type] adapter=object(), # ty: ignore[invalid-argument-type] selected_keys=_KEYS, enabled=test_case.enabled, @@ -199,4 +129,4 @@ def plan( planning.result() assert daemon_by_name.get("sqlbuild-test-planning", True) is True - assert test_case.expected_planner_calls == 1 + assert len(reads) == test_case.expected_planner_calls diff --git a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py index 463fa7cd55..f1b6418b8c 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py +++ b/tests/unit/src/sqlbuild/compiler/planner/classes/test_sql_test_planning_exit.py @@ -10,10 +10,12 @@ _SLOW_TEST_PLANNING: str = dedent( """ import sys + import threading import time import types import sqlbuild._native as native + from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter class SlowFinalizer: @@ -25,15 +27,16 @@ def __del__(self): holder = types.ModuleType("holder") holder.finalizer = SlowFinalizer() sys.modules["holder"] = holder - plan_tests = native.plan_selected_sql_tests + set_difference_operator = DuckDbAdapter.render_set_difference_operator - def slow_plan_selected_sql_tests(**arguments): - time.sleep(1.0) - return plan_tests(**arguments) + def slow_set_difference_operator(self): + if threading.current_thread() is not threading.main_thread(): + time.sleep(1.0) + return set_difference_operator(self) - native.plan_selected_sql_tests = slow_plan_selected_sql_tests + DuckDbAdapter.render_set_difference_operator = slow_set_difference_operator """ ) _FAILED_BUILD: str = dedent( @@ -47,8 +50,16 @@ def slow_plan_selected_sql_tests(**arguments): """ from sqlbuild._native import SqlTestSelection + + class SlowSqlTestsProject: + @property + def sql_tests(self): + time.sleep(1.0) + return () + + native.start_sql_test_planning( - project=None, + project=SlowSqlTestsProject(), adapter=None, selected_keys=frozenset(), sql_test_selection=SqlTestSelection(), @@ -99,6 +110,12 @@ def test_given_sql_test_planning_running_when_interpreter_finalizes_then_process 'SELECT * FROM __source("raw_orders")\n', encoding="utf-8", ) + (project_dir / "tests" / "unit").mkdir(parents=True) + (project_dir / "tests" / "unit" / "test_orders.sql").write_text( + "TEST ();\n\nWITH\n__source__raw_orders AS (\n SELECT 21 AS order_id\n),\n" + "__expected__orders AS (\n SELECT 21 AS order_id\n)\nSELECT 1\n", + encoding="utf-8", + ) script: Path = tmp_path / "planning_exit.py" script.write_text(test_case.script, encoding="utf-8") diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index 0b5ca170fb..f0a04e69af 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -11,6 +11,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, ModelPlanEntry, PlanAction, @@ -19,7 +20,7 @@ build_display_only_sqlbuild_plan, format_plan, ) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledModelView from tests.unit.src.sqlbuild.compiler.planner.main.execution._test_types import ( DisplayFullRefreshTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py index 1af69db657..e68966d93b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/helpers.py @@ -10,6 +10,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DeferralInputs, Fingerprint, @@ -30,7 +31,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 41b5526012..cc92e03d8a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -8,6 +8,7 @@ CompiledModel, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, DirectModelVersionIdentities, FunctionArgument, @@ -23,7 +24,6 @@ ) from sqlbuild.compiler.compile.types import ( CompiledModelView, - CompiledResourceType, FunctionLanguage, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py index 3cc6ca8e30..71ebdbb10b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/test_version_identity_function_hashes.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( CompiledFunction, CompiledObjectKey, + CompiledResourceType, DirectModelVersionIdentities, FunctionReturnColumn, build_function_local_hashes, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import build_compiled_function from tests.unit.src.sqlbuild.compiler.planner.main.identity._test_types import ( FunctionReturnContractIdentityTestCase, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index fb2b8b4c07..22a14412f2 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -15,6 +15,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSource, CompileModelConfig, DiscoveredAssetFunction, @@ -38,7 +39,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, CompiledSourceView, ) from sqlbuild.providers import Provider diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py index 4b7d565d33..09196671c4 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_lifecycle.py @@ -6,12 +6,12 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, PythonNodeGraph, PythonSqlRunLifecyclePlan, PythonSqlRunSelection, build_python_sql_run_lifecycle, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from tests.unit.src.sqlbuild.compiler.python_nodes._helpers._test_types import ( PythonSqlRunLifecycleTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py index 10ff692c35..cb789cef89 100644 --- a/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/helpers.py @@ -8,6 +8,7 @@ from sqlbuild._native import ( SOURCE_FRESHNESS_TABLE_NAME, CompiledObjectKey, + CompiledResourceType, SourceEntry, SourceFreshnessConfig, SourceFreshnessIdentity, @@ -27,7 +28,6 @@ from sqlbuild.adapter.contract.types import FrameworkType from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType def state_table_exists_map( diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py index 9820c3f510..7db523e4b5 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/helpers.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( AuditPlanEntry, CompiledObjectKey, + CompiledResourceType, MeasurementThresholdBound, MeasurementThresholds, ) @@ -20,7 +21,6 @@ AuditSeverity, ThresholdOperator, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.main._execute_reporting_errors import ( execute_audit_reporting_errors, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index c5144c16ce..2e2800504b 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -9,6 +9,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredLoaderFunction, MaterializationType, ModelPlanEntry, @@ -29,7 +30,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py index d50712544f..7c3c246bf8 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py @@ -8,13 +8,13 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, DiscoveredLoaderFunction, LoaderConnectionMode, PlanOutput, SourceEntry, SourceLoadPlanEntry, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build._helpers.external_source_loads import ( run_external_source_loads_before_connections, ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py index fb682d78a3..be9bbf98d6 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_indexes.py @@ -2,13 +2,13 @@ import pytest -from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, PlanOutput +from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, CompiledResourceType, PlanOutput from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.executor.build._helpers.indexes import build_execution_indexes from sqlbuild.executor.build.models import BuildIndexes from tests.unit.src.sqlbuild.executor.build._helpers._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py index 45fa8be653..a573ae34f9 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_scheduler.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, DiscoveredHookFunction, DiscoveredLoaderFunction, ModelPlanEntry, @@ -25,7 +26,6 @@ ) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build.main._execute import execute_build_plan from sqlbuild.executor.build.models import ( BuildCallbacks, diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py index 669a32dc4c..1d1d63143e 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_source_node.py @@ -6,9 +6,8 @@ import pytest -from sqlbuild._native import CompiledObjectKey, PlanOutput +from sqlbuild._native import CompiledObjectKey, CompiledResourceType, PlanOutput from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.build._helpers.source_node import execute_build_source_node from sqlbuild.executor.build.models import BuildCallbacks, BuildRuntimeParams from sqlbuild.executor.load.models import LoadExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py index dae1f86f97..442de8eb58 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_diagnostics.py @@ -7,8 +7,7 @@ import pytest -from sqlbuild._native import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledObjectKey, CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import BuildCallbacks, SchedulerState from tests.unit.src.sqlbuild.executor.build.classes._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py index c2c9626b84..573cdf857d 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_identity.py @@ -9,8 +9,7 @@ import pytest -from sqlbuild._native import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledObjectKey, CompiledResourceType from sqlbuild.executor.build.classes.build_scheduler import BuildScheduler from sqlbuild.executor.build.models import NodeCompletion from sqlbuild.executor.run.models import ModelExecutionResult diff --git a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py index fdd2c9506c..c1832b4d61 100644 --- a/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py +++ b/tests/unit/src/sqlbuild/executor/build/classes/test_build_scheduler_test_order.py @@ -4,8 +4,7 @@ import pytest -from sqlbuild._native import CompiledObjectKey, PlanOutput, SqlTestPlanEntry -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild._native import CompiledObjectKey, CompiledResourceType, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.build.classes.build_scheduler import _test_result_authored_order from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.executor.testing.types import SqlTestOutcome diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py index 18a2a6aa22..bd12b224da 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/fingerprinting/helpers.py @@ -9,6 +9,7 @@ FINGERPRINT_TABLE_NAME, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, Fingerprint, FingerprintSet, MaterializationType, @@ -19,7 +20,6 @@ SeedPlanEntry, ) from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.types import CompiledResourceType class CloneFingerprintAdapter: diff --git a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py index cf53cd8e3b..5785b253de 100644 --- a/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/clone/_helpers/helpers.py @@ -8,6 +8,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, FunctionPlanEntry, MaterializationType, @@ -22,7 +23,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index ff95e704a1..0adb02c07b 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -16,6 +16,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSeed, CompiledSource, CompileModelConfig, @@ -36,7 +37,6 @@ from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.types import ( CompiledProjectView, - CompiledResourceType, CompiledSourceView, ) from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 1ed9f9368a..4b292bd609 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -13,6 +13,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompiledSqlScenario, CompilePipelineResult, DiscoveredSqlScenarioFile, @@ -34,7 +35,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.scenario.constants import SCENARIO_LOCAL_JSONL_INVALID diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py index 09d1b648da..5c02f6ec76 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_auditing.py @@ -22,6 +22,7 @@ AuditPlanEntry, AuditProgressReporter, CompiledObjectKey, + CompiledResourceType, CostContext, PlanOutput, StatementLedgerEntry, @@ -37,7 +38,7 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, CompiledResourceType +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.cost.models import CostResourceContext from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers import auditing diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py index f3eb21e27d..f4521e26ec 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_testing.py @@ -11,11 +11,11 @@ ChainStep, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, PlanOutput, SqlTestPlanEntry, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.pipeline._helpers import testing as testing_pipeline from sqlbuild.executor.pipeline._helpers.testing import run_test_pipeline from sqlbuild.executor.pipeline.models import TestPipelineCallbacks as PipelineCallbacks diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py index aa1c4745d4..e0bf2cfe63 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/helpers.py @@ -6,6 +6,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, FunctionPlanEntry, MaterializationType, ModelPlanEntry, @@ -19,7 +20,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage +from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.runtime.contracts.types import ExecutionResourceKind from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py index 1118c704ba..05e84836d5 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/main/test_schema_preflight.py @@ -7,13 +7,13 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, PlanOutput, PlanReason, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.pipeline._helpers.graph_width import runnable_graph_width from sqlbuild.executor.pipeline._helpers.schema_preparation import prepare_build_schemas from tests.unit.src.sqlbuild.executor.pipeline.main._test_types import ( diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index 37a09440f3..e12691760d 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -6,6 +6,7 @@ AuditPlanEntry, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, DiscoveredHookFunction, MaterializationType, ModelPlanEntry, @@ -28,7 +29,6 @@ AuditRunScope, AuditSeverity, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks from sqlbuild.executor.run.models import HookContext, HookRelationLookup, HookRunContext diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py index 34da88d5e4..a597ba1243 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_cursor_bounds.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CursorBounds, CursorGrain, CursorInputRelation, @@ -31,7 +32,6 @@ from sqlbuild._native import cursor_sentinel_to_token as sentinel_to_token from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py index d9c4caf2ae..306e9ae570 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_snapshot_adapter_boundary.py @@ -11,6 +11,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, HistoricalInput, MaterializationType, ModelPlanEntry, @@ -18,7 +19,6 @@ PlanReason, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.run._helpers.materializations.snapshot import execute_snapshot_entry from sqlbuild.executor.run.models import ModelExecutionResult, ModelMaterializationContext from sqlbuild.executor.scheduling.types import ExecutionStatus diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py index 8d88deaef8..8dbd2bc5f1 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/helpers.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, @@ -21,7 +22,6 @@ ScenarioRelationPlan, SeedPlanEntry, ) -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.executor.scenario.models import ( ScenarioSnapshotCaptureRunResult, ScenarioSnapshotColumn, diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py index 16e0263698..a97f517c5a 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/helpers.py @@ -7,6 +7,7 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, MaterializationType, ModelPlanEntry, PlanAction, @@ -23,7 +24,6 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import ColumnInfo, QueryResult, RelationInfo -from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS diff --git a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py index 1f2a173ddf..29a01a85a1 100644 --- a/tests/unit/src/sqlbuild/executor/seed/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/seed/main/helpers.py @@ -7,11 +7,11 @@ from sqlbuild._native import ( CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, SeedCsvSettings, SeedPlanEntry, ) from sqlbuild.adapter.contract.models import ColumnInfo -from sqlbuild.compiler.compile.types import CompiledResourceType def build_seed_plan_entry(*, seed_name: str, file_path: Path) -> SeedPlanEntry: diff --git a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py index 2dc0788dde..0fb1a43388 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/helpers.py @@ -1,10 +1,15 @@ -from sqlbuild._native import ChainStep, CompiledObjectKey, SqlTestAssertionStep, SqlTestPlanEntry +from sqlbuild._native import ( + ChainStep, + CompiledObjectKey, + CompiledResourceType, + SqlTestAssertionStep, + SqlTestPlanEntry, +) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.compile.types import CompiledResourceType def build_comparison_test_entry(*, sql_analysis_enabled: bool = True) -> SqlTestPlanEntry: diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index 9d88c15421..984791a758 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -13,6 +13,7 @@ BackfillResult, CompiledObjectKey, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, CompileModelInput, CompileProjectInputs, @@ -39,7 +40,7 @@ extract_sql_references, transitive_closure, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledProjectView _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index 3ef0cd87cf..364dda5bf2 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -549,10 +549,9 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select unselected_file.parent.mkdir() unselected_file.write_text(fixture_sql, encoding="utf-8") native_format: Mock = Mock(wraps=native_module.format_lint_sql_bodies) - full_discovery: Mock = Mock(side_effect=AssertionError("full discovery must not run")) project_graph: Mock = Mock(side_effect=AssertionError("project graph must not build")) monkeypatch.setattr(native_module, "format_lint_sql_bodies", native_format) - monkeypatch.setattr(native_module, "discover_project_inputs", full_discovery) + _ = native_module.compile_stage_work_info(reset=True) monkeypatch.setattr(native_module, "build_project_graph", project_graph) exit_code: int = main( @@ -570,7 +569,7 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select assert f"{test_case.selected_file_count} files checked" in captured.err assert native_format.call_count == test_case.expected_native_batch_count assert len(native_format.call_args.args[0]) == test_case.expected_native_file_count - assert full_discovery.call_count == 0 + assert native_module.compile_stage_work_info().get("discover_project_inputs", 0) == 0 assert project_graph.call_count == 0 assert all( "CAST(NULL AS VARCHAR)" not in path.read_text(encoding="utf-8") for path in selected_files diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index d9f90d1c8a..557d46f66a 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -2,14 +2,12 @@ from __future__ import annotations -from collections.abc import Callable from pathlib import Path import pytest import sqlbuild._native as native_module from sqlbuild._native import ( - DiscoveredProjectInputs, LintBody, LintConfig, LintRunResult, @@ -44,30 +42,16 @@ def test_given_lint_context_when_building_then_project_is_discovered_once( test_case: LintBehaviorTestCase, tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") - discovery_count: list[int] = [0] - discover: Callable[..., DiscoveredProjectInputs] = native_module.discover_project_inputs - - def counted_discovery( - *, - project_dir: Path, - sql_analysis_enabled_override: bool | None = None, - extract_output_column_locations: bool = True, - ) -> DiscoveredProjectInputs: - discovery_count[0] += 1 - return discover( - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - ) - - monkeypatch.setattr(native_module, "discover_project_inputs", counted_discovery) + _ = native_module.compile_stage_work_info(reset=True) _ = build_lint_expansion_context(project_dir=tmp_path) - assert discovery_count[0] == test_case.expected_value + assert ( + native_module.compile_stage_work_info()["discover_project_inputs"] + == test_case.expected_value + ) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index 0b72c88ed7..802272118d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -15,6 +15,7 @@ CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, CompiledObjectKey, + CompiledResourceType, CompiledSqlTest, CompiledSqlTestResource, CompileSqlTestCte, @@ -37,7 +38,7 @@ ) from sqlbuild._native import evaluate_custom_rules as evaluate_custom_rules_cached from sqlbuild._native import evaluate_project_rules as evaluate -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import CustomRuleTestCase from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index 92cac5b2f2..e6852906cd 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -5,12 +5,13 @@ from sqlbuild._native import ( CompiledObjectKey, + CompiledResourceType, CompiledSqlScenario, CompiledSqlTest, DiscoveredSqlScenarioFile, project_with, ) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode +from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( NativeFactPayloadTestCase, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py index d3ce2e7515..ffa490ec3f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/helpers.py @@ -3,10 +3,11 @@ from sqlbuild._native import ( CompiledAudit, CompiledObjectKey, + CompiledResourceType, DiscoveredAuditBlock, DiscoveredAuditFile, ) -from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledResourceType +from sqlbuild.compiler.compile.types import CompiledAuditView def build_audit(*, name: str, attached_target_name: str | None) -> CompiledAuditView: diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py index 4c888d539f..b2ae63b01c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/helpers.py @@ -7,6 +7,7 @@ CompiledObjectKey, CompiledProject, CompiledRelationLocation, + CompiledResourceType, CompileModelConfig, CompileSqlReference, EnumDeclaration, @@ -15,7 +16,6 @@ from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, - CompiledResourceType, ) _ENUM_DECLARATION: EnumDeclaration = EnumDeclaration(