diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs index ced6725397..f98885a529 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use crate::compiler::_helpers::sql_tests::authored_macro_calls::calls_macro; @@ -15,6 +16,10 @@ use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_p use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed}; +/// Batches smaller than this are extracted on the calling thread. +const PARALLEL_EXTRACTION_MINIMUM: usize = 64; +const EXTRACTION_WORKERS: usize = 4; +const EXTRACTION_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; const MODEL_PREFIXES: [&str; 8] = [ "__macro__", "__ref__", @@ -134,10 +139,29 @@ pub(crate) fn extract_tests( tests: &[SqlTestRequest], syntax: &LexicalSyntax, ) -> Result, (usize, Failure)> { - tests - .iter() + let extracted: Vec> = + if tests.len() < PARALLEL_EXTRACTION_MINIMUM { + tests + .iter() + .map(|test| extract_test(test, syntax)) + .collect() + } else { + let pool: rayon::ThreadPool = rayon::ThreadPoolBuilder::new() + .num_threads(EXTRACTION_WORKERS) + .stack_size(EXTRACTION_WORKER_STACK_BYTES) + .build() + .map_err(|error| (0, Failure::from(error.to_string())))?; + pool.install(|| { + tests + .par_iter() + .map(|test| extract_test(test, syntax)) + .collect() + }) + }; + extracted + .into_iter() .enumerate() - .map(|(index, test)| extract_test(test, syntax).map_err(|failure| (index, failure))) + .map(|(index, result)| result.map_err(|failure| (index, failure))) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index b6d43b56f8..c15924e2b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -13,6 +13,9 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_s use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + AttachedSchema, attached_schema, +}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -299,9 +302,11 @@ fn model_rows<'py>( row.get_item(1)?.extract()?, )); } - let entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; + let schema: Option> = attached_schema(model_input)?; let recovers: bool = get("contract")?.eq(&enforced)? - || (!entry.is_none() && entry.getattr("type_enforcement")?.is_truthy()?); + || schema + .as_ref() + .is_some_and(|schema| schema.type_enforcement); let snapshot_columns: Option<(String, String)> = if get("materialized")?.eq(&snapshot)? { Some(( or_default(&get("valid_from_column")?, VALID_FROM_DEFAULT)?, @@ -310,10 +315,9 @@ fn model_rows<'py>( } else { None }; - let families = if entry.is_none() { - Vec::new() - } else { - family_rows(&entry.getattr("dynamic_columns")?)? + let families = match &schema { + Some(schema) => family_rows(&schema.dynamic_columns)?, + None => Vec::new(), }; let row: Vec> = vec![ model_input diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs index cd764d371c..fc6ded71a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs @@ -8,11 +8,11 @@ use sqlbuild_analysis::auditing::main::evaluate_measurement::evaluate_measuremen use sqlbuild_analysis::auditing::models::{MeasurementOutcome, Thresholds}; use crate::bindings::_helpers::auditing::identity::thresholds as read_thresholds; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; - fn measurement_error(py: Python<'_>, message: &str) -> PyErr { match registry::error(py, ErrorId::MeasurementAudit).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), @@ -40,9 +40,7 @@ fn evaluate_measurement<'py>( MeasurementOutcome::Error => "ERROR", MeasurementOutcome::Insufficient => "INSUFFICIENT", }; - py.import(AUDITING_TYPES_MODULE)? - .getattr("AuditOutcome")? - .getattr(member) + host::entry(py, FacadeHost::AuditOutcome)?.getattr(member) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs index 9ecb1ae789..727dc3ce7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs @@ -1,12 +1,13 @@ //! Project variable values rendered as the text SQL and config interpolation inserts. +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTypeMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -const ERRORS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; const JSON_MODULE: &str = "json"; /// Characters of a structured value's JSON preview before it is cut short. const PREVIEW_LIMIT: usize = 120; @@ -14,11 +15,7 @@ const PREVIEW_ELLIPSIS: &str = "..."; /// `SharedInputError(message)`. fn shared_input_error(py: Python<'_>, message: String) -> PyErr { - match py - .import(ERRORS_MODULE) - .and_then(|module| module.getattr("SharedInputError")) - .and_then(|class| class.call1((message,))) - { + match host::entry(py, FacadeHost::SharedInputError).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), Err(error) => error, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs index 55bdbb7ed1..ddaa1338cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs @@ -13,11 +13,10 @@ use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -const ERRORS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; -const SHARED_INPUT_ERROR: &str = "SharedInputError"; const RANDOM_MODULE: &str = "random"; const MAX_ATTEMPTS_TOO_LOW: &str = "RetryPolicy max_attempts must be at least 1"; const INITIAL_DELAY_NEGATIVE: &str = "RetryPolicy initial_delay_seconds must be non-negative"; @@ -36,8 +35,7 @@ const HOOK_READS_INVALID: &str = /// `SharedInputError(message)`. fn shared_input_error(py: Python<'_>, message: &str) -> PyErr { - match imported(py, ERRORS_MODULE, SHARED_INPUT_ERROR).and_then(|class| class.call1((message,))) - { + match host::entry(py, FacadeHost::SharedInputError).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), Err(error) => error, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs index fd1f70a8ff..39430e2020 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs @@ -5,6 +5,7 @@ use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -84,8 +85,8 @@ pub(crate) fn resolve_cursor_input_roles<'py>( Err(_) => { let cursor: Bound<'py, PyAny> = values.call_method1("get", ("cursor",))?; if cursor.is_instance_of::() { - for reference in model.getattr("references")?.try_iter()? { - filter.set_item(reference?.getattr("ref_name")?, &cursor)?; + for (_, name, _, _) in input_reference_rows(model)? { + filter.set_item(name, &cursor)?; } } if !rolling { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 96063ddc63..2f0c7060a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; use crate::bindings::_helpers::attachments::cursor_intrinsics::reserved_cursor_markers; @@ -11,9 +11,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; -use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; use crate::bindings::_helpers::native_host::host; @@ -51,31 +55,15 @@ fn opt_out_help(py: Python<'_>, model_owned: bool) -> PyResult> Ok(PyString::new(py, &sql_analysis_opt_out_help(model_owned)).into_any()) } -/// The host the loop calls back into: `constant_text(*, declaration, file_path)`. -fn loop_host<'py>(context: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = context.py(); - let context: Py = context.clone().unbind(); - let constant_text = move |arguments: &Bound<'_, PyTuple>, - keywords: Option<&Bound<'_, PyDict>>| - -> PyResult> { - let py: Python<'_> = arguments.py(); - let context: &Bound<'_, PyAny> = context.bind(py); - let passed: Bound<'_, PyDict> = - keywords.map_or_else(|| PyDict::new(py), |keywords| keywords.clone()); - let file_path: Bound<'_, PyAny> = passed.as_any().get_item("file_path")?; - let rendered: String = constant_declaration_sql( - &passed.as_any().get_item("declaration")?, - &context.getattr("value_renderer")?, - (&context.getattr("collection_rendering")?, Some(&file_path)), - )?; - Ok(PyString::new(py, &rendered).into_any().unbind()) - }; - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item( - "constant_text", - PyCFunction::new_closure(py, None, None, constant_text)?, - )?; - imported(py, "types", "SimpleNamespace")?.call((), Some(&keywords)) +/// The adapter's constant rendering every model, source and hook expansion shares. +fn constant_rendering<'py>(context: &Bound<'py, PyAny>) -> PyResult> { + Bound::new( + context.py(), + ConstantRendering::new( + context.getattr("value_renderer")?.unbind(), + context.getattr("collection_rendering")?.unbind(), + ), + ) } fn macro_inputs<'py>(context: &Bound<'py, PyAny>) -> PyResult> { @@ -283,14 +271,16 @@ pub(crate) fn build_model_inputs<'py>( project_config_path.str()?, ), ))?; - let files: Bound<'py, PyList> = PyList::new( - py, - discovered_inputs - .getattr("model_files")? - .try_iter()? - .collect::>>()?, + let files: Vec> = discovered_inputs + .getattr("model_files")? + .try_iter()? + .map(|file| file?.extract()) + .collect::>()?; + let inputs: Vec> = NativeModelLoop::run( + native_loop.cast::()?, + native_project.cast::()?.clone(), + constant_rendering(context)?, + files, )?; - let inputs: Bound<'py, PyAny> = - native_loop.call_method1("run", (native_project, loop_host(context)?, files))?; - PyTuple::new(py, inputs.try_iter()?.collect::>>()?) + PyTuple::new(py, inputs) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index e2588c9eea..ca31f38e93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -1,14 +1,19 @@ //! The SQL expansion inputs lint uses to expand authored SQL outside a compile. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use crate::bindings::_helpers::compile_attachment::declaration_scope::rebind_declaration_scope; +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_vars; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; /// Build the declaration scope of `discovered`; a static scope lends only its index to macros. fn declaration_scope<'py>( @@ -20,14 +25,9 @@ fn declaration_scope<'py>( let loaded_macros: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadProjectMacros)? .call1((discovered.getattr("macro_files")?,))?; if !static_scope.is_none() { - let rebound: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::RebindDeclarationScope)?, - &[ - ("scope", static_scope), - ("discovered_inputs", discovered.clone()), - ("loaded_macros", loaded_macros.clone()), - ], - )?; + let rebound: Bound<'py, PyAny> = + rebind_declaration_scope(&static_scope, &discovered.clone(), &loaded_macros.clone()) + .and_then(|value| value.into_bound_py_any(py))?; if !rebound.is_none() { return Ok(rebound); } @@ -74,87 +74,131 @@ fn local_declarations<'py>(discovered: &Bound<'py, PyAny>) -> PyResult( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_sql_expansion_context", - options, - ( - &["project_dir", "value_renderer"], +/// The inputs authored SQL is expanded with outside a compile; the handles carry user values. +pub(crate) struct SqlExpansionInputs { + pub(crate) effective_vars: Py, + pub(crate) loaded_macros: Py, + pub(crate) macro_context: Py, + pub(crate) enums: Py, + pub(crate) constants: Py, + pub(crate) value_renderer: Py, + pub(crate) collection_rendering: Py, + /// Each model file's own declarations, by file path. + pub(crate) local_declarations: Py, + pub(crate) declaration_resolver: Py, +} + +impl SqlExpansionInputs { + /// The inputs a `SqlExpansionContext` record holds. + pub(crate) fn from_record(record: &Bound<'_, PyAny>) -> PyResult { + let field = |name: &str| -> PyResult> { Ok(record.getattr(name)?.unbind()) }; + Ok(Self { + effective_vars: field("effective_vars")?, + loaded_macros: field("loaded_macros")?, + macro_context: field("macro_context")?, + enums: field("enums")?, + constants: field("constants")?, + value_renderer: field("value_renderer")?, + collection_rendering: field("collection_rendering")?, + local_declarations: record + .getattr("local_declarations")? + .downcast_into::()? + .unbind(), + declaration_resolver: field("declaration_resolver")?, + }) + } + + /// These inputs as a `SqlExpansionContext` record. + pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { + call_with( + ®istry::class(py, RecordId::SqlExpansionContext)?, &[ - "cli_vars", - "discovered_inputs", - "declaration_scope", - "static_declaration_scope", + ("effective_vars", self.effective_vars.bind(py).clone()), + ("loaded_macros", self.loaded_macros.bind(py).clone()), + ("macro_context", self.macro_context.bind(py).clone()), + ("enums", self.enums.bind(py).clone()), + ("constants", self.constants.bind(py).clone()), + ("value_renderer", self.value_renderer.bind(py).clone()), + ( + "collection_rendering", + self.collection_rendering.bind(py).clone(), + ), + ( + "local_declarations", + self.local_declarations.bind(py).clone().into_any(), + ), + ( + "declaration_resolver", + self.declaration_resolver.bind(py).clone(), + ), ], - ), - )?; - let value_renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; - let mut discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - if discovered.is_none() { - discovered = call_with( + ) + } +} + +/// Where SQL expansion inputs come from: the project, its discovery and the scope to reuse. +pub(crate) struct ExpansionSources<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) value_renderer: Bound<'py, PyAny>, + pub(crate) cli_vars: Option>, + pub(crate) discovered_inputs: Option>, + pub(crate) declaration_scope: Option>, + pub(crate) static_declaration_scope: Option>, +} + +/// Assemble SQL expansion inputs; a static scope lends only its index to private macros. +pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult { + let py: Python<'_> = sources.project_dir.py(); + let value_renderer = sources.value_renderer; + let discovered = match sources.discovered_inputs { + Some(discovered) => discovered, + None => call_with( ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, &[ - ("project_dir", arguments.value("project_dir")?), + ("project_dir", sources.project_dir.clone()), ( "sql_analysis_enabled_override", PyBool::new(py, false).to_owned().into_any(), ), ], - )?; - } - let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; - let mut cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; - if cli_vars.is_none() { - cli_vars = PyDict::new(py).into_any(); - } - let effective_vars: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildEffectiveVars)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("target_config", py.None().into_bound(py)), - ("cli_vars", cli_vars), - ], - )?; - let mut scope: Bound<'py, PyAny> = arguments.value("declaration_scope")?; - if scope.is_none() { - scope = declaration_scope( + )?, + }; + let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; + let cli_vars: Bound<'_, PyAny> = sources + .cli_vars + .unwrap_or_else(|| PyDict::new(py).into_any()); + let effective_vars: Bound<'_, PyAny> = build_effective_vars( + &project_config.clone(), + &local_config.clone(), + &py.None().into_bound(py), + &cli_vars, + ) + .and_then(|value| value.into_bound_py_any(py))?; + let scope: Bound<'_, PyAny> = match sources.declaration_scope { + Some(scope) => scope, + None => declaration_scope( &discovered, - arguments.value("static_declaration_scope")?, + sources + .static_declaration_scope + .unwrap_or_else(|| py.None().into_bound(py)), value_renderer.getattr("sql_lexical_syntax")?, - )?; - } - let (enums, constants): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + )?, + }; + let (enums, constants): (Bound<'_, PyAny>, Bound<'_, PyAny>) = registry::native(py, NativeAttr::PublicDeclarationIndexes)? .call1((&discovered,))? .extract()?; - let collection_rendering: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveCollectionRendering)?, - &[ - ("project_config", project_config.clone()), - ("declaration_override", py.None().into_bound(py)), - ], - )?; - let macro_context: Bound<'py, PyAny> = call_with( + let collection_rendering: Bound<'_, PyAny> = + resolve_effective_collection_rendering(&project_config.clone(), &py.None().into_bound(py)) + .and_then(|value| value.into_bound_py_any(py))?; + let macro_context: Bound<'_, PyAny> = call_with( ®istry::class(py, RecordId::MacroContext)?, &[ ( "adapter_name", - call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config), - ], - )?, + resolve_effective_adapter_name(&project_config.clone(), &local_config) + .and_then(|value| value.into_bound_py_any(py))?, ), ( "sql_analysis_enabled", @@ -172,21 +216,52 @@ pub(crate) fn build_sql_expansion_context<'py>( ), ], )?; - call_with( - ®istry::class(py, RecordId::SqlExpansionContext)?, - &[ - ("effective_vars", effective_vars), - ("loaded_macros", scope.getattr("loaded_macros")?), - ("macro_context", macro_context), - ("enums", enums), - ("constants", constants), - ("value_renderer", value_renderer), - ("collection_rendering", collection_rendering), - ( - "local_declarations", - local_declarations(&discovered)?.into_any(), - ), - ("declaration_resolver", scope.getattr("resolver")?), - ], - ) + Ok(SqlExpansionInputs { + effective_vars: effective_vars.unbind(), + loaded_macros: scope.getattr("loaded_macros")?.unbind(), + macro_context: macro_context.unbind(), + enums: enums.unbind(), + constants: constants.unbind(), + value_renderer: value_renderer.unbind(), + collection_rendering: collection_rendering.unbind(), + local_declarations: local_declarations(&discovered)?.unbind(), + declaration_resolver: scope.getattr("resolver")?.unbind(), + }) +} + +/// `value`, or `None` when it is Python `None`. +pub(crate) fn present(value: Bound<'_, PyAny>) -> Option> { + (!value.is_none()).then_some(value) +} + +/// Assemble SQL expansion inputs; a static scope lends only its index to private macros. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_sql_expansion_context<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_sql_expansion_context", + options, + ( + &["project_dir", "value_renderer"], + &[ + "cli_vars", + "discovered_inputs", + "declaration_scope", + "static_declaration_scope", + ], + ), + )?; + sql_expansion_inputs(ExpansionSources { + project_dir: arguments.value("project_dir")?, + value_renderer: arguments.value("value_renderer")?, + cli_vars: present(arguments.value("cli_vars")?), + discovered_inputs: present(arguments.value("discovered_inputs")?), + declaration_scope: present(arguments.value("declaration_scope")?), + static_declaration_scope: present(arguments.value("static_declaration_scope")?), + })? + .record(py) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 3100046009..21c49a298a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -8,16 +8,15 @@ use crate::bindings::_helpers::compile_attachment::effective_config::project_var use crate::bindings::_helpers::compile_command::constants::{ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, RETIRED_CACHE_DIRECTORY_NAMES, }; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::frontier::constants::{ COMPILER_CACHE_DIRECTORY_NAME, RETIRED_CACHE_NAMESPACE_SUFFIXES, }; use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; @@ -30,25 +29,19 @@ fn resolved_target<'py>( let py: Python<'py> = discovered.py(); let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; - let target_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("selected_target", selected_target.clone()), - ], - )?; + let target_name: Bound<'py, PyAny> = resolve_target_name( + py, + &project_config.clone(), + &local_config.clone(), + &selected_target.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; if target_name.is_none() { return Ok((target_name, py.None().into_bound(py))); } - let target_config: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetConfig)?, - &[ - ("project_config", project_config), - ("local_config", local_config), - ("target_name", target_name.clone()), - ], - )?; + let target_config: Bound<'py, PyAny> = + resolve_target_config(py, &project_config, &local_config, &target_name.clone()) + .and_then(|value| value.into_bound_py_any(py))?; Ok((target_name, target_config)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 8c038f3b60..cb6fbaf5bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -131,8 +131,10 @@ pub(crate) fn analyze<'py>( let selection: &Bound<'py, PyAny> = &options.analysis_selection; let enabled: bool = !(selection.getattr("select")?.is_truthy()? || selection.getattr("exclude")?.is_truthy()?); - let manager: Bound<'py, PyAny> = Bound::new(py, EarlySqlLint::new(py, enabled)?)?.into_any(); - within(&manager, |early_lint| analyzed(py, &options, early_lint)) + let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; + let analysis: PyResult> = analyzed(py, &options, &early_lint); + let _ = early_lint.borrow().stop(py); + analysis } /// The analysis phases sharing the command options, status line and early SQL lint. @@ -140,7 +142,7 @@ struct Driver<'a, 'py> { py: Python<'py>, options: &'a AnalysisOptions<'py>, status: Status<'py>, - early_lint: &'a Bound<'py, PyAny>, + early_lint: &'a Bound<'py, EarlySqlLint>, select: Bound<'py, PyAny>, exclude: Bound<'py, PyAny>, } @@ -156,7 +158,7 @@ struct ContractPhase<'py> { fn analyzed<'py>( py: Python<'py>, options: &AnalysisOptions<'py>, - early_lint: &Bound<'py, PyAny>, + early_lint: &Bound<'py, EarlySqlLint>, ) -> PyResult> { let driver: Driver<'_, 'py> = Driver { py, @@ -180,7 +182,7 @@ fn analyzed<'py>( let graph_failed: bool = any_error(&project.getattr("diagnostics")?)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { - early_lint_wait_ms = early_lint.call_method0("stop")?; + early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, &dialect)?; @@ -208,7 +210,7 @@ fn analyzed<'py>( (&core_diagnostics, &selected_keys), &lint_dialect, )?; - early_lint_wait_ms = early_lint.call_method0("stop")?; + early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); result }; let diagnostics: Bound<'py, PyTuple> = @@ -483,11 +485,7 @@ impl<'py> Driver<'_, 'py> { ("project_dir", project_dir.clone()), ("dialect", lint_dialect.clone()), ("selected_keys", selected), - ("prepared_sql", self.early_lint.getattr("preparation")?), - ( - "expansion_reuse", - self.early_lint.getattr("expansion_reuse")?, - ), + ("early_lint", self.early_lint.clone().into_any()), ("no_sql_analysis", options.no_sql_validation.clone()), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index a25a389139..2bd86cf8e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -6,9 +6,12 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; use crate::bindings::_helpers::compile_driver::project::lineage_mode; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, within, }; +use crate::bindings::_helpers::compiled_project::records::project_with; +use crate::bindings::_helpers::compiled_project::records::record_model_analyses; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::NativeAttr; @@ -35,7 +38,7 @@ pub(crate) struct AssemblyRequest { /// Convert compile inputs into the planner-ready project view. #[pyfunction] #[pyo3(signature = (**options))] -fn assemble_project<'py>( +pub(crate) fn assemble_project<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -60,9 +63,9 @@ fn assemble_project<'py>( return Ok(built); } let diagnostics: Bound<'py, PyTuple> = chained(&built.getattr("diagnostics")?, &collected)?; - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[("project", built), ("diagnostics", diagnostics.into_any())], + project_with( + &built, + Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), ) } @@ -125,13 +128,15 @@ fn referenced_project(py: Python<'_>, request: AssemblyRequest) -> PyResult = chained(&project.getattr("diagnostics")?, &reference_diagnostics)?; - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[ - ("project", project), - ("diagnostics", diagnostics.into_any()), - ("unmatched_literal_sql_relations", unmatched), - ], + project_with( + &project, + Some(keywords( + py, + &[ + ("diagnostics", diagnostics.into_any()), + ("unmatched_literal_sql_relations", unmatched), + ], + )?), ) } @@ -144,22 +149,13 @@ fn with_native_project<'py>( if native_project.is_none() { return Ok(project.clone()); } - call_with( - ®istry::native(py, NativeAttr::RecordModelAnalyses)?, - &[ - ("project", native_project.clone()), - ("models", project.getattr("models")?), - ], - )?; + record_model_analyses(native_project.cast()?, &project.getattr("models")?)?; if project.getattr("native_project")?.is(native_project) { return Ok(project.clone()); } - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[ - ("project", project.clone()), - ("native_project", native_project.clone()), - ], + project_with( + &project.clone(), + Some(keywords(py, &[("native_project", native_project.clone())])?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index b1c5cf17fd..df161e7c3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::compile_driver::compile_timings::{ record_compile_cpu_timing, record_compile_timing, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -120,13 +121,11 @@ fn dynamic_families_by_table<'py>(inputs: &Bound<'py, PyAny>) -> PyResult = model_input.getattr("schema_entry")?; - if schema_entry.is_none() { + let Some(schema) = attached_schema(&model_input)? else { continue; - } - let dynamic_columns: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; - if dynamic_columns.is_truthy()? { - families.set_item(schema_entry.getattr("name")?, dynamic_columns)?; + }; + if schema.dynamic_columns.is_truthy()? { + families.set_item(schema.name, schema.dynamic_columns)?; } } Ok(families) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 05c6331f5d..8e9a68e439 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -5,11 +5,14 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; +use pyo3::IntoPyObjectExt; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. const SCENARIO_FIELDS: [&str; 12] = [ "sql_body", @@ -110,13 +113,11 @@ fn model_macro_context<'py>(inputs: &Bound<'py, PyAny>) -> PyResult = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", inputs.getattr("project_config")?), - ("local_config", inputs.getattr("local_config")?), - ], - )?; + let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( + &inputs.getattr("project_config")?, + &inputs.getattr("local_config")?, + ) + .and_then(|value| value.into_bound_py_any(py))?; call_with( ®istry::class(py, RecordId::MacroContext)?, &[ @@ -141,7 +142,7 @@ pub(crate) fn compiled_sql_scenarios<'py>( let scenario_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledSqlScenario)?; let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let resource_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SQL_SCENARIO")?; + host::entry(py, FacadeHost::CompiledResourceType)?.getattr("SQL_SCENARIO")?; let mut scenarios: Vec> = Vec::new(); for scenario_input in inputs.getattr("scenario_inputs")?.try_iter()? { let scenario_input: Bound<'py, PyAny> = scenario_input?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index 81d7e22eac..aef011a80c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -7,8 +7,12 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; +use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; +use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; +use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_models; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; /// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. pub(crate) fn native_project_facts<'py>( @@ -31,31 +35,19 @@ pub(crate) fn native_project_facts<'py>( } else { PyDict::new(py).into_any() }; - let syntax_checks: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ModelSyntaxChecks)?, - &[ - ("model_inputs", model_inputs.clone()), - ("analysis_model_names", names.clone()), - ("analysis_succeeded", succeeded), - ], - )?; - let resources: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AssembleNativeProjectResources)?, - &[ - ("inputs", inputs.clone()), - ("dialect", dialect.clone()), - ("syntax_checks", syntax_checks), - ], - )?; + let syntax_checks: Bound<'py, PyAny> = + model_syntax_checks(&model_inputs.clone(), &names.clone(), &succeeded) + .and_then(|value| value.into_bound_py_any(py))?; + let resources: Bound<'py, PyAny> = assemble_native_project_resources( + &inputs.clone(), + dialect.clone().extract()?, + &syntax_checks, + ) + .and_then(|value| value.into_bound_py_any(py))?; let proofs: Bound<'py, PyDict> = pivot_proofs( - &call_with( - ®istry::native(py, NativeAttr::StandalonePivotModels)?, - &[ - ("model_inputs", model_inputs.clone()), - ("analysed", positions), - ], - )? - .cast_into::()?, + &standalone_pivot_models(&model_inputs.clone(), &positions) + .and_then(|value| value.into_bound_py_any(py))? + .cast_into::()?, context, session, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 1a57642e0f..0e57c1791c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -23,6 +24,7 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; +use pyo3::IntoPyObjectExt; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; @@ -164,13 +166,11 @@ fn connection_database_fallback<'py>( connection: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", inputs.getattr("project_config")?), - ("local_config", inputs.getattr("local_config")?), - ], - )?; + let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( + &inputs.getattr("project_config")?, + &inputs.getattr("local_config")?, + ) + .and_then(|value| value.into_bound_py_any(py))?; if adapter_name.eq(host::entry(py, AdapterHost::BuiltinAdapter)?.getattr("DUCKDB")?)? { return Ok(py.None().into_bound(py)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index 4eba2d7bfc..1dd915b13f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -6,10 +6,12 @@ use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_ta use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; @@ -143,13 +145,11 @@ fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) - let Some(group) = first_configured_group(config.local_config)? else { return Ok(()); }; - let adapter_name: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", config.project_config.clone()), - ("local_config", config.local_config.clone()), - ], - )?; + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &config.project_config.clone(), + &config.local_config.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; if session_warehouse_support(py, &adapter_name, config.project_dir)? { return Ok(()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 09b049f239..d422e28f3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -1,7 +1,7 @@ //! Project discovery: configuration, declaration files and imported project Python, validated. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PySet, PySetMethods, PyTuple}; +use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::config_validation::{ @@ -16,6 +16,7 @@ use crate::bindings::_helpers::discovery::declaration_records::{ discover_macro_files, discover_model_schema_files, discover_seed_files, discover_sql_function_files, discover_sql_hook_files, retained_discovery_session, }; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::discovery::model_records::{ ModelFileSelection, discovered_model_files, }; @@ -29,11 +30,13 @@ use crate::bindings::_helpers::discovery::yaml_records::{ discover_schema_files, discover_source_files, }; use crate::bindings::_helpers::event_exporting::scope::configure_discovered_event_exporters; +use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, produced}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; @@ -185,29 +188,11 @@ fn discover_project_inputs<'py>( || arguments.flag("extract_output_column_locations")?, }; within(&lifecycle(py, "project_discovery")?, |_| { - let stage: Bound<'py, PyCFunction> = PyCFunction::new_closure( - py, - None, - None, - move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { - let py: Python<'_> = arguments.py(); - within(&lifecycle(py, "discovery_project_assembly")?, |_| { - assembled_inputs(py, &request) - }) - .map(Bound::unbind) - }, - )?; - call_with( - ®istry::native(py, NativeAttr::CompileFrontier)?, - &[ - ( - "until", - registry::class(py, RecordId::CompilerStage)? - .getattr("DISCOVERED_PROJECT_INPUTS")?, - ), - ("python_stage", stage.into_any()), - ], - ) + let assembled: Bound<'py, PyAny> = + within(&lifecycle(py, "discovery_project_assembly")?, |_| { + assembled_inputs(py, &request) + })?; + produced(FrontierStage::DiscoveredProjectInputs, assembled) }) } @@ -371,10 +356,8 @@ fn declaration_files<'py>( /// Providers, then every kind of imported project Python that may use them. fn imported_python<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = project_dir.py(); - let providers: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProviderClasses)?, - &[("project_dir", project_dir.clone())], - )?; + let providers: Bound<'py, PyAny> = discover_provider_classes(project_dir.clone()) + .and_then(|value| value.into_bound_py_any(py))?; let python: Bound<'py, PyDict> = PyDict::new(py); python.set_item("providers", &providers)?; for (field, discoverer) in PYTHON_DISCOVERERS { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 9b41b1d58b..a5d9d34675 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -1,17 +1,26 @@ //! The first attached compile input snapshot: target, vars, macros, scopes, models and resources. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_attachment::effective_config::{ + build_effective_connection, build_effective_settings, build_effective_vars, +}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::compiled_project::attachment::{attach_checks, attach_resources}; +use crate::bindings::_helpers::compiled_project::records::{compile_inputs_view, inputs_with}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::model_config::template_data::run_id; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::render::declaration_indexes::{ + public_declaration_indexes, public_model_schema_index, +}; const REQUIRED: [&str; 2] = ["discovered_inputs", "adapter_context"]; const OPTIONAL: [&str; 8] = [ @@ -25,6 +34,42 @@ const OPTIONAL: [&str; 8] = [ "no_cache", ]; +/// The arguments of one compile input attachment; unset flags are `False`, other options `None`. +pub(crate) struct InputsRequest { + pub(crate) discovered_inputs: Py, + pub(crate) adapter_context: Py, + pub(crate) selected_target: Py, + pub(crate) cli_vars: Py, + pub(crate) run_id: Py, + pub(crate) no_sql_validation: Py, + pub(crate) defer_model_sql_validation: Py, + pub(crate) external_sql_reference_resolver: Py, + pub(crate) resolved_connection: Py, + pub(crate) no_cache: Py, +} + +impl InputsRequest { + /// The request `build_compile_inputs` was called with. + fn read(arguments: &KeywordArguments<'_>) -> PyResult { + Ok(Self { + discovered_inputs: arguments.value("discovered_inputs")?.unbind(), + adapter_context: arguments.value("adapter_context")?.unbind(), + selected_target: arguments.value("selected_target")?.unbind(), + cli_vars: arguments.value("cli_vars")?.unbind(), + run_id: arguments.value("run_id")?.unbind(), + no_sql_validation: arguments.value_or_false("no_sql_validation")?.unbind(), + defer_model_sql_validation: arguments + .value_or_false("defer_model_sql_validation")? + .unbind(), + external_sql_reference_resolver: arguments + .value("external_sql_reference_resolver")? + .unbind(), + resolved_connection: arguments.value("resolved_connection")?.unbind(), + no_cache: arguments.value_or_false("no_cache")?.unbind(), + }) + } +} + /// The effective target, vars, settings and macro context every later stage shares. struct TargetContext<'py> { target_name: Bound<'py, PyAny>, @@ -61,11 +106,19 @@ fn build_compile_inputs<'py>( ) -> PyResult> { let arguments: KeywordArguments<'py> = KeywordArguments::read(py, "build_compile_inputs", options, (&REQUIRED, &OPTIONAL))?; + attached_inputs(py, &InputsRequest::read(&arguments)?) +} + +/// The first compile input snapshot, with diagnostics collected while attaching appended. +pub(crate) fn attached_inputs<'py>( + py: Python<'py>, + request: &InputsRequest, +) -> PyResult> { let collector: Bound<'py, PyAny> = registry::native(py, NativeAttr::CollectCompileDiagnostics)?.call0()?; let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = within(&collector, |violations| { - Ok((compile_inputs(py, &arguments)?, violations.clone())) + Ok((compile_inputs(py, request)?, violations.clone())) })?; let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; if !collected.is_truthy()? { @@ -79,21 +132,18 @@ fn build_compile_inputs<'py>( .chain(collected.try_iter()?) .collect::>>()?, )?; - call_with( - ®istry::native(py, NativeAttr::InputsWith)?, - &[("inputs", built), ("diagnostics", diagnostics.into_any())], + inputs_with( + &built, + Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), ) } /// The undecorated `build_compile_inputs`. -fn compile_inputs<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, -) -> PyResult> { - let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - let adapter_context: Bound<'py, PyAny> = arguments.value("adapter_context")?; - let target: TargetContext<'py> = target_context(arguments, &discovered, &adapter_context)?; - let run_id: Bound<'py, PyAny> = run_id(&arguments.value("run_id")?)?; +fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult> { + let discovered: Bound<'py, PyAny> = request.discovered_inputs.bind(py).clone(); + let adapter_context: Bound<'py, PyAny> = request.adapter_context.bind(py).clone(); + let target: TargetContext<'py> = target_context(request, &discovered, &adapter_context)?; + let run_id: Bound<'py, PyAny> = run_id(&request.run_id.bind(py).clone())?; let loaded_macros: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadProjectMacros)? .call1((discovered.getattr("macro_files")?,))?; let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; @@ -129,9 +179,9 @@ fn compile_inputs<'py>( ), ], )?; - let models: ModelBuild<'py> = model_build(arguments, &discovered, &model_context)?; + let models: ModelBuild<'py> = model_build(request, &discovered, &model_context)?; let attached: AttachedResources<'py> = attached_resources( - arguments, + request, &adapter_context, (&discovered, &declaration_scope), &models, @@ -149,12 +199,12 @@ fn compile_inputs<'py>( ("compile_cache_dir", target.compile_cache_dir.clone()), ( "effective_connection", - effective_connection(arguments, &discovered, &target)?, + effective_connection(request, &discovered, &target)?, ), ("effective_settings", target.settings.clone()), ( "no_sql_validation", - arguments.value_or_false("no_sql_validation")?, + request.no_sql_validation.bind(py).clone(), ), ("effective_vars", target.vars.clone()), ("macro_context", target.macro_context.clone()), @@ -171,24 +221,18 @@ fn compile_inputs<'py>( ("diagnostics", attached.checks.get_item(3)?), ( "external_sql_reference_resolver", - arguments.value("external_sql_reference_resolver")?, + request.external_sql_reference_resolver.bind(py).clone(), ), ("scope_index", declaration_scope.getattr("index")?), ("declaration_scope", declaration_scope), ], )?; - call_with( - ®istry::native(py, NativeAttr::CompileInputsView)?, - &[ - ("fields", fields.into_any()), - ("native_project", models.native_project), - ], - ) + compile_inputs_view(&fields, Some(models.native_project.cast_into()?)) } /// The selected target with the macro call store, and the effective vars, settings and macro context. fn target_context<'py>( - arguments: &KeywordArguments<'py>, + request: &InputsRequest, discovered: &Bound<'py, PyAny>, adapter_context: &Bound<'py, PyAny>, ) -> PyResult> { @@ -197,43 +241,30 @@ fn target_context<'py>( &host::entry(py, UserCodeHost::TargetContextWithMacroCallStore)?, &[ ("discovered_inputs", discovered.clone()), - ("selected_target", arguments.value("selected_target")?), - ("no_cache", arguments.value_or_false("no_cache")?), + ("selected_target", request.selected_target.bind(py).clone()), + ("no_cache", request.no_cache.bind(py).clone()), ], )?; let target_name: Bound<'py, PyAny> = selected.get_item(0)?; let target: Bound<'py, PyAny> = selected.get_item(1)?; let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; - let cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; + let cli_vars: Bound<'py, PyAny> = request.cli_vars.bind(py).clone(); let cli_vars: Bound<'py, PyAny> = if cli_vars.is_none() { PyDict::new(py).into_any() } else { cli_vars }; - let vars: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildEffectiveVars)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("target_config", target.clone()), - ("cli_vars", cli_vars), - ], - )?; - let settings: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildEffectiveSettings)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ], - )?; - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config), - ], + let vars: Bound<'py, PyAny> = build_effective_vars( + &project_config.clone(), + &local_config.clone(), + &target.clone(), + &cli_vars, )?; + let settings: Bound<'py, PyAny> = + build_effective_settings(&project_config.clone(), &local_config.clone())?; + let adapter_name: Bound<'py, PyAny> = + resolve_effective_adapter_name(&project_config.clone(), &local_config)?; let macro_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::MacroContext)?, &[ @@ -269,17 +300,15 @@ fn target_context<'py>( /// `_build_models_with_declarations`: attach every model with the public declaration indexes. fn model_build<'py>( - arguments: &KeywordArguments<'py>, + request: &InputsRequest, discovered: &Bound<'py, PyAny>, context: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = discovered.py(); - let declarations: Bound<'py, PyAny> = - registry::native(py, NativeAttr::PublicDeclarationIndexes)?.call1((discovered,))?; - let enums: Bound<'py, PyAny> = declarations.get_item(0)?; - let constants: Bound<'py, PyAny> = declarations.get_item(1)?; - let schemas: Bound<'py, PyAny> = - registry::native(py, NativeAttr::PublicModelSchemaIndex)?.call1((discovered,))?; + let (enums, constants) = public_declaration_indexes(discovered.clone())?; + let (enums, constants): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + (enums.into_any(), constants.into_any()); + let schemas: Bound<'py, PyAny> = public_model_schema_index(discovered.clone())?.into_any(); let native_project: Bound<'py, PyAny> = registry::native(py, NativeAttr::NativeCompiledProject)?.call0()?; let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; @@ -301,15 +330,15 @@ fn model_build<'py>( ("context", model_context), ( "no_sql_validation", - arguments.value_or_false("no_sql_validation")?, + request.no_sql_validation.bind(py).clone(), ), ( "defer_model_sql_validation", - arguments.value_or_false("defer_model_sql_validation")?, + request.defer_model_sql_validation.bind(py).clone(), ), ( "external_sql_reference_resolver", - arguments.value("external_sql_reference_resolver")?, + request.external_sql_reference_resolver.bind(py).clone(), ), ("native_project", native_project.clone()), ], @@ -335,7 +364,7 @@ fn model_build<'py>( /// The seeds, functions, sources and checks attached to the models. fn attached_resources<'py>( - arguments: &KeywordArguments<'py>, + request: &InputsRequest, adapter_context: &Bound<'py, PyAny>, (discovered, declaration_scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), models: &ModelBuild<'py>, @@ -343,55 +372,63 @@ fn attached_resources<'py>( let py: Python<'py> = discovered.py(); let context: &Bound<'py, PyAny> = &models.context; let settings: Bound<'py, PyAny> = context.getattr("effective_settings")?; - let resources: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AttachResources)?, - &[ - ("discovered_inputs", discovered.clone()), - ("native_project", models.native_project.clone()), - ( - "settings", - PyTuple::new( - py, - [ - settings.clone(), - context.getattr("target_config")?, - context.getattr("macro_context")?.getattr("adapter_name")?, - ], - )? - .into_any(), - ), - ( - "no_sql_validation", - arguments.value_or_false("no_sql_validation")?, - ), - ( - "python_functions_inherit_default_namespace", - adapter_context.getattr("python_functions_inherit_default_namespace")?, - ), - ], - )?; + let resources: Bound<'py, PyAny> = attach_resources( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("native_project", models.native_project.clone()), + ( + "settings", + PyTuple::new( + py, + [ + settings.clone(), + context.getattr("target_config")?, + context.getattr("macro_context")?.getattr("adapter_name")?, + ], + )? + .into_any(), + ), + ( + "no_sql_validation", + request.no_sql_validation.bind(py).clone(), + ), + ( + "python_functions_inherit_default_namespace", + adapter_context.getattr("python_functions_inherit_default_namespace")?, + ), + ], + )?), + )? + .into_any(); let sources: Bound<'py, PyAny> = resources.get_item(2)?; - let checks: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AttachChecks)?, - &[ - ("discovered_inputs", discovered.clone()), - ("native_project", models.native_project.clone()), - ( - "settings", - PyTuple::new( - py, - [ - settings, - adapter_context.getattr("value_renderer")?, - context.getattr("sql_lexical_syntax")?, - ], - )? - .into_any(), - ), - ("scan_cache", declaration_scope.getattr("sql_test_scans")?), - ("source_inputs", sources.clone()), - ], - )?; + let checks: Bound<'py, PyAny> = attach_checks( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("native_project", models.native_project.clone()), + ( + "settings", + PyTuple::new( + py, + [ + settings, + adapter_context.getattr("value_renderer")?, + context.getattr("sql_lexical_syntax")?, + ], + )? + .into_any(), + ), + ("scan_cache", declaration_scope.getattr("sql_test_scans")?), + ("source_inputs", sources.clone()), + ], + )?), + )? + .into_any(); Ok(AttachedResources { seeds: resources.get_item(0)?, functions: resources.get_item(1)?, @@ -402,22 +439,19 @@ fn attached_resources<'py>( /// The resolved connection when given, otherwise the configured one for the target. fn effective_connection<'py>( - arguments: &KeywordArguments<'py>, + request: &InputsRequest, discovered: &Bound<'py, PyAny>, target: &TargetContext<'py>, ) -> PyResult> { - let resolved: Bound<'py, PyAny> = arguments.value("resolved_connection")?; + let resolved: Bound<'py, PyAny> = request.resolved_connection.bind(discovered.py()).clone(); if !resolved.is_none() { return Ok(resolved); } - call_with( - ®istry::native(discovered.py(), NativeAttr::BuildEffectiveConnection)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ("target_config", target.target.clone()), - ("effective_vars", target.vars.clone()), - ], + build_effective_connection( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + &target.target.clone(), + &target.vars.clone(), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index ce6ebd9b9e..276d018fa5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -5,11 +5,10 @@ use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::compile_timings::record_compile_timing; +use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::compile_driver::target_defaults::{ TargetDefaults, apply_target_defaults, }; @@ -17,9 +16,16 @@ use crate::bindings::_helpers::compile_driver::targets::{ managed_loader_target_isolation, managed_write_schemas, named_target_schema_strategy, project_targets, }; +use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; +use crate::bindings::_helpers::compile_driver::assembly::assemble_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compiled_project::records::project_with; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; @@ -112,23 +118,9 @@ impl ProjectRequest { /// The compiled project produced through the `COMPILED_PROJECT` frontier. fn compiled_project(py: Python<'_>, request: ProjectRequest) -> PyResult> { - let stage: Bound<'_, PyCFunction> = PyCFunction::new_closure( - py, - None, - None, - move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { - python_compiled_project(arguments.py(), &request).map(Bound::unbind) - }, - )?; - call_with( - ®istry::native(py, NativeAttr::CompileFrontier)?, - &[ - ( - "until", - registry::class(py, RecordId::CompilerStage)?.getattr("COMPILED_PROJECT")?, - ), - ("python_stage", stage.into_any()), - ], + produced( + FrontierStage::CompiledProject, + python_compiled_project(py, &request)?, ) } @@ -162,26 +154,30 @@ fn python_compiled_project<'py>( ("selection", selection.clone()), ], )?; - let project: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AssembleProject)?, - &[ - ("inputs", compile_inputs.clone()), - ("inference_profile", inference_profile), - ( - "skip_column_inference", - py_bool(py, request.skip_column_inference), - ), - ( - "column_lineage_mode", - request.column_lineage_mode.bind(py).clone(), - ), - ( - "analysis_cache_dir", - compile_inputs.getattr("compile_cache_dir")?, - ), - ("analysis_model_names", analysis_model_names), - ], - )?; + let project: Bound<'py, PyAny> = assemble_project( + py, + Some(keywords( + py, + &[ + ("inputs", compile_inputs.clone()), + ("inference_profile", inference_profile), + ( + "skip_column_inference", + py_bool(py, request.skip_column_inference), + ), + ( + "column_lineage_mode", + request.column_lineage_mode.bind(py).clone(), + ), + ( + "analysis_cache_dir", + compile_inputs.getattr("compile_cache_dir")?, + ), + ("analysis_model_names", analysis_model_names), + ], + )?), + ) + .and_then(|value| value.into_bound_py_any(py))?; targeted_project(py, request, &project) } @@ -210,13 +206,9 @@ fn targeted_project<'py>( let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", project_config), - ("local_config", local_config.clone()), - ], - )?; + let adapter_name: Bound<'py, PyAny> = + resolve_effective_adapter_name(&project_config, &local_config.clone()) + .and_then(|value| value.into_bound_py_any(py))?; project_targets(&adapter_name, &project)?; let promotion_key: Bound<'py, PyAny> = pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); @@ -247,12 +239,9 @@ fn targeted_project<'py>( .chain(conflicts.try_iter()?) .collect::>>()?, )?; - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[ - ("project", project), - ("diagnostics", diagnostics.into_any()), - ], + project_with( + &project, + Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), ) } @@ -264,13 +253,11 @@ fn frontier_compile_inputs<'py>( ) -> PyResult> { let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); - let collection_rendering: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveCollectionRendering)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("declaration_override", py.None().into_bound(py)), - ], - )?; + let collection_rendering: Bound<'py, PyAny> = resolve_effective_collection_rendering( + &discovered.getattr("project_config")?, + &py.None().into_bound(py), + ) + .and_then(|value| value.into_bound_py_any(py))?; let adapter_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompileAdapterContext)?, &[ @@ -283,38 +270,29 @@ fn frontier_compile_inputs<'py>( ("sql_lexical_syntax", adapter.getattr("sql_lexical_syntax")?), ], )?; - let stage: Bound<'py, PyAny> = imported(py, "functools", "partial")?.call( - (registry::native(py, NativeAttr::BuildCompileInputs)?,), - Some(&keywords( - py, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter_context", adapter_context), - ("selected_target", request.selected_target.bind(py).clone()), - ("no_sql_validation", py_bool(py, request.no_sql_validation)), - ("defer_model_sql_validation", py_bool(py, true)), - ("cli_vars", request.cli_vars.bind(py).clone()), - ( - "resolved_connection", - request.resolved_connection.bind(py).clone(), - ), - ( - "external_sql_reference_resolver", - request.external_sql_reference_resolver.bind(py).clone(), - ), - ("no_cache", py_bool(py, no_cache)), - ], - )?), + let inputs: InputsRequest = InputsRequest { + discovered_inputs: discovered.clone().unbind(), + adapter_context: adapter_context.unbind(), + selected_target: request.selected_target.clone_ref(py), + cli_vars: request.cli_vars.clone_ref(py), + run_id: py.None(), + no_sql_validation: py_bool(py, request.no_sql_validation).unbind(), + defer_model_sql_validation: py_bool(py, true).unbind(), + external_sql_reference_resolver: request.external_sql_reference_resolver.clone_ref(py), + resolved_connection: request.resolved_connection.clone_ref(py), + no_cache: py_bool(py, no_cache).unbind(), + }; + let stage: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + attached_inputs(arguments.py(), &inputs).map(Bound::unbind) + }, )?; - call_with( - ®istry::native(py, NativeAttr::CompileFrontier)?, - &[ - ( - "until", - registry::class(py, RecordId::CompilerStage)?.getattr("COMPILE_PROJECT_INPUTS")?, - ), - ("python_stage", stage), - ], + produced( + FrontierStage::CompileProjectInputs, + bridged(stage.as_any())?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index 27a3ec9e39..e0cc7f661d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiled_project::records::project_with; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -96,28 +97,30 @@ pub(crate) fn apply_target_defaults<'py>( ), )?); } - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[ - ("project", project.clone()), - ( - "effective_target_database", - or_else( - project.getattr("effective_target_database")?, - &defaults.default_database, - )?, - ), - ( - "effective_target_schema", - or_else( - project.getattr("effective_target_schema")?, - &defaults.default_schema, - )?, - ), - ("models", PyTuple::new(py, models)?.into_any()), - ("seeds", PyTuple::new(py, seeds)?.into_any()), - ("functions", PyTuple::new(py, functions)?.into_any()), - ], + project_with( + &project.clone(), + Some(keywords( + py, + &[ + ( + "effective_target_database", + or_else( + project.getattr("effective_target_database")?, + &defaults.default_database, + )?, + ), + ( + "effective_target_schema", + or_else( + project.getattr("effective_target_schema")?, + &defaults.default_schema, + )?, + ), + ("models", PyTuple::new(py, models)?.into_any()), + ("seeds", PyTuple::new(py, seeds)?.into_any()), + ("functions", PyTuple::new(py, functions)?.into_any()), + ], + )?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 3bdd7d99b4..9e1e0ecd0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -20,9 +20,7 @@ use sqlbuild_core::text::main::python_strip::python_strip; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::model_config::template_data::expanded_template_data; /// Connection settings that identify the warehouse two targets write to. @@ -39,8 +37,14 @@ const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databricks"]; const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; +use crate::bindings::_helpers::discovery::guidance::setting_help; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const DEFAULTS_SECTION: &str = "defaults"; @@ -126,25 +130,19 @@ pub(crate) fn named_target_schema_strategy( } let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; - let target_name: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("selected_target", selected_target.clone()), - ], - )?; + let target_name: Bound<'_, PyAny> = resolve_target_name( + py, + &project_config.clone(), + &local_config.clone(), + &selected_target.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; if target_name.is_none() { return Ok(()); } - let target_config: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetConfig)?, - &[ - ("project_config", project_config), - ("local_config", local_config), - ("target_name", target_name.clone()), - ], - )?; + let target_config: Bound<'_, PyAny> = + resolve_target_config(py, &project_config, &local_config, &target_name.clone()) + .and_then(|value| value.into_bound_py_any(py))?; let schema: Option = text_or_none(&target_config.getattr("schema")?)?; if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { return Ok(()); @@ -261,31 +259,21 @@ fn target_namespaces( ) -> PyResult { let py: Python<'_> = discovered.py(); let name: Bound<'_, PyAny> = PyString::new(py, target_name).into_any(); - let target_config: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetConfig)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ("target_name", name.clone()), - ], - )?; - let runtime: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildEffectiveRuntimeConfig)?, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", name.clone()), - ], - )?; + let target_config: Bound<'_, PyAny> = resolve_target_config( + py, + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + &name.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; + let runtime: Bound<'_, PyAny> = + build_effective_runtime_config(&discovered.clone(), Some(name.clone()), None) + .and_then(|value| value.into_bound_py_any(py))?; let (_, effective_vars, _): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = runtime.extract()?; let connection: Bound<'_, PyAny> = if resolved_connection.is_none() { - call_with( - ®istry::native(py, NativeAttr::BuildEffectiveConnectionConfig)?, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", name), - ], - )? + build_effective_connection_config(&discovered.clone(), Some(name), None) + .and_then(|value| value.into_bound_py_any(py))? } else { resolved_connection.clone() }; @@ -447,25 +435,29 @@ fn required_target_parts( let Some(first) = missing.first() else { return Ok(()); }; - let help: String = call_with( - ®istry::native(py, NativeAttr::SettingHelp)?, - &[ - ( - "purpose", - PyString::new(py, DEFAULTS_HELP_PURPOSE).into_any(), - ), - ( - "file_name", - PyString::new(py, PROJECT_CONFIG_FILENAME).into_any(), - ), - ("section", PyString::new(py, DEFAULTS_SECTION).into_any()), - ("key", PyString::new(py, first).into_any()), - ( - "value", - PyString::new(py, &format!("my_{first}")).into_any(), - ), - ], - )? + let help: String = setting_help( + py, + Some(keywords( + py, + &[ + ( + "purpose", + PyString::new(py, DEFAULTS_HELP_PURPOSE).into_any(), + ), + ( + "file_name", + PyString::new(py, PROJECT_CONFIG_FILENAME).into_any(), + ), + ("section", PyString::new(py, DEFAULTS_SECTION).into_any()), + ("key", PyString::new(py, first).into_any()), + ( + "value", + PyString::new(py, &format!("my_{first}")).into_any(), + ), + ], + )?), + ) + .and_then(|value| value.into_bound_py_any(py))? .extract()?; Err(planner_error( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index 3903924712..fc12fc6178 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -1,5 +1,6 @@ //! Layouts of the compile and discovery value records hosted natively. +use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; @@ -383,7 +384,7 @@ const R_COMPILED_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("references", FieldDefault::EmptyTuple), FieldSpec::new( "language", - FieldDefault::Member("sqlbuild.compiler.compile.types", "FunctionLanguage", "SQL"), + FieldDefault::HostMember(FacadeHost::FunctionLanguage.id(), "SQL"), ), FieldSpec::new("source_file_path", FieldDefault::None), FieldSpec::new("runtime_version", FieldDefault::None), @@ -446,7 +447,7 @@ const R_COMPILE_SQL_TEST_INPUT: RecordSpec = RecordSpec { FieldSpec::required("sql_body"), FieldSpec::new( "mode", - FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + FieldDefault::HostMember(FacadeHost::SqlTestMode.id(), "MODEL"), ), FieldSpec::new( "payload", @@ -528,7 +529,7 @@ const R_COMPILED_DIRECT_LOGIC_SQL_TEST_PAYLOAD: RecordSpec = RecordSpec { FieldSpec::required("expected_cte"), FieldSpec::new( "mode", - FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + FieldDefault::HostMember(FacadeHost::SqlTestMode.id(), "MODEL"), ), FieldSpec::new("helper_ctes", FieldDefault::EmptyTuple), FieldSpec::new("tested_resource_names", FieldDefault::EmptyTuple), @@ -725,7 +726,7 @@ const R_DISCOVERED_SQL_TEST_BLOCK: RecordSpec = RecordSpec { FieldSpec::new("name", FieldDefault::None), FieldSpec::new( "mode", - FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + FieldDefault::HostMember(FacadeHost::SqlTestMode.id(), "MODEL"), ), FieldSpec::new("parameters", FieldDefault::EmptyTuple), FieldSpec::new("cases", FieldDefault::EmptyTuple), @@ -765,11 +766,7 @@ const R_DISCOVERED_AUDIT_BLOCK: RecordSpec = RecordSpec { FieldSpec::new("name", FieldDefault::None), FieldSpec::new( "evaluation_mode", - FieldDefault::Member( - "sqlbuild.compiler.auditing.types", - "AuditEvaluationMode", - "VIOLATIONS", - ), + FieldDefault::HostMember(FacadeHost::AuditEvaluationMode.id(), "VIOLATIONS"), ), FieldSpec::new("measurement_contract", FieldDefault::None), FieldSpec::new("measure_sql", FieldDefault::None), @@ -1448,11 +1445,7 @@ const R_COMPILED_AUDIT: RecordSpec = RecordSpec { FieldSpec::required("sql_body"), FieldSpec::new( "evaluation_mode", - FieldDefault::Member( - "sqlbuild.compiler.auditing.types", - "AuditEvaluationMode", - "VIOLATIONS", - ), + FieldDefault::HostMember(FacadeHost::AuditEvaluationMode.id(), "VIOLATIONS"), ), FieldSpec::new("measurement_contract", FieldDefault::None), FieldSpec::new("thresholds", FieldDefault::None), @@ -1577,7 +1570,7 @@ const R_COMPILED_SQL_TEST: RecordSpec = RecordSpec { FieldSpec::required("sql_body"), FieldSpec::new( "mode", - FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + FieldDefault::HostMember(FacadeHost::SqlTestMode.id(), "MODEL"), ), FieldSpec::new( "payload", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index c1da8fb1c9..5ac9ebb48a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -23,14 +23,13 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ TestBlockRow, TestCaseRow, TestFileRow, }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; const PATHLIB_MODULE: &str = "pathlib"; const TEST_TIMING_KEY: &str = "test_input_compile_ms"; @@ -57,18 +56,6 @@ fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult( - py: Python<'py>, - (module, class, member): (&str, &str, &str), -) -> PyResult> { - imported(py, module, class)?.getattr(member) -} - -/// `{member.value: member for member in enum_class}`. -fn by_value<'py>(py: Python<'py>, module: &str, class: &str) -> PyResult> { - members_by_value(&imported(py, module, class)?) -} - /// `{member.value: member for member in enum_class}` of a native string enum. fn native_by_value(py: Python<'_>, class: RecordId) -> PyResult> { members_by_value(®istry::class(py, class)?) @@ -350,7 +337,10 @@ fn sql_test_settings<'py>( ] { settings.set_item(key, registry::class(py, class)?)?; } - settings.set_item("modes", by_value(py, COMPILE_TYPES_MODULE, "SqlTestMode")?)?; + settings.set_item( + "modes", + members_by_value(&host::entry(py, FacadeHost::SqlTestMode)?)?, + )?; settings.set_item( "test_kind", registry::member(py, RecordId::ResourceKind, "TEST")?, @@ -379,14 +369,8 @@ fn sql_test_settings<'py>( fn audit_block_row(block: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = block.py(); - let measurement: Bound<'_, PyAny> = enum_member( - py, - ( - AUDITING_TYPES_MODULE, - "AuditEvaluationMode", - MEASUREMENT_MODE, - ), - )?; + let measurement: Bound<'_, PyAny> = + host::entry(py, FacadeHost::AuditEvaluationMode)?.getattr(MEASUREMENT_MODE)?; Ok(( block.clone().unbind(), attr(&block, "name")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs index 3f7ea723c5..0429bd3606 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs @@ -392,3 +392,21 @@ audit_getters!(NativeCompiledAudit, { }) } }); + +/// A native audit view's reference rows, or `None` for a hand-built audit record. +pub(crate) fn audit_reference_rows( + audit: &Bound<'_, PyAny>, +) -> PyResult>> { + let py: Python<'_> = audit.py(); + let (owner, index): (&Py, usize) = + if let Ok(view) = audit.cast::() { + (&view.get().owner, view.get().index) + } else if let Ok(view) = audit.cast::() { + (&view.get().owner, view.get().index) + } else { + return Ok(None); + }; + read_audit(owner, index, py, |record, _| { + Ok(Some(record.attached.references.clone())) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index 511b6b8033..a6c7a57f8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -236,7 +236,7 @@ impl NativeCompiledModel { Self::with_overlay(self.owner.clone_ref(py), self.index, overlay) } - fn read<'py, T>( + pub(crate) fn read<'py, T>( &self, py: Python<'py>, read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index f7fe29395d..5610e7f072 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -5,20 +5,16 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; /// The materializations a model's storage policies are inherited for, besides none. const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; /// The declaration kinds whose project-owned records the loop names. const NAMED_KINDS: [&str; 4] = ["AUDIT", "SCHEMA", "SQL_HOOK", "PYTHON_HOOK"]; -fn class<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { - imported(py, module, name) -} - fn dict<'py>( py: Python<'py>, entries: Vec<(&str, Bound<'py, PyAny>)>, @@ -301,11 +297,11 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { vec![ ( "audit_severity", - class(py, AUDITING_TYPES_MODULE, "AuditSeverity")?, + host::entry(py, FacadeHost::AuditSeverity)?, ), ( "attached_target_kind", - class(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?, + host::entry(py, FacadeHost::AttachedAuditTargetKind)?, ), ], )? @@ -326,7 +322,7 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { ), ( "function_language", - class(py, COMPILE_TYPES_MODULE, "FunctionLanguage")?, + host::entry(py, FacadeHost::FunctionLanguage)?, ), ], )? @@ -436,7 +432,6 @@ pub(crate) fn model_loop_classes<'py>( "reference_kind", registry::class(py, RecordId::SqlReferenceKind)?, ), - ("partial", imported(py, "functools", "partial")?), ( "config", native_config_classes(&host("config_session")?, &host("record_template_reads")?)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index dc0789f477..3d70d29b34 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -109,7 +109,7 @@ impl NativeModelInput { Self { owner, index } } - fn read<'py, T>( + pub(crate) fn read<'py, T>( &self, py: Python<'py>, read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, @@ -447,3 +447,70 @@ fn expansion_object( fields.set_item("passes", PyTuple::new(py, pass_objects)?)?; Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) } + +/// A model input's attached schema: its name, columns and dynamic column families. +pub(crate) struct AttachedSchema<'py> { + pub(crate) name: Bound<'py, PyAny>, + pub(crate) columns: Bound<'py, PyAny>, + pub(crate) dynamic_columns: Bound<'py, PyAny>, + pub(crate) type_enforcement: bool, +} + +/// The attached schema of `model_input`; a native input answers without building its entry. +pub(crate) fn attached_schema<'py>( + model_input: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native.get().read(py, |record, py| { + let Some(schema) = &record.schema else { + return Ok(None); + }; + Ok(Some(AttachedSchema { + name: record + .model_file + .bind(py) + .getattr("file_path")? + .getattr("stem")?, + columns: schema.columns.bind(py).clone().into_any(), + dynamic_columns: schema.dynamic_columns.bind(py).clone().into_any(), + type_enforcement: schema.type_enforcement, + })) + }); + } + let entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; + if entry.is_none() { + return Ok(None); + } + Ok(Some(AttachedSchema { + name: entry.getattr("name")?, + columns: entry.getattr("columns")?, + dynamic_columns: entry.getattr("dynamic_columns")?, + type_enforcement: entry.getattr("type_enforcement")?.is_truthy()?, + })) +} + +/// Each reference of `model_input` as `(kind, name, package, call argument count)`. +pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native + .get() + .read(py, |record, _| Ok(record.references.clone())); + } + model_input + .getattr("references")? + .try_iter()? + .map(|reference| reference_row_of(&reference?)) + .collect() +} + +/// One `CompileSqlReference` record as its row. +fn reference_row_of(reference: &Bound<'_, PyAny>) -> PyResult { + Ok(( + reference.getattr("ref_kind")?.str()?.extract()?, + reference.getattr("ref_name")?.extract()?, + reference.getattr("ref_package")?.extract()?, + reference.getattr("call_argument_count")?.extract()?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index a9a455c55b..52967e92af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -62,6 +62,7 @@ use crate::bindings::_helpers::render::declaration_expansion::{ DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, expand_declarations, }; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; @@ -98,8 +99,6 @@ const UNMANAGED_ATTRIBUTE: &str = "unmanaged"; const TABLE_TYPE_ATTRIBUTE: &str = "table_type"; const DECLARED_ATTRIBUTE: &str = "declared"; const VALIDATE_METHOD: &str = "validate"; -const FILE_PATH_ARGUMENT: &str = "file_path"; -const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; const HELP_ARGUMENT: &str = "help"; const CODE_ARGUMENT: &str = "code"; const ANALYSIS_CONFLICT: &str = "MODEL sql_analysis conflicts with legacy sql_validation"; @@ -142,8 +141,6 @@ pub(crate) struct LoopClasses<'py> { pub(crate) external_resolver: Bound<'py, PyAny>, /// `SqlReferenceKind`. pub(crate) reference_kind: Bound<'py, PyAny>, - /// `functools.partial`, binding the model's file to its constant renderer. - pub(crate) partial: Bound<'py, PyAny>, /// What a model's effective config is built from. pub(crate) config: ConfigClasses<'py>, /// What a model's visible declarations are resolved from. @@ -180,7 +177,7 @@ pub(crate) struct NativeModelLoop { /// What one run reads beside each model: the host, Python classes and header contract classes. pub(crate) struct LoopContext<'a, 'py> { - pub(crate) host: &'a Bound<'py, PyAny>, + pub(crate) host: &'a Bound<'py, ConstantRendering>, pub(crate) classes: &'a LoopClasses<'py>, pub(crate) contracts: &'a ContractClasses<'py>, } @@ -230,22 +227,6 @@ impl NativeModelLoop { loop_classes, }) } - - /// Build every model in order into `project`; return each model's input view. - fn run<'py>( - slf: &Bound<'py, Self>, - project: Bound<'py, NativeCompiledProject>, - host: Bound<'py, PyAny>, - model_files: Vec>, - ) -> PyResult>> { - let py: Python<'py> = slf.py(); - let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; - let models: Vec = prepared_rows(py, &classes, model_files)?; - project - .borrow_mut() - .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); - slf.get().run_models(py, project, host, models) - } } /// Replace the SQL hook classes' discovered `hook_files` with their `definitions` by name. @@ -324,7 +305,7 @@ impl NativeModelLoop { &self, py: Python<'py>, project: Bound<'py, NativeCompiledProject>, - host: Bound<'py, PyAny>, + host: Bound<'py, ConstantRendering>, models: Vec, ) -> PyResult>> { let contracts: ContractClasses<'py> = self.contract_classes.bind(py).extract()?; @@ -390,11 +371,27 @@ impl NativeModelLoop { } impl NativeModelLoop { + /// Build every model in order into `project`; return each model's input view. + pub(crate) fn run<'py>( + slf: &Bound<'py, Self>, + project: Bound<'py, NativeCompiledProject>, + host: Bound<'py, ConstantRendering>, + model_files: Vec>, + ) -> PyResult>> { + let py: Python<'py> = slf.py(); + let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; + let models: Vec = prepared_rows(py, &classes, model_files)?; + project + .borrow_mut() + .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); + slf.get().run_models(py, project, host, models) + } + /// Attach every discovered source in order into `project`; return each source's view. pub(crate) fn run_sources<'py>( &self, project: &Bound<'py, NativeCompiledProject>, - host: &Bound<'py, PyAny>, + host: &Bound<'py, ConstantRendering>, rows: Vec, settings: SourceSettings, ) -> PyResult> { @@ -413,7 +410,7 @@ impl NativeModelLoop { /// Attach every singular audit, then each owner's attached audits. pub(crate) fn run_audits<'py>( &self, - host: &Bound<'py, PyAny>, + host: &Bound<'py, ConstantRendering>, files: &[AuditFileRow], owners: Vec>, settings: &AuditSettings, @@ -432,7 +429,7 @@ impl NativeModelLoop { /// Attach every SQL test case, then every scenario, against the project's resources. pub(crate) fn run_sql_tests<'py>( &self, - host: &Bound<'py, PyAny>, + host: &Bound<'py, ConstantRendering>, rows: (&[TestFileRow], &[ScenarioRow]), settings: &SqlTestSettings, table_function_arities: HashMap, @@ -499,7 +496,7 @@ impl NativeModelLoop { /// Attach every SQL, then Python, function in order, checking table function call arities. pub(crate) fn run_functions<'py>( &self, - host: &Bound<'py, PyAny>, + host: &Bound<'py, ConstantRendering>, rows: (Vec, Vec), settings: FunctionSettings, models: &[(String, Vec)], @@ -567,15 +564,6 @@ impl NativeModelLoop { }; let interpolated: String = applied_interpolation(context, interpolation)?; let (references, stop) = scan.clone().unwrap_or_default(); - let file_keyword: Bound<'py, PyDict> = PyDict::new(py); - file_keyword.set_item( - FILE_PATH_ARGUMENT, - context.classes.path_class.call1((file_path,))?, - )?; - let render_constant: Bound<'py, PyAny> = context.classes.partial.call( - (context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), - Some(&file_keyword), - )?; let expansion_classes: ExpansionClasses<'py> = ExpansionClasses { usage_record: context.classes.usage_record.clone(), compile_error: context.classes.compile_error.clone(), @@ -590,7 +578,7 @@ impl NativeModelLoop { relationship_visibility: false, path: file_path, }, - render_constant: &render_constant, + rendering: context.host.get(), classes: &expansion_classes, })?; let macros: Option = if calls_macros(&declarations.spliced.sql) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs index fbc622549d..5ba537a17c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs @@ -231,8 +231,10 @@ pub(crate) fn schema_row( let mut column_rows: Vec = Vec::with_capacity(columns.len()); for column in columns.iter() { let location: Bound<'_, PyAny> = column.getattr(LOCATION_FIELD)?; - let declared_outside: bool = - !location.is_none() && location.getattr(PATH_FIELD)?.ne(model_path)?; + let declared_outside: bool = !location.is_none() && { + let path: Bound<'_, PyAny> = location.getattr(PATH_FIELD)?; + !path.is(model_path) && path.ne(model_path)? + }; column_rows.push(( column.getattr(NAME_FIELD)?.extract()?, column.getattr(TYPE_FIELD)?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index f73e9a239a..3b32a577ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -155,10 +155,7 @@ pub(crate) fn header_schema<'py>( enum_columns(context, (&enums, source.visible), source.values, &columns)?; let fact: ModelSchemaFact = schema_fact(schema_row( &resolved.columns, - ( - audits.len(), - &context.classes.path_class.call1((source.relative_path,))?, - ), + (audits.len(), source.model_file.paths(py).1), &families, (type_enforcement, source.schema.is_some()), )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs index a0a5504022..8fdd710adf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs @@ -54,8 +54,6 @@ const FILE_PATH_ATTRIBUTE: &str = "file_path"; const SQL_BODY_ATTRIBUTE: &str = "sql_body"; const READS_ATTRIBUTE: &str = "reads"; const KIND_ATTRIBUTE: &str = "kind"; -const FILE_PATH_ARGUMENT: &str = "file_path"; -const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; const MACRO_TOKEN: char = '@'; /// `_VisibleModelDeclarations` fields an unscoped hook's `DeclarationResolutionContext` reads. const UNSCOPED_FIELDS: [&str; 4] = [ @@ -468,17 +466,7 @@ impl<'py> HookRun<'_, '_, 'py> { declarations: &Bound<'py, PyAny>, scope: &Scope<'_, 'py>, ) -> PyResult> { - let py: Python<'py> = scope.consumer.py(); let (references, stop) = declaration_scan(sql); - let file_keyword: Bound<'py, PyDict> = PyDict::new(py); - file_keyword.set_item( - FILE_PATH_ARGUMENT, - self.context.classes.path_class.call1((scope.file_path,))?, - )?; - let render_constant: Bound<'py, PyAny> = self.context.classes.partial.call( - (self.context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), - Some(&file_keyword), - )?; let classes: ExpansionClasses<'py> = ExpansionClasses { usage_record: self.context.classes.usage_record.clone(), compile_error: self.context.classes.compile_error.clone(), @@ -493,7 +481,7 @@ impl<'py> HookRun<'_, '_, 'py> { relationship_visibility: false, path: scope.file_path, }, - render_constant: &render_constant, + rendering: self.context.host.get(), classes: &classes, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index e74ac57b62..c02bcdf032 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -55,6 +55,7 @@ use crate::bindings::_helpers::compiled_project::source_views::{ AttachedSource, NativeCompiledSource, NativeSourceInput, SourceNamespace, SourceRecord, }; use crate::bindings::_helpers::compiled_project::values::{object_value, plain_value}; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; pub(crate) const CONTRACT_KEY: &str = "contract"; const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; @@ -133,8 +134,8 @@ pub(crate) struct NativeCompiledProject { functions: Vec, /// Each singular, then model-, source- and seed-attached, audit in attachment order. audits: Vec, - /// The model loop and its host, which attach the project's other resources. - attachment: Option<(Py, Py)>, + /// The model loop and its constant rendering, which attach the project's other resources. + attachment: Option<(Py, Py)>, } impl NativeCompiledProject { @@ -229,19 +230,22 @@ impl NativeCompiledProject { Ok(owners) } - /// The kept model loop and its host. - fn attachment_loop(&self, py: Python<'_>) -> PyResult<(Py, Py)> { + /// The kept model loop and its constant rendering. + fn attachment_loop( + &self, + py: Python<'_>, + ) -> PyResult<(Py, Py)> { match &self.attachment { Some((model_loop, host)) => Ok((model_loop.clone_ref(py), host.clone_ref(py))), None => Err(PyRuntimeError::new_err(ATTACHMENT_LOOP_UNSET)), } } - /// Keep the model loop and its host for attaching the project's other resources. + /// Keep the model loop and its constant rendering for attaching the project's other resources. pub(crate) fn keep_attachment_loop( &mut self, model_loop: Py, - host: Py, + host: Py, ) { self.attachment = Some((model_loop, host)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index 88bc987e5c..44bb8ad7ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -250,6 +250,11 @@ impl NativeProjectView { )) } + /// The compiled project this view projects. + pub(crate) fn owner(&self) -> &Py { + &self.owner + } + fn with_fields(owner: Py, fields: ProjectFields) -> Self { Self { owner, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs index 01ab413136..e7e0122c61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs @@ -30,8 +30,6 @@ use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolated}; const MACRO_TOKEN: char = '@'; -const FILE_PATH_ARGUMENT: &str = "file_path"; -const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; const GENERIC_DIALECT: &str = "generic"; const EXPRESSION_STATEMENTS: [&str; 3] = ["select", "with", "values"]; @@ -226,15 +224,6 @@ pub(crate) fn expanded_authored_sql<'py>( let mut argument_references: Vec = Vec::new(); let mut expanded_sql: String = interpolated_sql.clone(); if !references.is_empty() || stop.is_some() { - let file_keyword: Bound<'py, PyDict> = PyDict::new(py); - file_keyword.set_item( - FILE_PATH_ARGUMENT, - context.classes.path_class.call1((file_path,))?, - )?; - let render_constant: Bound<'py, PyAny> = context.classes.partial.call( - (context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), - Some(&file_keyword), - )?; let classes: ExpansionClasses<'py> = ExpansionClasses { usage_record: context.classes.usage_record.clone(), compile_error: context.classes.compile_error.clone(), @@ -249,7 +238,7 @@ pub(crate) fn expanded_authored_sql<'py>( relationship_visibility: false, path: file_path, }, - render_constant: &render_constant, + rendering: context.host.get(), classes: &classes, })?; expanded_sql = expanded.spliced.sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs index 23bb0332fe..947a084832 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs @@ -34,6 +34,7 @@ use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names; use sqlbuild_sqltext::compiler::models::DeclarationReferenceStop; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::cache::sql_test_scan_store::SqlTestScanStore; use crate::bindings::_helpers::compiled_project::model_loop::{ @@ -662,8 +663,10 @@ impl<'py> TestScope<'_, 'py> { scenario: false, }) .collect(); - let classified: Vec = - extract_sql_tests(&requests, &self.model_loop.syntax).map_err(|failure| { + let syntax: &LexicalSyntax = &self.model_loop.syntax; + let classified: Vec = py + .detach(|| extract_sql_tests(&requests, syntax)) + .map_err(|failure| { let test: &ExpandedTest<'_, 'py> = &tests[pending[failure.index]]; self.located(&failure, test.file, (test.block, &test.sql)) })?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 85bd0efb7a..7d6bf00b85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -324,41 +324,6 @@ const R_LINT_BODY: RecordSpec = RecordSpec { FieldSpec::new("dependency_relations", FieldDefault::EmptyTuple), ], }; -const R_PENDING_LINT_BODY: RecordSpec = RecordSpec { - name: "PendingLintBody", - doc: "One expanded authored body awaiting its native lexical preparation.", - fields: &[ - FieldSpec::required("file_path"), - FieldSpec::required("body_start"), - FieldSpec::required("body_end"), - FieldSpec::required("expansion_input"), - FieldSpec::required("expanded"), - FieldSpec::required("expansion_passes"), - FieldSpec::required("pre_expansion_sites"), - FieldSpec::required("external_identifiers"), - FieldSpec::required("allows_ceremonial_select"), - FieldSpec::required("allows_dynamic_output_star"), - FieldSpec::required("allows_empty_fixture_star"), - FieldSpec::required("request"), - ], -}; -const R_EXPANDED_LINT_FILE: RecordSpec = RecordSpec { - name: "ExpandedLintFile", - doc: "The expanded bodies of one file, up to the first body that could not be expanded.", - fields: &[ - FieldSpec::required("file_path"), - FieldSpec::required("bodies"), - FieldSpec::required("failure"), - ], -}; -const R_PREPARED_LINT_FILES: RecordSpec = RecordSpec { - name: "PreparedLintFiles", - doc: "Lint bodies of every expandable file, and why the skipped files could not expand.", - fields: &[ - FieldSpec::required("bodies"), - FieldSpec::required("unexpandable"), - ], -}; const R_LINT_EDIT: RecordSpec = RecordSpec { name: "LintEdit", doc: "One deterministic authored-source edit proposed by a lint rule.", @@ -1007,9 +972,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_LINT_FILE_ROLE, R_INTERPOLATION_SITE, R_LINT_BODY, - R_PENDING_LINT_BODY, - R_EXPANDED_LINT_FILE, - R_PREPARED_LINT_FILES, R_LINT_EDIT, R_LINT_VIOLATION, R_RULE_FIX_RESULT, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index 1a8778bef5..d28ff6345e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -18,10 +18,11 @@ use crate::bindings::_helpers::cursor::values::{ DATETIME_MODULE, civil_date, local_date_time, naive_date_time, plain_date, replaced, timedelta_micros, }; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -const EXCEPTIONS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; /// Cursor grain names, finest first. pub(crate) const GRAIN_NAMES: [(&str, Grain); 6] = [ ("second", Grain::Second), @@ -34,8 +35,7 @@ pub(crate) const GRAIN_NAMES: [(&str, Grain); 6] = [ /// `CursorAlgebraError(message)`. pub(crate) fn algebra_error(py: Python<'_>, message: &str) -> PyErr { - match imported(py, EXCEPTIONS_MODULE, "CursorAlgebraError") - .and_then(|class| class.call1((message,))) + match host::entry(py, FacadeHost::CursorAlgebraError).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), Err(error) => error, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 1759145de6..3f427818eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -27,12 +27,13 @@ use crate::bindings::_helpers::discovery::snapshot::{ }; use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; use crate::bindings::_helpers::model_config::header_columns::schema_columns; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const PATHLIB_MODULE: &str = "pathlib"; const SESSION_MEMO_KEY: &str = "native_discovery_session"; const MEASUREMENT_MODE: &str = "MEASUREMENT"; @@ -283,12 +284,10 @@ fn hook_file<'py>(context: &FileContext<'_, 'py>, file: &HookFile) -> PyResult(py: Python<'py>, block: &AuditBlock) -> PyResult> { let header_values: Bound<'py, PyDict> = authored(py, &block.header_values)?; - let mode: Bound<'py, PyAny> = imported(py, AUDITING_TYPES_MODULE, "AuditEvaluationMode")? - .call1((block.evaluation_mode,))?; + let mode: Bound<'py, PyAny> = + host::entry(py, FacadeHost::AuditEvaluationMode)?.call1((block.evaluation_mode,))?; let measurement: Bound<'py, PyAny> = - if mode.is(&imported(py, AUDITING_TYPES_MODULE, "AuditEvaluationMode")? - .getattr(MEASUREMENT_MODE)?) - { + if mode.is(&host::entry(py, FacadeHost::AuditEvaluationMode)?.getattr(MEASUREMENT_MODE)?) { call_with( ®istry::class(py, RecordId::MeasurementContract)?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs index 5ed0391081..4fadf5da43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs @@ -77,7 +77,10 @@ fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { /// Render `, set this in :` followed by the exact TOML section and key. #[pyfunction] #[pyo3(signature = (**options))] -fn setting_help<'py>(py: Python<'py>, options: Option>) -> PyResult { +pub(crate) fn setting_help<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { let arguments: KeywordArguments<'py> = KeywordArguments::read( py, "setting_help", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index ca782ab0ae..72f5ae26dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -5,11 +5,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::discovery::locations::project_relative_path; use crate::bindings::_helpers::event_exporting::scope::current_scope; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; /// One project directory under discovery. pub(crate) struct ProjectDir<'py> { @@ -28,13 +30,8 @@ impl<'py> ProjectDir<'py> { /// `project_relative_path(path=file_path, project_dir=project_dir)`. pub(crate) fn relative(&self, file_path: &Bound<'py, PyAny>) -> PyResult> { - call_with( - ®istry::native(self.py(), NativeAttr::ProjectRelativePath)?, - &[ - ("path", file_path.clone()), - ("project_dir", self.path.clone()), - ], - ) + project_relative_path(&file_path.clone(), &self.path.clone()) + .and_then(|value| value.into_bound_py_any(self.py())) } /// `str(file_path.relative_to(project_dir))`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs index 2aacc4c22e..9974b14861 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs @@ -13,10 +13,12 @@ use crate::bindings::_helpers::discovery::imported_modules::{ }; use crate::bindings::_helpers::discovery::imported_providers; +use crate::bindings::_helpers::discovery::python_hook_files::discover_python_hook_files; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PYTHON_NODE_ROOT; const NODE_ERROR: ErrorId = ErrorId::PythonNodeDiscovery; const AUDIT_CASE: UserCodeHost = UserCodeHost::AuditCase; @@ -449,10 +451,8 @@ pub(crate) fn discover_hook_functions<'py>( Some(providers) => provider_by_name(&providers)?, None => PyDict::new(py), }; - let hook_files: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverPythonHookFiles)?, - &[("project_dir", project.path.clone())], - )?; + let hook_files: Bound<'py, PyAny> = discover_python_hook_files(&project.path.clone()) + .and_then(|value| value.into_bound_py_any(py))?; let mut hooks: Vec> = Vec::new(); let mut seen: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); for hook_file in hook_files.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs index db24afda82..976d68d0ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs @@ -17,7 +17,7 @@ fn yaml_entry_line(contents: &str, name: &str) -> Option { /// `path.relative_to(project_dir)` without pathlib's per-parent comparison scan. #[pyfunction] #[pyo3(signature = (*, path, project_dir))] -fn project_relative_path<'py>( +pub(crate) fn project_relative_path<'py>( path: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs index 4afa8b43dd..4caad201fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs @@ -20,7 +20,7 @@ use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; /// Each public Python hook file as `file_path`, `relative_path` and its role's scope fields. #[pyfunction] #[pyo3(signature = (*, project_dir))] -fn discover_python_hook_files<'py>( +pub(crate) fn discover_python_hook_files<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { in_snapshot(project_dir, || { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 21d9b44d28..0c7ff037c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -9,11 +9,11 @@ use sqlbuild_model_config::sql_tests::models::{SqlTestBlock, SqlTestCase}; use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue}; use crate::bindings::_helpers::contracts::sql_values::{SqlValueClasses, raw_value, sql_value}; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; -const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - pub(crate) type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); fn parse_error(py: Python<'_>, message: String) -> PyResult { @@ -82,9 +82,7 @@ fn block_record<'py>( keywords.set_item("name", block.name)?; keywords.set_item( "mode", - py.import(TYPES_MODULE)? - .getattr("SqlTestMode")? - .call1((block.mode,))?, + host::entry(py, FacadeHost::SqlTestMode)?.call1((block.mode,))?, )?; keywords.set_item("parameters", PyTuple::new(py, parameters)?)?; keywords.set_item("cases", PyTuple::new(py, cases)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index e1df98565a..728eb2b891 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; +use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::discovery::description_inputs::externally_described_models; use crate::bindings::_helpers::format_command::files::{ @@ -14,6 +15,11 @@ use crate::bindings::_helpers::format_command::files::{ }; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::dsl_headers::{header_violations, rules_config}; +use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; +use crate::bindings::_helpers::rules::lint_text::apply_lint_suppressions; +use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers; +use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderViolation}; const DESCRIPTION_PRESENT: &str = "description-present"; @@ -100,10 +106,6 @@ fn apply_fixes<'py>( return Ok((updated, faults)); } let current: Bound<'py, PyDict> = overlaid(files, &updated)?; - call_with( - &native(py, NativeAttr::PrepareNativeHeaderCache)?, - &[("files", current.clone().into_any())], - )?; for (path, contents) in sorted_items(¤t)? { let result: Bound<'py, PyAny> = call_with( &native(py, NativeAttr::FormatNativeHeaders)?, @@ -158,23 +160,16 @@ fn lint_final_contents<'py>( ) -> PyResult>> { let py: Python<'py> = files.py(); let mut violations: Vec> = Vec::new(); + let header_rules: HeaderRulesConfig = rules_config(request.config)?; + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; for (path, contents) in sorted_items(files)? { - let final_contents: Bound<'py, PyAny> = updated.get_item(&path)?.unwrap_or(contents); - let headers: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::ScanHeaders)?, - &[("contents", final_contents.clone())], - )?; - let found: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::LintNativeHeaders)?, - &[ - ("contents", final_contents), - ("file_path", path), - ("headers", headers), - ("config", request.config.clone()), - ], - )?; - for violation in found.try_iter()? { - violations.push(violation?); + let final_contents: String = updated.get_item(&path)?.unwrap_or(contents).extract()?; + let text: Vec = final_contents.chars().collect(); + let found: Vec = + header_violations(&text, &scan_headers(&text, false), header_rules) + .map_err(value_error)?; + for violation in found { + violations.push(classes.violation(&path, &LintViolationRow::header(0, violation))?); } } Ok(violations) @@ -307,10 +302,9 @@ pub(crate) fn run_format<'py>(request: &FormatRequest<'_, 'py>) -> PyResult
> = drop_resolved_description_faults(request.project_dir, violations)?; let final_contents: Bound<'py, PyDict> = overlaid(&files, &updated)?; - let allow_suppressions: Bound<'py, PyAny> = request.config.getattr("allow_suppressions")?; - let violations: Vec> = native(py, NativeAttr::ApplyLintSuppressions)? - .call1((violations, &final_contents, allow_suppressions))? - .extract()?; + let allow_suppressions: bool = request.config.getattr("allow_suppressions")?.extract()?; + let violations: Vec> = + apply_lint_suppressions(py, violations, final_contents.clone(), allow_suppressions)?; Ok(FormatOutcome { files_checked, violations, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs index 086309d934..9c05ed2d96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs @@ -44,17 +44,47 @@ fn native_frontier<'py>( if !native_stage.is_none() { return native_stage.call0(); } - let inputs_stage: Bound<'py, PyAny> = - registry::class(py, RecordId::CompilerStage)?.getattr("COMPILE_PROJECT_INPUTS")?; - if until.is(&inputs_stage) { - return call_with( - &host::entry(py, UserCodeHost::RunWithMacroBridge)?, - &[("stage", python_stage.clone())], - ); + if until.is(&frontier_stage(py, FrontierStage::CompileProjectInputs)?) { + return bridged(python_stage); } python_stage.call0() } +/// The compiler stages native callers produce through the frontier. +#[derive(Clone, Copy)] +pub(crate) enum FrontierStage { + DiscoveredProjectInputs, + CompileProjectInputs, + CompiledProject, +} + +/// The `CompilerStage` member of `stage`. +pub(crate) fn frontier_stage(py: Python<'_>, stage: FrontierStage) -> PyResult> { + let member: &str = match stage { + FrontierStage::DiscoveredProjectInputs => "DISCOVERED_PROJECT_INPUTS", + FrontierStage::CompileProjectInputs => "COMPILE_PROJECT_INPUTS", + FrontierStage::CompiledProject => "COMPILED_PROJECT", + }; + registry::class(py, RecordId::CompilerStage)?.getattr(member) +} + +/// `stage()` run inside the macro bridge user macros render through. +pub(crate) fn bridged<'py>(stage: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &host::entry(stage.py(), UserCodeHost::RunWithMacroBridge)?, + &[("stage", stage.clone())], + ) +} + +/// The frontier object `produced` for `stage`, captured on request. +pub(crate) fn produced<'py>( + stage: FrontierStage, + produced: Bound<'py, PyAny>, +) -> PyResult> { + write_stage_capture(&frontier_stage(produced.py(), stage)?, &produced)?; + Ok(produced) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(compile_frontier, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index ccf852eb11..556a280393 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,12 +1,12 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - type Object<'py> = Bound<'py, PyAny>; /// The compiled model of `model_input` from its session outcome, fallback rows and proof. @@ -17,7 +17,6 @@ pub(crate) fn compiled_model<'py>( macro_deps: &Object<'py>, ) -> PyResult> { let py: Python<'py> = model_input.py(); - let types = py.import(COMPILE_TYPES_MODULE)?; let model_file: Object<'py> = model_input.getattr("model_file")?; let model_name: Object<'py> = model_file.getattr("file_path")?.getattr("stem")?; let none: Object<'py> = py.None().into_bound(py); @@ -68,7 +67,7 @@ pub(crate) fn compiled_model<'py>( inferred = proof.getattr("fixed_columns")?; has_star = true_object(py); } - let resource_type: Object<'py> = types.getattr("CompiledResourceType")?; + let resource_type: Object<'py> = host::entry(py, FacadeHost::CompiledResourceType)?; let key_class: Object<'py> = registry::class(py, RecordId::CompiledObjectKey)?; let mut dependency_keys: Vec> = Vec::new(); for dependency in deps.try_iter()? { @@ -253,11 +252,10 @@ fn located<'py>( }; let located: Object<'py> = registry::native(py, NativeAttr::LocateModelBindingDiagnostics)? .call1((PyList::new(py, rows)?, texts, expansion_row))?; - let types = py.import(COMPILE_TYPES_MODULE)?; let diagnostic_class: Object<'py> = registry::class(py, RecordId::CompilerDiagnostic)?; let location_class: Object<'py> = registry::class(py, RecordId::SourceLocation)?; let phase: Object<'py> = registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?; - let model: Object<'py> = types.getattr("CompiledResourceType")?.getattr("MODEL")?; + let model: Object<'py> = host::entry(py, FacadeHost::CompiledResourceType)?.getattr("MODEL")?; let name: Object<'py> = model_file.getattr("file_path")?.getattr("stem")?; let mut built: Vec> = Vec::new(); for row in located.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs index a6dc231cfd..2ff6218008 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs @@ -4,13 +4,12 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::third_party::ThirdPartyHost; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const CONTRACT_CLASSES: [(&str, RecordId); 6] = [ ("schema_column", RecordId::SchemaColumn), ("schema_audit_instance", RecordId::SchemaAuditInstance), @@ -79,11 +78,11 @@ fn built_contract_classes(py: Python<'_>) -> PyResult> { } classes.set_item( "severities", - members_by_value(&imported(py, AUDITING_TYPES_MODULE, "AuditSeverity")?)?, + members_by_value(&host::entry(py, FacadeHost::AuditSeverity)?)?, )?; classes.set_item( "threshold_operators", - members_by_value(&imported(py, AUDITING_TYPES_MODULE, "ThresholdOperator")?)?, + members_by_value(&host::entry(py, FacadeHost::ThresholdOperator)?)?, )?; Ok(classes) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs index d9daaee8a8..a2d1968c07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs @@ -5,6 +5,9 @@ use crate::bindings::_helpers::native_host::models::{HostEdge, HostEntry, HostId /// One executor entry of `sqlbuild._native_host`, indexing `ENTRIES`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ExecutorHost { + ProjectRelation, + ProjectRelationIndex, + RelationName, CostResourceContext, RunCostSummary, CostCapability, @@ -166,6 +169,21 @@ pub(crate) enum ExecutorHost { /// Every executor entry, in `ExecutorHost` order. pub(crate) const ENTRIES: &[HostEntry] = &[ + HostEntry { + name: "ProjectRelation", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, + HostEntry { + name: "ProjectRelationIndex", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, + HostEntry { + name: "RelationName", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, HostEntry { name: "CostResourceContext", kind: HostKind::Class, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs index 09f2323f5e..d40ae9e79f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs @@ -10,6 +10,16 @@ pub(crate) enum FacadeHost { ProviderSettingsInputs, StartupInputs, CompileReuseRequest, + AuditEvaluationMode, + AuditOutcome, + AuditSeverity, + ThresholdOperator, + AttachedAuditTargetKind, + CompiledResourceType, + FunctionLanguage, + SqlTestMode, + CursorAlgebraError, + SharedInputError, } /// Every facade entry, in `FacadeHost` order. @@ -39,6 +49,56 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Facade, }, + HostEntry { + name: "AuditEvaluationMode", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "AuditOutcome", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "AuditSeverity", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "ThresholdOperator", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "AttachedAuditTargetKind", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "CompiledResourceType", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "FunctionLanguage", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "SqlTestMode", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "CursorAlgebraError", + kind: HostKind::Exception, + edge: HostEdge::Facade, + }, + HostEntry { + name: "SharedInputError", + kind: HostKind::Exception, + edge: HostEdge::Facade, + }, ]; impl FacadeHost { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs new file mode 100644 index 0000000000..2c03b0b48e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs @@ -0,0 +1,32 @@ +//! Joining a native thread from Python's main thread without hiding Ctrl+C. + +use std::thread::JoinHandle; +use std::time::Duration; + +use pyo3::prelude::{PyErr, Python}; + +/// How often a join checks for pending signals such as Ctrl+C. +const SIGNAL_POLL_INTERVAL: Duration = Duration::from_millis(5); + +/// A joined thread's outcome, or the signal raised while waiting for it. +pub(crate) enum Joined { + Finished(std::thread::Result), + Interrupted(PyErr), +} + +/// Join `thread` without the GIL; on a signal call `interrupt`, finish the join and return the signal. +pub(crate) fn join_interruptibly( + py: Python<'_>, + thread: JoinHandle, + interrupt: impl FnOnce(), +) -> Joined { + while !thread.is_finished() { + if let Err(signal) = py.check_signals() { + interrupt(); + let _ = py.detach(|| thread.join()); + return Joined::Interrupted(signal); + } + py.detach(|| std::thread::sleep(SIGNAL_POLL_INTERVAL)); + } + Joined::Finished(py.detach(|| thread.join())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs index 9d58356de9..c379120844 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod constants; pub(crate) mod exit_signal; +pub(crate) mod interruptible_join; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index db69992068..09b0ad43d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -60,9 +60,9 @@ pub(crate) struct ModelPass<'py> { } /// Selected model and function changes. -#[pyfunction] +#[pyfunction(name = "detect_changes")] #[pyo3(signature = (**options))] -pub(crate) fn detect_changes<'py>( +fn detect_changes_of<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -75,21 +75,29 @@ pub(crate) fn detect_changes<'py>( &["expected_version_hashes", "expected_metadata_jsons"], ), )?; - let (project, scope, snapshot) = ( - arguments.value("project")?, - arguments.value("scope")?, - arguments.value("snapshot")?, - ); - let (project, scope, snapshot) = (&project, &scope, &snapshot); - let full_refresh: bool = arguments.flag("full_refresh")?; - let models: Bound<'py, PyDict> = scope_model_changes( - (project, scope, snapshot), - None, + detect_changes( + ( + &arguments.value("project")?, + &arguments.value("scope")?, + &arguments.value("snapshot")?, + ), + arguments.flag("full_refresh")?, ( arguments.value("expected_version_hashes")?, arguments.value("expected_metadata_jsons")?, ), - )?; + ) +} + +/// Selected model and function changes against expected version hashes and metadata JSON. +pub(crate) fn detect_changes<'py>( + (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + full_refresh: bool, + expected: (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = project.py(); + let models: Bound<'py, PyDict> = + scope_model_changes((project, scope, snapshot), None, expected)?; let functions: Bound<'py, PyDict> = PyDict::new(py); let result_class: Bound<'py, PyAny> = registry::class(py, RecordId::FunctionChangeResult)?; let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; @@ -784,7 +792,7 @@ impl<'py> ChangeResult<'py> { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(detect_changes, module)?)?; + module.add_function(wrap_pyfunction!(detect_changes_of, module)?)?; module.add_function(wrap_pyfunction!(detect_model_changes_in_scope, module)?)?; module.add_function(wrap_pyfunction!(detect_model_changes, module)?)?; module.add_function(wrap_pyfunction!(schema_changes, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index d12fd6e241..99d1c06bd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -5,6 +5,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, }; +use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; +use crate::bindings::_helpers::planner_resolution::orchestration::{Resolution, resolve_model_sql}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -208,32 +210,27 @@ fn clone_model_entries<'py>( ), ], )?; - native( - py, - NativeAttr::ResolveModelSql, - &[ - ("adapter", adapter.clone()), - ("model", model.clone()), - ("snapshot", native(py, RecordId::WarehouseSnapshot, &[])?), - ("context", context), - ( - "backfill", - native( - py, - RecordId::BackfillResult, - &[( - "action", - planner_type(py, RecordId::BackfillAction, "FORWARD_ONLY")?, - )], - )?, - ), - ("full_refresh", PyBool::new(py, false).to_owned().into_any()), - ( - "cursor_overrides", - native(py, RecordId::CursorOverridePair, &[])?, - ), - ], - )? + let sql = resolve_model_sql( + &Resolution::of_context(adapter.clone(), &context, py.None().into_bound(py))?, + &context, + &ModelSqlRequest { + model: model.clone(), + snapshot: native(py, RecordId::WarehouseSnapshot, &[])?, + backfill: native( + py, + RecordId::BackfillResult, + &[( + "action", + planner_type(py, RecordId::BackfillAction, "FORWARD_ONLY")?, + )], + )?, + full_refresh: false, + cursor_overrides: (py.None().into_bound(py), py.None().into_bound(py)), + replaces_relation: false, + full_rebuild_cause: None, + }, + )?; + string(py, &sql) } else { string(py, "") }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 3edf007fad..1930333df0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, }; @@ -107,39 +108,31 @@ fn resolve_static_relation_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { - let options = kwargs.unwrap_or_else(|| PyDict::new(py)); - KeywordArguments::read( - py, - "resolve_static_relation_context", - Some(options.clone()), - ( - &["project", "adapter", "scope"], - &[ - "deferral", - "project_config", - "local_config", - "relation_keys", - ], - ), - )?; - options.set_item("connection", none(py))?; - options.set_item("known_source_columns", PyDict::new(py))?; let args = KeywordArguments::read( py, "resolve_static_relation_context", - Some(options), + kwargs, ( - &["project", "adapter", "scope", "connection"], + &["project", "adapter", "scope"], &[ "deferral", "project_config", "local_config", "relation_keys", - "known_source_columns", ], ), )?; - planner_relations_context(&args) + planner_relations_context(RelationsRequest { + project: args.value("project")?, + adapter: args.value("adapter")?, + connection: none(py), + scope: args.value("scope")?, + deferral: args.value("deferral")?, + project_config: args.value("project_config")?, + local_config: args.value("local_config")?, + known_source_columns: PyDict::new(py).into_any(), + relation_keys: args.value("relation_keys")?, + }) } /// Project selected audits from canonical static state. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index a47e075b37..74cbae755f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -1,7 +1,8 @@ //! Top-level planner orchestration producing an execution plan. +use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, keywords, within, + KeywordArguments, call_with, imported, within, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -13,21 +14,23 @@ use crate::bindings::_helpers::planner_entries::test_planning::{ use crate::bindings::_helpers::planner_entries::values::{COMPILE_TYPES, native, replaced}; use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; use crate::bindings::_helpers::planner_phases::execution_changes::{ - build_planner_entry_results, prune_planner_execution_scope, reconcile_execution_changes, + build_planner_entry_results, reconcile_execution_changes, }; use crate::bindings::_helpers::planner_phases::identities::{ build_planner_identity_context, detect_stale_warning_changes, }; +use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; use crate::bindings::_helpers::planner_phases::output_assembly::{ add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, }; +use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyTuple}; +use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use std::time::Instant; @@ -55,16 +58,7 @@ fn scopes_and_cursor_overrides<'py>( } } let overrides = args.value("overrides")?; - let typed = type_cursor_overrides( - py, - Some(keywords( - py, - &[ - ("cursor_overrides", overrides.getattr("cursor_overrides")?), - ("selected_models", PyTuple::new(py, models)?.into_any()), - ], - )?), - )?; + let typed = type_cursor_overrides(&overrides.getattr("cursor_overrides")?, &models)?; Ok(( scopes, replaced(&overrides, &[("cursor_overrides", typed)])?, @@ -81,24 +75,12 @@ fn change_results<'py>( let py = project.py(); let versions = identities.getattr("version_identities")?; let changes = detect_changes( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("scope", scopes.getattr("inspection_scope")?), - ("snapshot", snapshot.clone()), - ("full_refresh", overrides.getattr("full_refresh")?), - ( - "expected_version_hashes", - versions.getattr("model_version_hashes")?, - ), - ( - "expected_metadata_jsons", - versions.getattr("model_metadata_jsons")?, - ), - ], - )?), + (project, &scopes.getattr("inspection_scope")?, snapshot), + overrides.getattr("full_refresh")?.is_truthy()?, + ( + versions.getattr("model_version_hashes")?, + versions.getattr("model_metadata_jsons")?, + ), )?; if !policies.getattr("selection_diagnostics")?.is_truthy()? { let empty = native( @@ -111,19 +93,7 @@ fn change_results<'py>( )?; return Ok((changes, empty)); } - let stale = detect_stale_warning_changes( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("scopes", scopes.clone()), - ("snapshot", snapshot.clone()), - ("identities", identities.clone()), - ("execution_changes", changes.clone()), - ], - )?), - )?; + let stale = detect_stale_warning_changes((project, scopes), snapshot, (identities, &changes))?; Ok((changes, stale)) } @@ -146,7 +116,7 @@ fn planned_output<'py>( policies.getattr("selection_diagnostics")?.extract()?, )?; check_selected_scope_buildability(runtime, scopes, &warehouse, &deferral)?; - let snapshot = warehouse.getattr("snapshot")?; + let snapshot = warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?; let (changes, stale_warning_changes) = change_results( &project, (scopes, &snapshot, &identities), @@ -156,55 +126,41 @@ fn planned_output<'py>( let inspection_scope = scopes.getattr("inspection_scope")?; let resolved_actions = resolve_model_actions(&inspection_scope, &changes)?; let source_freshness = build_planner_source_freshness_result( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("adapter", args.value("adapter")?), - ("connection", args.value("connection")?), - ("scope", inspection_scope), - ("relations", warehouse.getattr("inspection_relations")?), - ( - "freshness_state_schemas", - snapshot.getattr("source_freshness_state_schemas")?, - ), - ], - )?), + (&project, &inspection_scope), + (&args.value("adapter")?, &args.value("connection")?), + &warehouse_field(&warehouse, |warehouse| &warehouse.inspection_relations)?, + &snapshot.getattr("source_freshness_state_schemas")?, + )?; + let reconciliation = reconcile_execution_changes( + (&inspection_scope, &snapshot), + &identities, + &resolved_actions, + &changes, )?; - let pruning = prune_planner_execution_scope(scopes, &resolved_actions)?; - let reconciliation = reconcile_execution_changes(&warehouse, &identities, &pruning, &changes)?; let entries = build_planner_entry_results( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("overrides", overrides.clone()), - ("policies", policies.clone()), - ("deferral", deferral), - ("pruning", pruning.clone()), - ("reconciliation", reconciliation.clone()), - ("source_freshness", source_freshness.clone()), - ], - )?), + (runtime, &warehouse), + (overrides, &deferral), + (&inspection_scope, &reconciliation), + &source_freshness, )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); let output = base_plan_output( (runtime, &warehouse, &identities, overrides), - (&pruning, &reconciliation, &entries, &source_freshness), + ( + &inspection_scope, + &reconciliation, + entries, + &source_freshness, + ), &planned_sql_tests, )?; let output = add_storage_policies(output, (runtime, &warehouse, scopes, &policies))?; let output = add_plan_warnings( output, (scopes, &warehouse, &identities, &stale_warning_changes), - (&pruning, &source_freshness, &policies), + (&source_freshness, &policies), )?; - let plan_output = - add_plan_metadata(output, &pruning, &source_freshness, &policies)?.build(py)?; + let plan_output = add_plan_metadata(output, &source_freshness, &policies)?.build(py)?; let on_progress = args.value("on_progress")?; if !on_progress.is_none() { on_progress.call1((format!( @@ -242,18 +198,19 @@ fn build_execution_plan<'py>( let project = args.value("project")?; let adapter = args.value("adapter")?; let connection = args.value("connection")?; - let runtime = native( + let runtime = Bound::new( py, - RecordId::PlannerRuntime, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ("project_config", args.value("project_config")?), - ("local_config", args.value("local_config")?), - ("on_progress", args.value("on_progress")?), - ], - )?; + PlannerRuntime { + project: project.clone().unbind(), + adapter: adapter.clone().unbind(), + connection: connection.clone().unbind(), + project_config: args.value("project_config")?.unbind(), + local_config: args.value("local_config")?.unbind(), + on_progress: args.value("on_progress")?.unbind(), + invocation_time: utc_now(py)?.unbind(), + }, + )? + .into_any(); let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; let inspection = scopes.getattr("inspection_scope")?; let test_planning = start_planning( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 8ae3efa1f4..519d1ac918 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -4,7 +4,8 @@ mod clone_plans; mod command_plans; mod display_plan; mod execution_plan; -mod model_entries; +pub(crate) mod model_entries; +pub(crate) mod models; pub(crate) mod plan_output; pub(crate) mod registration; pub(crate) mod relations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 2b7b859fa8..5b9898a4b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,24 +1,25 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, imported, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; +use crate::bindings::_helpers::planner_entries::models::PlanEntriesRequest; use crate::bindings::_helpers::planner_entries::values::{ - COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, boolean, error_of, native, none, planner_type, - replaced, string, + COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, + string, }; use crate::bindings::_helpers::planner_phases::full_refresh::{ check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, }; +use crate::bindings::_helpers::planner_phases::models::ModelEntryResults; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, model_cursor_overrides, plan_cursor_bounds, }; use crate::bindings::_helpers::planner_resolution::cursor_warnings::with_cursor_safety_warnings; +use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest as SqlRequest; use crate::bindings::_helpers::planner_resolution::orchestration::{ - resolve_model_hook_entries, resolve_model_sql, + Resolution, model_hook_entries, resolve_model_sql, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ build_cursor_input_relations, reject_unexposed_cursor_input_columns, @@ -26,6 +27,9 @@ use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ use crate::bindings::_helpers::planner_strategy::lineage::reject_capped_watermark_inputs; use crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits; use crate::bindings::_helpers::planner_strategy::model_entries::assemble_model_plan_entry; +use crate::bindings::_helpers::planner_strategy::models::{ + CursorInputLookups, ModelEntryDecisions, +}; use crate::bindings::_helpers::planner_strategy::strategy::{ materialization_of, resolve_model_plan_action, }; @@ -33,7 +37,7 @@ use crate::bindings::_helpers::planner_strategy::values::text; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; @@ -41,26 +45,28 @@ const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; /// The `ModelPlanContext` fields every model's planning reads, read once per plan. struct ContextReads<'py> { context: Bound<'py, PyAny>, - models_by_name: Bound<'py, PyAny>, - functions_by_name: Bound<'py, PyAny>, - source_map: Bound<'py, PyAny>, - source_warehouse_columns: Bound<'py, PyAny>, - model_locations: Bound<'py, PyAny>, - seed_locations: Bound<'py, PyAny>, - runtime_cursor_producer_names: Bound<'py, PyAny>, + lookups: CursorInputLookups<'py>, + resolution: Resolution<'py>, invocation_time: Bound<'py, PyAny>, } impl<'py> ContextReads<'py> { - fn of(context: Bound<'py, PyAny>) -> PyResult { + fn of( + context: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + external_resolver: Bound<'py, PyAny>, + ) -> PyResult { Ok(Self { - models_by_name: context.getattr("models_by_name")?, - functions_by_name: context.getattr("functions_by_name")?, - source_map: context.getattr("source_map")?, - source_warehouse_columns: context.getattr("source_warehouse_columns")?, - model_locations: context.getattr("model_locations")?, - seed_locations: context.getattr("seed_locations")?, - runtime_cursor_producer_names: context.getattr("runtime_cursor_producer_names")?, + lookups: CursorInputLookups { + models_by_name: context.getattr("models_by_name")?, + functions_by_name: context.getattr("functions_by_name")?, + source_map: context.getattr("source_map")?, + source_warehouse_columns: context.getattr("source_warehouse_columns")?, + model_locations: context.getattr("model_locations")?, + seed_locations: context.getattr("seed_locations")?, + runtime_cursor_producer_names: context.getattr("runtime_cursor_producer_names")?, + }, + resolution: Resolution::of_context(adapter, &context, external_resolver)?, invocation_time: context.getattr("invocation_time")?, context, }) @@ -71,13 +77,11 @@ impl<'py> ContextReads<'py> { struct ModelPlanRequest<'a, 'py> { model: Bound<'py, PyAny>, snapshot: Bound<'py, PyAny>, - adapter: Bound<'py, PyAny>, context: &'a ContextReads<'py>, - sql_analysis_enabled: Bound<'py, PyAny>, + sql_analysis_enabled: bool, full_refresh: bool, overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), change_result: Bound<'py, PyAny>, - external_resolver: Bound<'py, PyAny>, protect_full_refresh: bool, } @@ -88,8 +92,8 @@ fn plan_model_from_change<'py>( let model = &request.model; let reads = request.context; let context = &reads.context; + let adapter = &reads.resolution.adapter; let py = model.py(); - let full_refresh = boolean(py, request.full_refresh)?; let backfill = if request.full_refresh { native( py, @@ -103,20 +107,7 @@ fn plan_model_from_change<'py>( request.change_result.getattr("backfill")? }; let cursor_column = config_str(model, "cursor")?; - reject_unexposed_cursor_input_columns(Some(&keywords( - py, - &[ - ("model", model.clone()), - ("cursor_column", cursor_column.clone()), - ("models_by_name", reads.models_by_name.clone()), - ("functions_by_name", reads.functions_by_name.clone()), - ("source_map", reads.source_map.clone()), - ( - "source_warehouse_columns", - reads.source_warehouse_columns.clone(), - ), - ], - )?))?; + reject_unexposed_cursor_input_columns(model, &cursor_column, &reads.lookups)?; let resolved = resolve_model_plan_action(model, &request.change_result, request.full_refresh)?; let (action, reason) = (resolved.get_item(0)?, resolved.get_item(1)?); let materialization_type = materialization_of(model)?; @@ -128,69 +119,32 @@ fn plan_model_from_change<'py>( if request.protect_full_refresh && !request.full_refresh { check_full_rebuild_allowed(model, &request.change_result, &cause)?; } - string(py, &cause) + Some(cause) } else { - none(py) + None }; - let pair = native( - py, - RecordId::CursorOverridePair, - &[ - ("start_cursor_override", request.overrides.0.clone()), - ("end_cursor_override", request.overrides.1.clone()), - ], - )?; let resolved_sql = resolve_model_sql( - py, - Some(keywords( - py, - &[ - ("adapter", request.adapter.clone()), - ("model", model.clone()), - ("snapshot", request.snapshot.clone()), - ("context", context.clone()), - ("backfill", backfill.clone()), - ("full_refresh", full_refresh.clone()), - ("cursor_overrides", pair), - ( - "external_sql_reference_resolver", - request.external_resolver.clone(), - ), - ("replaces_relation", boolean(py, replaces_relation)?), - ("full_rebuild_cause", full_rebuild_cause), - ], - )?), - )?; - let hooks = resolve_model_hook_entries( - model, - request.adapter.clone(), - context.clone(), - request.external_resolver.clone(), + &reads.resolution, + context, + &SqlRequest { + model: model.clone(), + snapshot: request.snapshot.clone(), + backfill: backfill.clone(), + full_refresh: request.full_refresh, + cursor_overrides: request.overrides.clone(), + replaces_relation, + full_rebuild_cause, + }, )?; + let hooks = model_hook_entries(model, &reads.resolution)?; let (future_config, start_config) = cursor_safety_configs(model, context)?; reject_capped_watermark_inputs( model, - reads.models_by_name.clone(), - Some(reads.functions_by_name.clone()), + reads.lookups.models_by_name.clone(), + Some(reads.lookups.functions_by_name.clone()), )?; - let cursor_input_relations = build_cursor_input_relations(Some(&keywords( - py, - &[ - ("model", model.clone()), - ("adapter", request.adapter.clone()), - ("model_locations", reads.model_locations.clone()), - ("models_by_name", reads.models_by_name.clone()), - ("functions_by_name", reads.functions_by_name.clone()), - ("seed_locations", reads.seed_locations.clone()), - ("source_map", reads.source_map.clone()), - ("cursor_column", cursor_column.clone()), - ( - "runtime_cursor_producer_names", - reads.runtime_cursor_producer_names.clone(), - ), - ], - )?))? - .into_any(); + let cursor_input_relations = + build_cursor_input_relations((model, adapter), &cursor_column, &reads.lookups)?.into_any(); let mut any_runtime_owned = false; for relation in cursor_input_relations.try_iter()? { if relation?.getattr("is_runtime_owned")?.is_truthy()? { @@ -214,49 +168,40 @@ fn plan_model_from_change<'py>( let cursor_bounds = plan_cursor_bounds(model, &window)?; let (empty_inputs, waiting_on_empty_inputs) = empty_cursor_inputs(model, &window)?; let range = microbatch_range(model, &window)?; - let assembled = assemble_model_plan_entry(Some(&keywords( - py, - &[ - ("model", model.clone()), - ("adapter", request.adapter.clone()), - ("snapshot", request.snapshot.clone()), - ("change_result", request.change_result.clone()), - ("action", action), - ("reason", reason), - ("materialization_type", materialization_type), - ("resolved_sql", resolved_sql), - ("sql_analysis_enabled", request.sql_analysis_enabled.clone()), - ("pre_hooks", hooks.get_item(0)?), - ("post_hooks", hooks.get_item(1)?), - ("cursor_column", cursor_column), - ("cursor_input_relations", cursor_input_relations), - ("runtime_owned_cursor_bounds", boolean(py, runtime_owned)?), - ("cursor_bounds", cursor_bounds.clone()), - ("microbatch_range", range.clone()), - ("empty_cursor_inputs", empty_inputs), - ( - "waiting_on_empty_inputs", - boolean(py, waiting_on_empty_inputs)?, - ), - ("backfill", backfill), - ("start_cursor_override", request.overrides.0.clone()), - ("end_cursor_override", request.overrides.1.clone()), - ("future_cursor_config", future_config), - ("start_cursor_config", start_config), - ("invocation_time", reads.invocation_time.clone()), - ], - )?))?; + let (entry, entry_warnings) = assemble_model_plan_entry(&ModelEntryDecisions { + model: model.clone(), + adapter: adapter.clone(), + snapshot: request.snapshot.clone(), + change_result: request.change_result.clone(), + action, + reason, + materialization_type, + resolved_sql: string(py, &resolved_sql), + sql_analysis_enabled: request.sql_analysis_enabled, + hooks: (hooks.get_item(0)?, hooks.get_item(1)?), + cursor_column, + cursor_input_relations, + runtime_owned_cursor_bounds: runtime_owned, + cursor_bounds: cursor_bounds.clone(), + microbatch_range: range.clone(), + empty_cursor_inputs: empty_inputs, + waiting_on_empty_inputs, + backfill, + cursor_overrides: request.overrides.clone(), + cursor_configs: (future_config, start_config), + invocation_time: reads.invocation_time.clone(), + })?; let bounds = if range.is_truthy()? { range } else { cursor_bounds }; let warnings = with_cursor_safety_warnings( - &assembled.get_item(1)?, + &entry_warnings.into_any(), &model_reads::name_object(model)?, &bounds, )?; - Ok((assembled.get_item(0)?, warnings)) + Ok((entry, warnings)) } /// The binding-diagnostic failure that stops planning, if any diagnostic is an error. @@ -424,17 +369,21 @@ fn check_microbatch_concurrency<'py>( } /// Build model plan entries from snapshot and per-model resolved actions. -fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult> { - let project = args.value("project")?; +pub(crate) fn plan_entries<'py>( + request: PlanEntriesRequest<'py>, +) -> PyResult> { + let PlanEntriesRequest { + project, + adapter, + scope, + snapshot, + relations, + resolved_models: models, + cursor_overrides, + full_refresh, + build_inputs: inputs, + } = request; let py = project.py(); - let adapter = args.value("adapter")?; - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; - let relations = args.value("relations")?; - let models = args.value("resolved_actions")?.getattr("models")?; - let cursor_overrides = args.value("cursor_overrides")?; - let full_refresh = args.value("full_refresh")?; - let inputs = args.value("build_inputs")?; let inputs = if inputs.is_none() { native(py, RecordId::PlanEntryBuildInputs, &[])? } else { @@ -494,9 +443,12 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult ("invocation_time", inputs.getattr("invocation_time")?), ], )?; - let reads = ContextReads::of(context)?; - let sql_analysis_enabled = settings.getattr("sql_analysis")?; - let external_resolver = project.getattr("external_sql_reference_resolver")?; + let reads = ContextReads::of( + context, + adapter.clone(), + project.getattr("external_sql_reference_resolver")?, + )?; + let sql_analysis_enabled = settings.getattr("sql_analysis")?.is_truthy()?; let skip = planner_type(py, RecordId::PlanAction, "SKIP")?; let microbatch = registry::class(py, RecordId::IncrementalMode)?.getattr("MICROBATCH")?; let mut entries: Vec> = Vec::new(); @@ -517,17 +469,15 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult &cursor_overrides, (start_override.clone(), end_override.clone()), )?; - let effective_full_refresh = resolve_model_full_refresh(&model, full_refresh.is_truthy()?)?; + let effective_full_refresh = resolve_model_full_refresh(&model, full_refresh)?; let (mut entry, entry_warnings) = plan_model_from_change(&ModelPlanRequest { model: model.clone(), snapshot: snapshot.clone(), - adapter: adapter.clone(), context: &reads, - sql_analysis_enabled: sql_analysis_enabled.clone(), + sql_analysis_enabled, full_refresh: effective_full_refresh, overrides, change_result: resolved.getattr("change")?, - external_resolver: external_resolver.clone(), protect_full_refresh: !freshness_blocked.contains(&name)? && !external_blocked.contains(&name)?, })?; @@ -580,42 +530,7 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries)? { warnings.push(warning); } - native( - py, - RecordId::PlannerModelEntryResults, - &[ - ("entries", PyTuple::new(py, entries)?.into_any()), - ("warnings", PyTuple::new(py, warnings)?.into_any()), - ], - ) -} - -/// Build model plan entries from snapshot and per-model resolved actions. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -fn build_plan_entries<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "build_plan_entries", - kwargs, - ( - &[ - "project", - "adapter", - "scope", - "snapshot", - "relations", - "resolved_actions", - "cursor_overrides", - "full_refresh", - ], - &["build_inputs"], - ), - )?; - plan_entries(&args) + Ok(ModelEntryResults { entries, warnings }) } /// Detect changes and build a plan entry and warnings for a single model. @@ -663,16 +578,18 @@ fn plan_model<'py>( plan_model_from_change(&ModelPlanRequest { model, snapshot, - adapter: args.value("adapter")?, - context: &ContextReads::of(args.value("context")?)?, - sql_analysis_enabled: args.value("sql_analysis_enabled")?, + context: &ContextReads::of( + args.value("context")?, + args.value("adapter")?, + args.value("external_sql_reference_resolver")?, + )?, + sql_analysis_enabled: args.flag("sql_analysis_enabled")?, full_refresh: args.flag("full_refresh")?, overrides: ( overrides.getattr("start_cursor_override")?, overrides.getattr("end_cursor_override")?, ), change_result, - external_resolver: args.value("external_sql_reference_resolver")?, protect_full_refresh: true, }) } @@ -710,7 +627,6 @@ pub(crate) fn scope_overlaps( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_plan_entries, module)?)?; module.add_function(wrap_pyfunction!(plan_model, module)?)?; module.add_function(wrap_pyfunction!(build_model_materializations, module)?)?; module.add_function(wrap_pyfunction!(scope_overlaps, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs new file mode 100644 index 0000000000..0cc640c9cb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -0,0 +1,46 @@ +//! Typed requests of plan entry construction. + +use pyo3::prelude::{Bound, PyAny}; + +/// What resolving relation locations and source metadata for one scope reads. +#[derive(Debug)] +pub(crate) struct RelationsRequest<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) deferral: Bound<'py, PyAny>, + pub(crate) project_config: Bound<'py, PyAny>, + pub(crate) local_config: Bound<'py, PyAny>, + pub(crate) known_source_columns: Bound<'py, PyAny>, + pub(crate) relation_keys: Bound<'py, PyAny>, +} + +/// What building model plan entries for one execution scope reads. +#[derive(Debug)] +pub(crate) struct PlanEntriesRequest<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) relations: Bound<'py, PyAny>, + pub(crate) resolved_models: Bound<'py, PyAny>, + pub(crate) cursor_overrides: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) build_inputs: Bound<'py, PyAny>, +} + +/// What assembling the plan output's fields reads. +#[derive(Debug)] +pub(crate) struct PlanOutputInputs<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) relations: Bound<'py, PyAny>, + pub(crate) changes: Bound<'py, PyAny>, + pub(crate) model_entries: Vec>, + pub(crate) model_warnings: Vec>, + pub(crate) reload_sources: Bound<'py, PyAny>, + pub(crate) extras: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index 87d8d2f91b..d4f97a8f78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -1,15 +1,17 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; +use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::models::SqlTestAssembly; use crate::bindings::_helpers::sql_test_glue::plan_assembly::assemble_sql_test_plans; +use crate::bindings::_helpers::sql_test_glue::plan_driver::resolve_sql_test_model_chains; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -482,21 +484,13 @@ pub(super) fn selected_test_entries<'py>( none(py) }; let results = assemble_sql_test_plans( - py, - Some(keywords( - py, - &[ - ("tests", tuple_of(py, selected)?), - ("project", project.clone()), - ("adapter", adapter.clone()), - ("sql_analysis_enabled", analysis), - ( - "validate_fixtures", - pyo3::types::PyBool::new(py, true).to_owned().into_any(), - ), - ("fixture_planning_context", fixture_context), - ], - )?), + &selected, + (project, adapter), + &SqlTestAssembly { + sql_analysis_enabled: analysis, + validate_fixtures: true, + fixture_planning_context: fixture_context, + }, )?; let mut diagnostics: Vec = Vec::new(); for result in results.try_iter()? { @@ -547,16 +541,21 @@ fn planned_sql_tests<'py>( /// The plan output's fields from planned entries and resolved relation inputs. pub(crate) fn plan_output_fields<'py>( - args: &KeywordArguments<'py>, + inputs: PlanOutputInputs<'py>, ) -> PyResult)>> { - let project = args.value("project")?; + let PlanOutputInputs { + project, + adapter, + scope, + snapshot, + relations, + changes, + model_entries, + model_warnings, + reload_sources, + extras, + } = inputs; let py = project.py(); - let adapter = args.value("adapter")?; - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; - let relations = args.value("relations")?; - let model_entry_results = args.value("model_entry_results")?; - let extras = args.value("extras")?; let extras = if extras.is_none() { native(py, RecordId::PlanOutputExtras, &[])? } else { @@ -573,20 +572,18 @@ pub(crate) fn plan_output_fields<'py>( ("execution_order", execution_order.clone()), ("selected_keys", selected_keys.clone()), ("source_map", source_map.clone()), - ("is_reload", args.value("reload_sources")?), + ("is_reload", reload_sources), ], )?; require_intermediate_loader_targets(&project, &snapshot, &source_map, &source_load_entries)?; let functions = function_entries( (&project, &adapter, &snapshot), &relations, - &args.value("changes")?, + &changes, &selected_keys, )?; - let model_entries = model_entry_results.getattr("entries")?; let materializations = PyDict::new(py); - for entry in model_entries.try_iter()? { - let entry = entry?; + for entry in &model_entries { materializations.set_item( entry.getattr("name")?, entry.getattr("materialization_type")?, @@ -624,13 +621,11 @@ pub(crate) fn plan_output_fields<'py>( scoped_order.push(key); } } - let mut warnings: Vec> = Vec::new(); - for warning in model_entry_results.getattr("warnings")?.try_iter()? { - warnings.push(warning?); - } + let mut warnings: Vec> = model_warnings; for warning in planned.getattr("warnings")?.try_iter()? { warnings.push(warning?); } + let model_entries = tuple_of(py, model_entries)?; Ok(vec![ ("execution_order", tuple_of(py, scoped_order)?), ("model_entries", model_entries.clone()), @@ -686,22 +681,6 @@ pub(crate) fn plan_output_fields<'py>( ]) } -/// Assemble the plan output from planned entries and resolved relation inputs. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -fn build_plan_output<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "build_plan_output", - kwargs, - (PLAN_OUTPUT_INPUTS, &["extras"]), - )?; - planner_model(py, RecordId::PlanOutput, &plan_output_fields(&args)?) -} - /// Build one canonical function entry from resolved relation inputs. #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -755,11 +734,7 @@ fn sql_test_model_chain_names_by_key<'py>( tests: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let chains = native( - py, - NativeAttr::ResolveSqlTestModelChains, - &[("project", project.clone()), ("tests", tests.clone())], - )?; + let chains = resolve_sql_test_model_chains(project, tests)?; let by_key = PyDict::new(py); for (test, chain) in tests.try_iter()?.zip(chains.try_iter()?) { by_key.set_item(test?.getattr("key")?, chain?)?; @@ -807,7 +782,6 @@ fn build_selected_test_entries<'py>( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_plan_output, module)?)?; module.add_function(wrap_pyfunction!(plan_function, module)?)?; module.add_function(wrap_pyfunction!(plan_selected_sql_tests, module)?)?; module.add_function(wrap_pyfunction!(sql_test_is_selected, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 07db82f0c2..7a2e29c8e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -1,6 +1,7 @@ //! Relation locations and source metadata for plan entry construction. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::values::{native, none, planner_model}; use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -66,21 +67,25 @@ fn source_warehouse_columns<'py>( /// Resolve relation locations and source metadata for plan entry construction. pub(crate) fn planner_relations_context<'py>( - args: &KeywordArguments<'py>, + request: RelationsRequest<'py>, ) -> PyResult> { - let project = args.value("project")?; + let RelationsRequest { + project, + adapter, + connection, + scope, + deferral, + project_config, + local_config, + known_source_columns, + relation_keys, + } = request; let py = project.py(); - let adapter = args.value("adapter")?; - let connection = args.value("connection")?; - let scope = args.value("scope")?; - let deferral = args.value("deferral")?; let deferral = if deferral.is_none() { planner_model(py, RecordId::DeferralInputs, &[])? } else { deferral }; - let project_config = args.value("project_config")?; - let local_config = args.value("local_config")?; let selected_keys = scope.getattr("selected_keys")?; let mut model_locations = native( py, @@ -113,7 +118,6 @@ pub(crate) fn planner_relations_context<'py>( seed_locations = applied.get_item(1)?; } let audit_keys = selected_audit_relation_keys(&project, &selected_keys)?; - let relation_keys = args.value("relation_keys")?; let load_keys = if relation_keys.is_none() { selected_keys.call_method1("__or__", (&audit_keys,))? } else { @@ -174,7 +178,7 @@ pub(crate) fn planner_relations_context<'py>( let (columns, listed_source_names) = source_warehouse_columns( (&project, &adapter, &connection), &source_read_map, - &args.value("known_source_columns")?, + &known_source_columns, )?; let semantic_columns = native( py, @@ -211,7 +215,7 @@ pub(crate) fn planner_relations_context<'py>( /// Resolve relation locations and source metadata for plan entry construction. #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(crate) fn build_planner_relations_context<'py>( +fn build_planner_relations_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -230,7 +234,17 @@ pub(crate) fn build_planner_relations_context<'py>( ], ), )?; - planner_relations_context(&args) + planner_relations_context(RelationsRequest { + project: args.value("project")?, + adapter: args.value("adapter")?, + connection: args.value("connection")?, + scope: args.value("scope")?, + deferral: args.value("deferral")?, + project_config: args.value("project_config")?, + local_config: args.value("local_config")?, + known_source_columns: args.value("known_source_columns")?, + relation_keys: args.value("relation_keys")?, + }) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index 4ba5097a0f..8d4f973716 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -1,21 +1,22 @@ //! Gather live candidate evidence without changing adapter inspection ownership. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, }; -use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; +use crate::bindings::_helpers::planner_migrations::models::{ + ColumnCandidate, ColumnMigrationInputs, MigrationPhase, +}; use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; pub(super) fn candidates<'py>( - args: &KeywordArguments<'py>, + (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), dialect: &Bound<'py, PyAny>, ) -> PyResult>> { - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; + let scope = phase.scope.clone(); + let snapshot = phase.snapshot.clone(); let py = scope.py(); let mut result: Vec> = Vec::new(); for key in scope.getattr("execution_order")?.try_iter()? { @@ -36,7 +37,7 @@ pub(super) fn candidates<'py>( continue; } let name = model_reads::name_object(&model)?; - if args.value("full_refresh_model_names")?.contains(&name)? + if columns.full_refresh_model_names.contains(&name)? || !snapshot.getattr("existing_relations")?.contains(&name)? { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 9a8539396a..668ac1047d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::planner_migrations::column_objects::{ }; use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::migration_projections::newest_column; @@ -168,7 +169,10 @@ pub(super) fn checked<'py>( .getattr("adapter")? .getattr("column_rename_unavailable_reason")?, &[ - ("connection", runtime.getattr("connection")?), + ( + "connection", + runtime_field(runtime, |runtime| &runtime.connection)?, + ), ("destination", relation), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 2aa146b874..1126347f21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -1,13 +1,14 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, replace, }; use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; +use crate::bindings::_helpers::planner_migrations::models::{ + ColumnMigrationInputs, MigrationPhase, +}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -116,12 +117,12 @@ pub(super) fn overlay<'py>( } pub(super) fn cursors<'py>( - args: &KeywordArguments<'py>, + (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), snapshot: Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], ) -> PyResult> { let py = snapshot.py(); - let scope = args.value("scope")?; + let scope = phase.scope.clone(); let models = scope.getattr("models_by_name")?; let origins = PyDict::new(py); for entry in entries { @@ -144,9 +145,7 @@ pub(super) fn cursors<'py>( } let targets = PyDict::new(py); for (name, _) in origins.iter() { - let physical = args - .value("physical_relations")? - .call_method1("get", (&name,))?; + let physical = columns.physical_relations.call_method1("get", (&name,))?; let target = if physical.is_truthy()? { physical } else { @@ -165,15 +164,15 @@ pub(super) fn cursors<'py>( let redirected = call_with( ®istry::native(py, NativeAttr::MigrationRedirectedCursorSnapshots)?, &[ - ("runtime", args.value("runtime")?), + ("runtime", phase.runtime.clone()), ("scope", scope), ( "existing_relations", snapshot.getattr("existing_relations")?, ), ("target_relations", targets.into_any()), - ("overrides", args.value("overrides")?), - ("deferral", args.value("deferral")?), + ("overrides", phase.overrides.clone()), + ("deferral", phase.deferral.clone()), ("origin_cursor_columns", origins.into_any()), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 33c059bba3..929aa2a268 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -2,22 +2,25 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::replace; use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_migrations::models::{ + ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, +}; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; +use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{pyfunction, wrap_pyfunction}; /// Decide column renames and return the existing planning result model. -#[pyfunction] +#[pyfunction(name = "plan_column_migrations")] #[pyo3(signature = (**options))] -pub(crate) fn plan_column_migrations<'py>( +fn plan_column_migrations_of<'py>( py: Python<'py>, options: Option>, -) -> PyResult> { +) -> PyResult<()> { let args = KeywordArguments::read( py, "plan_column_migrations", @@ -36,19 +39,43 @@ pub(crate) fn plan_column_migrations<'py>( &[], ), )?; - let runtime = args.value("runtime")?; - let snapshot = args.value("snapshot")?; - let adapter = runtime.getattr("adapter")?; + plan_column_migrations( + &MigrationPhase { + runtime: args.value("runtime")?, + scope: args.value("scope")?, + snapshot: args.value("snapshot")?, + overrides: args.value("overrides")?, + deferral: args.value("deferral")?, + }, + &ColumnMigrationInputs { + full_refresh_model_names: args.value("full_refresh_model_names")?, + physical_relations: args.value("physical_relations")?, + source_columns: args.value("source_columns")?, + }, + ) + .map(|_| ()) +} + +/// Decide column renames for the selected incremental and snapshot models. +pub(crate) fn plan_column_migrations<'py>( + phase: &MigrationPhase<'py>, + columns: &ColumnMigrationInputs<'py>, +) -> PyResult> { + let py = phase.runtime.py(); + let runtime = phase.runtime.clone(); + let snapshot = phase.snapshot.clone(); + let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; let candidates = super::column_candidates::candidates( - &args, + (phase, columns), &adapter.call_method0("sql_analysis_dialect")?, )?; if candidates.is_empty() { - return native( - py, - RecordId::ColumnMigrationPlanning, - &[("snapshot", snapshot)], - ); + return Ok(ColumnMigrationPlanning { + snapshot, + entries: Vec::new(), + hints: Vec::new(), + warnings: Vec::new(), + }); } let database = call_with( ®istry::native(py, NativeAttr::MigrationPlanningDatabase)?, @@ -58,7 +85,10 @@ pub(crate) fn plan_column_migrations<'py>( ®istry::native(py, NativeAttr::MigrationStateInspection)?, &[ ("adapter", adapter), - ("connection", runtime.getattr("connection")?), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ("database", database), ], )?; @@ -77,7 +107,7 @@ pub(crate) fn plan_column_migrations<'py>( py, KnownInputColumns::new( &snapshot.getattr("existing_columns")?, - &args.value("source_columns")?, + &columns.source_columns, ), )? .into_any(); @@ -91,8 +121,8 @@ pub(crate) fn plan_column_migrations<'py>( &state.getattr("column_events")?, &inputs, )?; - let physical = args - .value("physical_relations")? + let physical = columns + .physical_relations .call_method1("get", (model_reads::name_object(&candidate.model)?,))?; let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { @@ -112,19 +142,15 @@ pub(crate) fn plan_column_migrations<'py>( } } let overlaid = super::column_overlay::overlay(&snapshot, &candidates, &entries, &inputs)?; - let snapshot = super::column_overlay::cursors(&args, overlaid, &entries)?; - native( - py, - RecordId::ColumnMigrationPlanning, - &[ - ("snapshot", snapshot), - ("entries", PyTuple::new(py, entries)?.into_any()), - ("hints", PyTuple::new(py, hints)?.into_any()), - ("warnings", PyTuple::new(py, warnings)?.into_any()), - ], - ) + let snapshot = super::column_overlay::cursors((phase, columns), overlaid, &entries)?; + Ok(ColumnMigrationPlanning { + snapshot, + entries, + hints, + warnings, + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_column_migrations, module)?) + module.add_function(wrap_pyfunction!(plan_column_migrations_of, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index da8768b2ea..00f3fa21a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -20,23 +20,20 @@ fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { .extract() } +/// The schema findings that block migrating one model's history; `snapshots_config` is given for snapshots. fn blocking<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, - snapshot: bool, + (model, findings_in): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + dialect: &Bound<'py, PyAny>, + snapshots_config: Option<&Bound<'py, PyAny>>, ) -> PyResult> { - let model = arguments.value("model")?; - let policy = if snapshot { - config(&model, "snapshot_schema_change")? + let py = model.py(); + let snapshot = snapshots_config.is_some(); + let policy = if let Some(snapshots_config) = snapshots_config { + config(model, "snapshot_schema_change")? .filter(|value| !value.is_empty()) - .unwrap_or( - arguments - .value("snapshots_config")? - .getattr("schema_change")? - .extract()?, - ) + .unwrap_or(snapshots_config.getattr("schema_change")?.extract()?) } else { - config(&model, "on_schema_change")? + config(model, "on_schema_change")? .filter(|value| !value.is_empty()) .unwrap_or_else(|| "append_new_columns".to_owned()) }; @@ -50,7 +47,7 @@ fn blocking<'py>( )? .call1((&policy,))?; let mut findings: Vec = Vec::new(); - for finding in arguments.value("findings")?.try_iter()? { + for finding in findings_in.try_iter()? { let finding = finding?; findings.push(CompatibilityFinding { kind: finding.getattr("kind")?.extract()?, @@ -59,7 +56,7 @@ fn blocking<'py>( actual_type: finding.getattr("actual_type")?.extract()?, }); } - let dialect: Option = arguments.value("dialect")?.extract()?; + let dialect: Option = dialect.extract()?; PyTuple::new( py, compatibility_findings( @@ -83,7 +80,11 @@ fn migration_incremental_blocking_findings<'py>( options, (&["model", "findings", "dialect"], &[]), )?; - blocking(py, &arguments, false) + blocking( + (&arguments.value("model")?, &arguments.value("findings")?), + &arguments.value("dialect")?, + None, + ) } #[pyfunction] @@ -98,7 +99,11 @@ fn migration_snapshot_blocking_findings<'py>( options, (&["model", "findings", "dialect", "snapshots_config"], &[]), )?; - blocking(py, &arguments, true) + blocking( + (&arguments.value("model")?, &arguments.value("findings")?), + &arguments.value("dialect")?, + Some(&arguments.value("snapshots_config")?), + ) } #[pyfunction] @@ -181,16 +186,12 @@ fn check_migration_compatibility<'py>( return result(py, "not_checked", PyTuple::empty(py)); } let snapshot = config(&model, "materialized")?.as_deref() == Some("snapshot"); - let options = PyDict::new(py); - options.set_item("model", model)?; - options.set_item("findings", change.getattr("schema_findings")?)?; - options.set_item("dialect", arguments.value("column_dialect")?)?; - let findings = if snapshot { - options.set_item("snapshots_config", arguments.value("snapshots_config")?)?; - migration_snapshot_blocking_findings(py, Some(options))? - } else { - migration_incremental_blocking_findings(py, Some(options))? - }; + let snapshots_config = arguments.value("snapshots_config")?; + let findings = blocking( + (&model, &change.getattr("schema_findings")?), + &arguments.value("column_dialect")?, + snapshot.then_some(&snapshots_config), + )?; result( py, if findings.is_empty() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 6676aacc5f..41f8d72f48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -5,6 +5,7 @@ use crate::bindings::_helpers::graph::node_set::PyNodeSet; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -16,7 +17,10 @@ use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let item = item?; let name: String = item.getattr("name")?.extract()?; let mut upstream: Vec = Vec::new(); @@ -73,7 +77,7 @@ pub(super) fn metadata<'py>( models: &[Bound<'py, PyAny>], ) -> PyResult> { let py = runtime.py(); - let project = runtime.getattr("project")?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; let functions = native( py, NativeAttr::BuildFunctionLocalHashes, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 44516c9d52..b59d161658 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -20,7 +20,7 @@ mod model_handover; mod model_overlay; pub(crate) mod model_planning; mod model_warnings; -mod models; +pub(crate) mod models; mod old_name_entries; mod old_name_objects; pub(crate) mod old_name_planning; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs index a224f6ec86..6d8cca941a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs @@ -1,24 +1,26 @@ //! Project inherited fingerprints and live relation facts before normal planning. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; +use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; pub(super) fn overlay<'py>( - args: &KeywordArguments<'py>, + phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], handovers: &Bound<'py, PyDict>, ) -> PyResult> { - let snapshot = args.value("snapshot")?; + let snapshot = phase.snapshot.clone(); if handovers.is_empty() { return Ok(snapshot); } - let runtime = args.value("runtime")?; + let runtime = phase.runtime.clone(); let py = runtime.py(); let relations = snapshot .getattr("existing_relations")? @@ -91,11 +93,11 @@ pub(super) fn overlay<'py>( ®istry::native(py, NativeAttr::MigrationRedirectedCursorSnapshots)?, &[ ("runtime", runtime), - ("scope", args.value("scope")?), + ("scope", phase.scope.clone()), ("existing_relations", relations.clone().into_any()), ("target_relations", redirected.into_any()), - ("overrides", args.value("overrides")?), - ("deferral", args.value("deferral")?), + ("overrides", phase.overrides.clone()), + ("deferral", phase.deferral.clone()), ], )?; cursors.call_method1("update", (gathered,))?; @@ -123,7 +125,10 @@ pub(super) fn overlay<'py>( } fn cursor_enabled(runtime: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult { - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let item = item?; if item.getattr("name")?.eq(name)? { return Ok(config(&item, "materialized")?.eq("incremental")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index f16dd1a153..515c7577c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -2,6 +2,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_migrations::column_objects::{config, native, replace}; +use crate::bindings::_helpers::planner_migrations::models::{ + MigrationPhase, ModelMigrationPlanning, +}; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -100,30 +104,14 @@ fn decide_model_migration<'py>( )?) } -#[pyfunction] -#[pyo3(signature = (**options))] +/// Model migrations of the selected scope: discovered and declared moves, decided against live state. pub(crate) fn plan_model_migrations<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "plan_model_migrations", - options, - ( - &[ - "runtime", - "scope", - "snapshot", - "overrides", - "deferral", - "fingerprints", - ], - &[], - ), - )?; - let runtime = args.value("runtime")?; - let snapshot = args.value("snapshot")?; + phase: &MigrationPhase<'py>, + fingerprints: &Bound<'py, PyAny>, +) -> PyResult> { + let py = phase.runtime.py(); + let runtime = phase.runtime.clone(); + let snapshot = phase.snapshot.clone(); let schemas = call_with( ®istry::native(py, NativeAttr::MigrationProjectSchemas)?, &[("runtime", runtime.clone())], @@ -131,8 +119,14 @@ pub(crate) fn plan_model_migrations<'py>( let state = call_with( ®istry::native(py, NativeAttr::MigrationStateInspection)?, &[ - ("adapter", runtime.getattr("adapter")?), - ("connection", runtime.getattr("connection")?), + ( + "adapter", + runtime_field(&runtime, |runtime| &runtime.adapter)?, + ), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ( "database", call_with( @@ -143,21 +137,24 @@ pub(crate) fn plan_model_migrations<'py>( ("known_fingerprints", snapshot.getattr("fingerprints")?), ], )?; - let discovery = discover(&args, &state, &schemas)?; - finish(&args, &state, &schemas, &discovery) + let discovery = discover((phase, fingerprints), &state, &schemas)?; + finish(phase, &state, &schemas, &discovery) } fn discover<'py>( - args: &KeywordArguments<'py>, + (phase, fingerprints): (&MigrationPhase<'py>, &Bound<'py, PyAny>), state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, ) -> PyResult> { - let runtime = args.value("runtime")?; + let runtime = phase.runtime.clone(); let py = runtime.py(); - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; + let scope = phase.scope.clone(); + let snapshot = phase.snapshot.clone(); let mut declared_requests: Vec> = Vec::new(); - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { if let Some(request) = declared(&item?)? { declared_requests.push(request); } @@ -198,21 +195,21 @@ fn discover<'py>( ("declarations", PyTuple::new(py, declarations)?.into_any()), ("state", state.clone()), ("project_schemas", schemas.clone()), - ("fingerprints", args.value("fingerprints")?), + ("fingerprints", fingerprints.clone()), ], ) } fn finish<'py>( - args: &KeywordArguments<'py>, + phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, discovery: &Bound<'py, PyAny>, -) -> PyResult> { - let runtime = args.value("runtime")?; +) -> PyResult> { + let runtime = phase.runtime.clone(); let py = runtime.py(); - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; + let scope = phase.scope.clone(); + let snapshot = phase.snapshot.clone(); let mut resolved: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); for request in discovery.getattr("requests")?.try_iter()? { let request = request?; @@ -250,7 +247,7 @@ fn finish<'py>( &state.getattr("inspect_relations")?, &[("locations", PyTuple::new(py, origins)?.into_any())], )?; - let project_config = runtime.getattr("project_config")?; + let project_config = runtime_field(&runtime, |runtime| &runtime.project_config)?; let snapshots_config = if project_config.is_none() { registry::class(py, RecordId::SnapshotsConfig)?.call0()? } else { @@ -320,7 +317,7 @@ fn finish<'py>( let overlaid = if entries.is_empty() { snapshot.clone() } else { - super::model_overlay::overlay(args, state, &entries, &handovers)? + super::model_overlay::overlay(phase, state, &entries, &handovers)? }; let reference_renames = native( py, @@ -338,19 +335,14 @@ fn finish<'py>( if !entries.is_empty() { fields.push(("renamed_models", PyFrozenSet::new(py, renamed)?.into_any())); } - native( - py, - RecordId::ModelMigrationPlanning, - &[ - ("snapshot", replace(&overlaid, &fields)?), - ("entries", PyTuple::new(py, entries)?.into_any()), - ("warnings", PyTuple::new(py, warnings)?.into_any()), - ], - ) + Ok(ModelMigrationPlanning { + snapshot: replace(&overlaid, &fields)?, + entries, + warnings, + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(manual_migration_requests, module)?)?; - module.add_function(wrap_pyfunction!(decide_model_migration, module)?)?; - module.add_function(wrap_pyfunction!(plan_model_migrations, module)?) + module.add_function(wrap_pyfunction!(decide_model_migration, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs index 1c11e9b2fd..abac12f8cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs @@ -26,3 +26,46 @@ pub(crate) struct CheckedColumns<'py> { pub entries: Vec>, pub warnings: Vec>, } + +/// The planning-phase inputs every migration planner reads. +#[derive(Debug)] +pub(crate) struct MigrationPhase<'py> { + pub(crate) runtime: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) overrides: Bound<'py, PyAny>, + pub(crate) deferral: Bound<'py, PyAny>, +} + +/// The column migration planning inputs beyond the phase. +#[derive(Debug)] +pub(crate) struct ColumnMigrationInputs<'py> { + pub(crate) full_refresh_model_names: Bound<'py, PyAny>, + pub(crate) physical_relations: Bound<'py, PyAny>, + pub(crate) source_columns: Bound<'py, PyAny>, +} + +/// The snapshot after model migrations overlay it, with the planned entries and their warnings. +#[derive(Debug)] +pub(crate) struct ModelMigrationPlanning<'py> { + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) entries: Vec>, + pub(crate) warnings: Vec>, +} + +/// The snapshot after column renames overlay it, with the planned entries, rename hints and warnings. +#[derive(Debug)] +pub(crate) struct ColumnMigrationPlanning<'py> { + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) entries: Vec>, + pub(crate) hints: Vec>, + pub(crate) warnings: Vec>, +} + +/// The old-name view steps, the recorded live views and the warnings they raise. +#[derive(Debug)] +pub(crate) struct OldNameViewPlanning<'py> { + pub(crate) entries: Bound<'py, PyAny>, + pub(crate) views: Bound<'py, PyAny>, + pub(crate) warnings: Vec>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index 8265c05d0f..0b3fb70282 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -1,14 +1,13 @@ //! Assemble new and resumed old-name view steps from recorded facts. -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; use crate::bindings::_helpers::planner_migrations::old_name_objects::{ expires, location, relation, retention, stores_history, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -16,14 +15,14 @@ use pyo3::types::{PyBool, PyList, PyTuple}; impl<'py> OldNamePlanner<'py> { pub(super) fn new( - args: &KeywordArguments<'py>, + (runtime, columns): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), models: Vec>, histories: Vec>, views: Bound<'py, PyAny>, ) -> PyResult { - let runtime = args.value("runtime")?; + let runtime = runtime.clone(); let py = runtime.py(); - let project_config = runtime.getattr("project_config")?; + let project_config = runtime_field(&runtime, |runtime| &runtime.project_config)?; let config = if project_config.is_none() { registry::class(py, RecordId::MigrationsConfig)?.call0()? } else { @@ -33,9 +32,9 @@ impl<'py> OldNamePlanner<'py> { models, histories, views, - columns: args.value("column_entries")?, + columns: columns.clone(), retention: config.getattr("old_name_views")?, - now: runtime.getattr("invocation_time")?, + now: runtime_field(&runtime, |runtime| &runtime.invocation_time)?, runtime, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 2b506c7163..ff0045eecc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -1,23 +1,24 @@ //! Old-name planning orchestration over existing live state inspection. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; fn schemas<'py>( - args: &KeywordArguments<'py>, + (phase, migration_entries): (&MigrationPhase<'py>, &Bound<'py, PyAny>), selected: &[Bound<'py, PyAny>], ) -> PyResult> { - let runtime = args.value("runtime")?; + let runtime = phase.runtime.clone(); let py = runtime.py(); - let snapshot = args.value("snapshot")?; + let snapshot = phase.snapshot.clone(); let needed = PySet::empty(py)?; for item in selected { let schema = item.getattr("destination")?.getattr("schema")?; @@ -25,7 +26,7 @@ fn schemas<'py>( needed.add(schema)?; } } - for entry in args.value("migration_entries")?.try_iter()? { + for entry in migration_entries.try_iter()? { let schema = entry?.getattr("origin")?.getattr("schema")?; if !schema.is_none() { needed.add(schema)?; @@ -67,30 +68,15 @@ fn schemas<'py>( ) } -#[pyfunction] -#[pyo3(signature = (**options))] +/// Old-name compatibility views to create, resume or retire for the planned model and column migrations. pub(crate) fn plan_old_name_views<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "plan_old_name_views", - options, - ( - &[ - "runtime", - "scope", - "snapshot", - "migration_entries", - "column_entries", - ], - &[], - ), - )?; - let runtime = args.value("runtime")?; - let scope = args.value("scope")?; - let project = runtime.getattr("project")?; + phase: &MigrationPhase<'py>, + moves: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let py = phase.runtime.py(); + let runtime = phase.runtime.clone(); + let scope = phase.scope.clone(); + let project = runtime_field(&runtime, |runtime| &runtime.project)?; let models: Vec> = project .getattr("models")? .try_iter()? @@ -112,8 +98,14 @@ pub(crate) fn plan_old_name_views<'py>( let state = call_with( ®istry::native(py, NativeAttr::MigrationStateInspection)?, &[ - ("adapter", runtime.getattr("adapter")?), - ("connection", runtime.getattr("connection")?), + ( + "adapter", + runtime_field(&runtime, |runtime| &runtime.adapter)?, + ), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ( "database", native( @@ -124,7 +116,7 @@ pub(crate) fn plan_old_name_views<'py>( ), ], )?; - let needed = schemas(&args, &selected)?; + let needed = schemas((phase, moves.0), &selected)?; if needed.is_truthy()? { call_with( &state.getattr("inspect_old_name_views")?, @@ -146,10 +138,15 @@ pub(crate) fn plan_old_name_views<'py>( histories.push(history); } let views = PyTuple::new(py, views)?.into_any(); - let planner = super::models::OldNamePlanner::new(&args, models, histories, views.clone())?; + let planner = super::models::OldNamePlanner::new( + (&phase.runtime, moves.1), + models, + histories, + views.clone(), + )?; let mut entries: Vec> = Vec::new(); let covered = PySet::empty(py)?; - for entry in args.value("migration_entries")?.try_iter()? { + for entry in moves.0.try_iter()? { let entry = entry?; let name = entry.getattr("model_name")?; if let Some(item) = by_name.get_item(&name)? @@ -172,26 +169,21 @@ pub(crate) fn plan_old_name_views<'py>( NativeAttr::OldNameClaimErrors, &[ ("selected", PyTuple::new(py, selected)?.into_any()), - ("snapshot", args.value("snapshot")?), + ("snapshot", phase.snapshot.clone()), ("views", views.clone()), - ("now", runtime.getattr("invocation_time")?), + ( + "now", + runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + ), ], )?; let mut warnings: Vec> = claims.try_iter()?.collect::>()?; warnings.extend(super::old_name_references::errors( &runtime, &views, &entries, )?); - native( - py, - RecordId::OldNameViewPlanning, - &[ - ("entries", entries), - ("views", views), - ("warnings", PyTuple::new(py, warnings)?.into_any()), - ], - ) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_old_name_views, module)?) + Ok(OldNameViewPlanning { + entries, + views, + warnings, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs index 05ae35389a..7afa215fba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -45,8 +46,8 @@ fn migration_redirected_cursor_snapshots<'py>( let runtime = args.value("runtime")?; let scope = args.value("scope")?; let overrides = args.value("overrides")?; - let project = runtime.getattr("project")?; - let config = runtime.getattr("project_config")?; + let project = runtime_field(&runtime, |runtime| &runtime.project)?; + let config = runtime_field(&runtime, |runtime| &runtime.project_config)?; let start = if config.is_none() { py.None().into_bound(py) } else { @@ -58,7 +59,10 @@ fn migration_redirected_cursor_snapshots<'py>( &[ ("model_keys", scope.getattr("selected_keys")?), ("runtime_producer_keys", scope.getattr("selected_keys")?), - ("invocation_time", runtime.getattr("invocation_time")?), + ( + "invocation_time", + runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + ), ("start_cursor_config", start), ("cursor_overrides", overrides.getattr("cursor_overrides")?), ], @@ -74,8 +78,14 @@ fn migration_redirected_cursor_snapshots<'py>( ®istry::native(py, NativeAttr::GatherRedirectedCursorSnapshots)?, &[ ("project", project), - ("adapter", runtime.getattr("adapter")?), - ("connection", runtime.getattr("connection")?), + ( + "adapter", + runtime_field(&runtime, |runtime| &runtime.adapter)?, + ), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ("existing_relations", args.value("existing_relations")?), ("target_relations", args.value("target_relations")?), ("cursor_scope", cursor_scope), @@ -84,7 +94,10 @@ fn migration_redirected_cursor_snapshots<'py>( "deferred_locations", args.value("deferral")?.getattr("deferred_locations")?, ), - ("on_progress", runtime.getattr("on_progress")?), + ( + "on_progress", + runtime_field(&runtime, |runtime| &runtime.on_progress)?, + ), ( "origin_cursor_columns", args.value("origin_cursor_columns")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs index 07f52831e7..2fd4af82e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::replace; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; @@ -25,7 +26,7 @@ fn migration_promotion<'py>( runtime: &Bound<'py, PyAny>, decision: &Bound<'py, PyAny>, ) -> PyResult> { - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let promotion = registry::class(runtime.py(), RecordId::MigrationPromotion)?; if adapter .call_method0("supports_transactional_ddl")? @@ -47,7 +48,7 @@ fn migration_with_execution<'py>( entry: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let transient = if adapter.getattr("adapter_name")?.eq("snowflake")? { PyBool::new( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 3a14bdae93..0f3289d8a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -32,7 +33,10 @@ fn named<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, ) -> PyResult> { - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let item = item?; if item.getattr("name")?.eq(name)? { return Ok( @@ -70,7 +74,7 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { #[pyfunction] #[pyo3(signature = (*, runtime))] fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { - let project = runtime.getattr("project")?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = project.getattr("effective_target_database")?; for item in project.getattr("models")?.try_iter()? { let database = item?.getattr("destination")?.getattr("database")?; @@ -85,7 +89,10 @@ fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let schema = item?.getattr("destination")?.getattr("schema")?; if !schema.is_none() { schemas.add(schema)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 2718ea8c91..576f52c48e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -7,7 +7,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::compatibility::register(module)?; super::matching::register(module)?; super::old_names::register(module)?; - super::old_name_planning::register(module)?; super::columns::register(module)?; super::column_planning::register(module)?; super::model_planning::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 0f02cc0ce8..4372fa93b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -2,6 +2,8 @@ use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, planner_error}; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -16,7 +18,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( deferral: &Bound<'py, PyAny>, ) -> PyResult<()> { let py = runtime.py(); - let project = runtime.getattr("project")?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; let selected_scope = scopes.getattr("selected_scope")?; let mut external_seeds: Vec<_> = Vec::new(); for seed in project.getattr("seeds")?.try_iter()? { @@ -31,7 +33,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( &[ ("selected_keys", selected_scope.getattr("selected_keys")?), ("upstream_deps", selected_scope.getattr("upstream_deps")?), - ("snapshot", warehouse.getattr("snapshot")?), + ( + "snapshot", + warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + ), ( "deferred_relations", deferral.getattr("deferred_relations")?, @@ -65,10 +70,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( check_selected_source_tables_exist( &project, ( - &runtime.getattr("adapter")?, - &runtime.getattr("connection")?, + &runtime_field(runtime, |runtime| &runtime.adapter)?, + &runtime_field(runtime, |runtime| &runtime.connection)?, ), &selected_scope, - &warehouse.getattr("inspection_relations")?, + &warehouse_field(warehouse, |warehouse| &warehouse.inspection_relations)?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index ac9c86d04a..5c45d15791 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -12,14 +12,3 @@ pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; pub(crate) const RETENTION_DECREASE: &str = "decrease"; pub(crate) const RETENTION_INCREASE: &str = "increase"; pub(crate) const RETENTION_MATCH: &str = "match"; -/// The required inputs of plan output assembly. -pub(crate) const PLAN_OUTPUT_INPUTS: &[&str] = &[ - "project", - "adapter", - "scope", - "snapshot", - "relations", - "changes", - "model_entry_results", - "reload_sources", -]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 0b13e121f6..460fcb822d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,55 +1,31 @@ -//! Execution-scope pruning, write-hash reconciliation and plan entry construction. +//! Write-hash reconciliation and plan entry construction. -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; +use crate::bindings::_helpers::planner_entries::models::{PlanEntriesRequest, RelationsRequest}; +use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; +use crate::bindings::_helpers::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; use crate::bindings::_helpers::planner_phases::objects::native; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::pyfunction; -use pyo3::types::PyDict as KeywordDict; -use pyo3::types::{PyFrozenSet, PyTuple}; +use pyo3::types::PyFrozenSet; -#[pyfunction] -#[pyo3(signature = (*, scopes, resolved_actions))] -pub(crate) fn prune_planner_execution_scope<'py>( - scopes: &Bound<'py, PyAny>, - resolved_actions: &Bound<'py, PyAny>, -) -> PyResult> { - let py = scopes.py(); - let inspection = scopes.getattr("inspection_scope")?; - native( - py, - RecordId::PlannerScopePruningResult, - &[ - ("inspection_scope", inspection.clone()), - ("execution_scope", inspection), - ("resolved_actions", resolved_actions.clone()), - ("pruned_direct_model_names", PyTuple::empty(py).into_any()), - ( - "direct_identity_stale_model_names", - PyFrozenSet::empty(py)?.into_any(), - ), - ], - ) -} - -#[pyfunction] -#[pyo3(signature = (*, warehouse, identities, pruning, changes))] +/// The execution scope's changes with honest write hashes, and the actions merged from them. pub(crate) fn reconcile_execution_changes<'py>( - warehouse: &Bound<'py, PyAny>, + (execution_scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), identities: &Bound<'py, PyAny>, - pruning: &Bound<'py, PyAny>, + resolved_actions: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, -) -> PyResult> { - let py = warehouse.py(); +) -> PyResult> { + let py = execution_scope.py(); let honest = native( py, NativeAttr::WithHonestModelWriteHashes, &[ - ("scope", pruning.getattr("execution_scope")?), - ("snapshot", warehouse.getattr("snapshot")?), + ("scope", execution_scope.clone()), + ("snapshot", snapshot.clone()), ("changes", changes.clone()), ( "version_identities", @@ -61,15 +37,14 @@ pub(crate) fn reconcile_execution_changes<'py>( py, NativeAttr::MergeRecomputedModelChanges, &[ - ("resolved_actions", pruning.getattr("resolved_actions")?), + ("resolved_actions", resolved_actions.clone()), ("changes", honest.clone()), ], )?; - native( - py, - RecordId::PlannerChangeReconciliation, - &[("changes", honest), ("resolved_actions", resolved)], - ) + Ok(ChangeReconciliation { + changes: honest, + resolved_actions: resolved, + }) } fn config_or<'py>( @@ -87,64 +62,29 @@ fn config_or<'py>( Ok(value) } -#[pyfunction] -#[pyo3(signature = (**options))] +/// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. pub(crate) fn build_planner_entry_results<'py>( - py: pyo3::Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "build_planner_entry_results", - options, - ( - &[ - "runtime", - "warehouse", - "identities", - "overrides", - "policies", - "deferral", - "pruning", - "reconciliation", - "source_freshness", - ], - &[], - ), - )?; - let runtime = &args.value("runtime")?; - let warehouse = &args.value("warehouse")?; - let identities = &args.value("identities")?; - let overrides = &args.value("overrides")?; - let policies = &args.value("policies")?; - let deferral = &args.value("deferral")?; - let pruning = &args.value("pruning")?; - let reconciliation = &args.value("reconciliation")?; - let source_freshness = &args.value("source_freshness")?; - - let _ = (identities, policies); + (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (overrides, deferral): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (execution_scope, reconciliation): (&Bound<'py, PyAny>, &ChangeReconciliation<'py>), + source_freshness: &Bound<'py, PyAny>, +) -> PyResult> { let py = runtime.py(); - let project = runtime.getattr("project")?; - let config = runtime.getattr("project_config")?; - let execution_scope = pruning.getattr("execution_scope")?; - let relations = call_with( - ®istry::native(py, NativeAttr::BuildPlannerRelationsContext)?, - &[ - ("project", project.clone()), - ("adapter", runtime.getattr("adapter")?), - ("connection", runtime.getattr("connection")?), - ("scope", execution_scope.clone()), - ("deferral", deferral.clone()), - ("project_config", config.clone()), - ("local_config", runtime.getattr("local_config")?), - ( - "known_source_columns", - warehouse - .getattr("inspection_relations")? - .getattr("source_warehouse_columns")?, - ), - ], - )?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; + let config = runtime_field(runtime, |runtime| &runtime.project_config)?; + let relations = planner_relations_context(RelationsRequest { + project: project.clone(), + adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, + connection: runtime_field(runtime, |runtime| &runtime.connection)?, + scope: execution_scope.clone(), + deferral: deferral.clone(), + project_config: config.clone(), + local_config: runtime_field(runtime, |runtime| &runtime.local_config)?, + known_source_columns: warehouse + .getattr("inspection_relations")? + .getattr("source_warehouse_columns")?, + relation_keys: py.None().into_bound(py), + })?; let propagation = source_freshness.getattr("propagation")?; let blocked = if propagation.is_none() { PyFrozenSet::empty(py)?.into_any() @@ -190,7 +130,10 @@ pub(crate) fn build_planner_entry_results<'py>( "start_cursor_config", config_or(&config, &["cursors", "start"], none())?, ), - ("invocation_time", runtime.getattr("invocation_time")?), + ( + "invocation_time", + runtime_field(runtime, |runtime| &runtime.invocation_time)?, + ), ("max_microbatches", max_microbatches), ( "max_microbatches_is_override", @@ -202,26 +145,15 @@ pub(crate) fn build_planner_entry_results<'py>( ("microbatch_limit_action", limit_action), ], )?; - let entries = call_with( - ®istry::native(py, NativeAttr::BuildPlanEntries)?, - &[ - ("project", project), - ("adapter", runtime.getattr("adapter")?), - ("scope", execution_scope), - ("snapshot", warehouse.getattr("snapshot")?), - ("relations", relations), - ( - "resolved_actions", - reconciliation.getattr("resolved_actions")?, - ), - ("cursor_overrides", overrides.getattr("cursor_overrides")?), - ("full_refresh", overrides.getattr("full_refresh")?), - ("build_inputs", build_inputs), - ], - )?; - native( - py, - RecordId::PlannerEntryResults, - &[("model_entry_results", entries)], - ) + plan_entries(PlanEntriesRequest { + project, + adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, + scope: execution_scope.clone(), + snapshot: warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + relations, + resolved_models: reconciliation.resolved_actions.getattr("models")?, + cursor_overrides: overrides.getattr("cursor_overrides")?, + full_refresh: overrides.getattr("full_refresh")?.is_truthy()?, + build_inputs, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index be031788b3..8d08fb586b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -76,9 +76,9 @@ fn reusable_model_names( Ok(names) } -#[pyfunction] +#[pyfunction(name = "detect_stale_warning_changes")] #[pyo3(signature = (**options))] -pub(crate) fn detect_stale_warning_changes<'py>( +pub(crate) fn detect_stale_warning_changes_of<'py>( py: pyo3::Python<'py>, options: Option>, ) -> PyResult> { @@ -97,12 +97,22 @@ pub(crate) fn detect_stale_warning_changes<'py>( &[], ), )?; - let project = &args.value("project")?; - let scopes = &args.value("scopes")?; - let snapshot = &args.value("snapshot")?; - let identities = &args.value("identities")?; - let execution_changes = &args.value("execution_changes")?; + detect_stale_warning_changes( + (&args.value("project")?, &args.value("scopes")?), + &args.value("snapshot")?, + ( + &args.value("identities")?, + &args.value("execution_changes")?, + ), + ) +} +/// Changes of stale-warning models whose identities differ from the execution identities. +pub(crate) fn detect_stale_warning_changes<'py>( + (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + snapshot: &Bound<'py, PyAny>, + (identities, execution_changes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { let py = project.py(); let reusable = reusable_model_names( identities, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index e750afe4fb..b75082aa1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -10,5 +10,6 @@ pub(crate) mod objects; pub(crate) mod output_assembly; pub(crate) mod registration; pub(crate) mod retention; +pub(crate) mod runtime; pub(crate) mod scopes; pub(crate) mod warehouse_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index 0fb08904de..3df8d67d83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -1,6 +1,7 @@ //! Plan output fields gathered across the plan phases before the output is built once. -use pyo3::prelude::{Bound, PyAny}; +use pyo3::prelude::{Bound, Py, PyAny}; +use pyo3::pyclass; /// The plan output's fields and metadata entries, each set before the output is built. #[derive(Debug)] @@ -8,3 +9,45 @@ pub(crate) struct PlanOutputFields<'py> { pub(crate) fields: Vec<(&'static str, Bound<'py, PyAny>)>, pub(crate) metadata: Vec<(&'static str, Bound<'py, PyAny>)>, } + +/// The model changes and resolved actions after write-hash reconciliation. +#[derive(Debug)] +pub(crate) struct ChangeReconciliation<'py> { + pub(crate) changes: Bound<'py, PyAny>, + pub(crate) resolved_actions: Bound<'py, PyAny>, +} + +/// Planned model entries and the warnings planning them raised. +#[derive(Debug)] +pub(crate) struct ModelEntryResults<'py> { + pub(crate) entries: Vec>, + pub(crate) warnings: Vec>, +} + +/// Resolved execution environment shared by planner phases. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerRuntime { + pub(crate) project: Py, + pub(crate) adapter: Py, + pub(crate) connection: Py, + pub(crate) project_config: Py, + pub(crate) local_config: Py, + pub(crate) on_progress: Py, + pub(crate) invocation_time: Py, +} + +/// Warehouse snapshot, inspection relations and migration plans gathered once per plan. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerWarehouseState { + pub(crate) snapshot: Py, + pub(crate) inspection_relations: Py, + pub(crate) migration_entries: Py, + pub(crate) migration_warnings: Py, + pub(crate) column_migration_entries: Py, + pub(crate) column_rename_hints: Py, + pub(crate) old_name_view_entries: Py, + pub(crate) old_name_views: Py, + pub(crate) migration_fingerprints: Py, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 3bf915e1a4..e310d807c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -1,22 +1,23 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; -use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; -use crate::bindings::_helpers::planner_phases::models::PlanOutputFields; +use crate::bindings::_helpers::planner_phases::models::{ + ChangeReconciliation, ModelEntryResults, PlanOutputFields, +}; use crate::bindings::_helpers::planner_phases::objects::{ - model_key_names, native, replace, sorted_names, strings, text, tuple, + native, replace, sorted_names, strings, text, tuple, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use std::collections::BTreeSet; fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { let mut items: Vec<_> = Vec::new(); @@ -65,10 +66,10 @@ pub(crate) fn base_plan_output<'py>( &Bound<'py, PyAny>, &Bound<'py, PyAny>, ), - (pruning, reconciliation, entries, source_freshness): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, + (execution_scope, reconciliation, entries, source_freshness): ( &Bound<'py, PyAny>, + &ChangeReconciliation<'py>, + ModelEntryResults<'py>, &Bound<'py, PyAny>, ), planned_sql_tests: &Bound<'py, PyAny>, @@ -90,38 +91,23 @@ pub(crate) fn base_plan_output<'py>( ("planned_sql_tests", planned_sql_tests.clone()), ], )?; - let inputs = keywords( - py, - &[ - ("project", runtime.getattr("project")?), - ("adapter", runtime.getattr("adapter")?), - ("scope", pruning.getattr("execution_scope")?), - ("snapshot", warehouse.getattr("snapshot")?), - ("relations", warehouse.getattr("inspection_relations")?), - ("changes", reconciliation.getattr("changes")?), - ( - "model_entry_results", - entries.getattr("model_entry_results")?, - ), - ("reload_sources", overrides.getattr("reload_sources")?), - ("extras", extras), - ], - )?; let mut output = PlanOutputFields { - fields: plan_output_fields(&KeywordArguments::read( - py, - "build_plan_output", - Some(inputs), - (PLAN_OUTPUT_INPUTS, &["extras"]), - )?)?, + fields: plan_output_fields(PlanOutputInputs { + project: runtime_field(runtime, |runtime| &runtime.project)?, + adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, + scope: execution_scope.clone(), + snapshot: warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + relations: warehouse_field(warehouse, |warehouse| &warehouse.inspection_relations)?, + changes: reconciliation.changes.clone(), + model_entries: entries.entries, + model_warnings: entries.warnings, + reload_sources: overrides.getattr("reload_sources")?, + extras, + })?, metadata: Vec::new(), }; output.set("source_freshness", source_freshness.clone()); output.set("cursor_overrides", overrides.getattr("cursor_overrides")?); - let pruned = pruning.getattr("pruned_direct_model_names")?; - if pruned.is_truthy()? { - output.metadata.push(("direct_pruned_model_names", pruned)); - } Ok(output) } @@ -177,7 +163,10 @@ pub(crate) fn add_storage_policies<'py>( .is_truthy()? { let models = PyDict::new(py); - for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for model in runtime_field(runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let model = model?; models.set_item(model_reads::name_object(&model)?, model)?; } @@ -187,7 +176,10 @@ pub(crate) fn add_storage_policies<'py>( &[ ("entries", entries), ("models_by_name", models.into_any()), - ("fingerprints", warehouse.getattr("migration_fingerprints")?), + ( + "fingerprints", + warehouse_field(warehouse, |warehouse| &warehouse.migration_fingerprints)?, + ), ( "dialect", runtime @@ -197,15 +189,20 @@ pub(crate) fn add_storage_policies<'py>( ], )?; } - let entries = with_column_rename_hints(&entries, &warehouse.getattr("column_rename_hints")?)?; - let migration_entries = warehouse.getattr("migration_entries")?; - let column_entries = warehouse.getattr("column_migration_entries")?; - let old_name_entries = warehouse.getattr("old_name_view_entries")?; - let old_name_views = warehouse.getattr("old_name_views")?; + let entries = with_column_rename_hints( + &entries, + &warehouse_field(warehouse, |warehouse| &warehouse.column_rename_hints)?, + )?; + let migration_entries = warehouse_field(warehouse, |warehouse| &warehouse.migration_entries)?; + let column_entries = + warehouse_field(warehouse, |warehouse| &warehouse.column_migration_entries)?; + let old_name_entries = + warehouse_field(warehouse, |warehouse| &warehouse.old_name_view_entries)?; + let old_name_views = warehouse_field(warehouse, |warehouse| &warehouse.old_name_views)?; let warnings = concat( py, &[ - warehouse.getattr("migration_warnings")?, + warehouse_field(warehouse, |warehouse| &warehouse.migration_warnings)?, output.field("warnings")?, ], )?; @@ -237,11 +234,7 @@ pub(crate) fn add_plan_warnings<'py>( &Bound<'py, PyAny>, &Bound<'py, PyAny>, ), - (pruning, source_freshness, policies): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + (source_freshness, policies): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = scopes.py(); let freshness = source_freshness_unknown_warnings(source_freshness)?; @@ -257,9 +250,12 @@ pub(crate) fn add_plan_warnings<'py>( NativeAttr::BuildStaleOutOfSelectionWarnings, &[ ("original_scope", scopes.getattr("stale_warning_scope")?), - ("execution_scope", pruning.getattr("execution_scope")?), + ("execution_scope", scopes.getattr("inspection_scope")?), ("changes", stale_warning_changes.clone()), - ("snapshot", warehouse.getattr("snapshot")?), + ( + "snapshot", + warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + ), ( "version_identities", identities.getattr("stale_warning_identities")?, @@ -355,24 +351,10 @@ fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult( mut output: PlanOutputFields<'py>, - pruning: &Bound<'py, PyAny>, source_freshness: &Bound<'py, PyAny>, policies: &Bound<'py, PyAny>, ) -> PyResult> { - let py = pruning.py(); - let selected: BTreeSet = model_key_names( - &pruning - .getattr("inspection_scope")? - .getattr("selected_keys")?, - )? - .into_iter() - .collect(); - let remaining: Vec = strings(&pruning.getattr("direct_identity_stale_model_names")?)? - .into_iter() - .filter(|name| !selected.contains(name)) - .collect::>() - .into_iter() - .collect(); + let py = source_freshness.py(); let diagnostics = PyDict::new(py); diagnostics.set_item("mode", text(py, "direct"))?; diagnostics.set_item("enabled", policies.getattr("selection_diagnostics")?)?; @@ -382,7 +364,7 @@ pub(crate) fn add_plan_metadata<'py>( )); output.metadata.push(( "direct_remaining_stale_model_names", - sorted_names(py, remaining)?, + sorted_names(py, Vec::new())?, )); output .metadata diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 66fc997a07..3ed0682940 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -4,6 +4,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!( super::scopes::resolve_planner_scopes, module @@ -13,25 +15,13 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!( - super::identities::detect_stale_warning_changes, + super::identities::detect_stale_warning_changes_of, module )?)?; module.add_function(wrap_pyfunction!( super::buildability::check_selected_scope_buildability, module )?)?; - module.add_function(wrap_pyfunction!( - super::execution_changes::prune_planner_execution_scope, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::execution_changes::reconcile_execution_changes, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::execution_changes::build_planner_entry_results, - module - )?)?; module.add_function(wrap_pyfunction!( super::full_refresh::resolve_model_full_refresh, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index 0559d5227e..e58fa52e5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::planner_phases::objects::{ construct, member, model_key_names, native, native_member, optional_text, planner_error, planner_model, replace, sorted_names, text, tuple, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObject; @@ -30,8 +31,8 @@ pub(crate) fn effective_target_config<'py>( runtime: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let project_config = runtime.getattr("project_config")?; - let local_config = runtime.getattr("local_config")?; + let project_config = runtime_field(runtime, |runtime| &runtime.project_config)?; + let local_config = runtime_field(runtime, |runtime| &runtime.local_config)?; let target_name = runtime .getattr("project")? .getattr("effective_target_name")?; @@ -64,7 +65,10 @@ fn selected_models<'py>( .into_iter() .collect(); let mut models: Vec<_> = Vec::new(); - for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for model in runtime_field(runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let model = model?; if names.contains(&model_reads::model_name(&model)?) { models.push(model); @@ -119,7 +123,7 @@ pub(crate) fn plan_table_types<'py>( let relations = warehouse .getattr("snapshot")? .getattr("existing_relations")?; - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let materialization = registry::class(py, RecordId::MaterializationType)?; let mut entries: Vec<_> = Vec::new(); for model in models { @@ -282,9 +286,12 @@ fn inspect<'py>( request: &Bound<'py, PyAny>, ) -> PyResult> { call_with( - &runtime.getattr("adapter")?.getattr("inspect_retention")?, + &runtime_field(runtime, |runtime| &runtime.adapter)?.getattr("inspect_retention")?, &[ - ("connection", runtime.getattr("connection")?), + ( + "connection", + runtime_field(runtime, |runtime| &runtime.connection)?, + ), ("request", request.clone()), ], ) @@ -463,7 +470,10 @@ fn bigquery_retention<'py>( } type Scope = (Option, String); let mut managed: BTreeMap>> = BTreeMap::new(); - for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for model in runtime_field(runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let model = model?; let retention = model.getattr("config")?.getattr("time_travel_retention")?; let destination = model_reads::destination(&model)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs new file mode 100644 index 0000000000..96c8604552 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs @@ -0,0 +1,116 @@ +//! The planner runtime and warehouse state: built from keyword arguments and read by planner phases. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pymethods; +use pyo3::types::{PyDict, PyTuple}; + +use crate::bindings::_helpers::build_command::support::utc_now; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; +use crate::bindings::_helpers::records::ids::NativeAttr; +use crate::bindings::_helpers::records::registry; + +#[pymethods] +impl PlannerRuntime { + #[new] + #[pyo3(signature = (**kwargs))] + fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "PlannerRuntime", + kwargs, + ( + &["project", "adapter", "connection"], + &[ + "project_config", + "local_config", + "on_progress", + "invocation_time", + ], + ), + )?; + let invocation_time = if args.passed("invocation_time")? { + args.value("invocation_time")? + } else { + utc_now(py)? + }; + Ok(Self { + project: args.value("project")?.unbind(), + adapter: args.value("adapter")?.unbind(), + connection: args.value("connection")?.unbind(), + project_config: args.value("project_config")?.unbind(), + local_config: args.value("local_config")?.unbind(), + on_progress: args.value("on_progress")?.unbind(), + invocation_time: invocation_time.unbind(), + }) + } +} + +/// One field of a planner runtime passed as a Python object. +pub(crate) fn runtime_field<'py>( + runtime: &Bound<'py, PyAny>, + field: fn(&PlannerRuntime) -> &Py, +) -> PyResult> { + Ok(field(runtime.cast::()?.get()) + .bind(runtime.py()) + .clone()) +} + +#[pymethods] +impl PlannerWarehouseState { + #[new] + #[pyo3(signature = (**kwargs))] + fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "PlannerWarehouseState", + kwargs, + ( + &["snapshot", "inspection_relations"], + &[ + "migration_entries", + "migration_warnings", + "column_migration_entries", + "column_rename_hints", + "old_name_view_entries", + "old_name_views", + "migration_fingerprints", + ], + ), + )?; + let entries = |name: &str| -> PyResult> { + Ok(if args.passed(name)? { + args.value(name)?.unbind() + } else { + PyTuple::empty(py).into_any().unbind() + }) + }; + Ok(Self { + snapshot: args.value("snapshot")?.unbind(), + inspection_relations: args.value("inspection_relations")?.unbind(), + migration_entries: entries("migration_entries")?, + migration_warnings: entries("migration_warnings")?, + column_migration_entries: entries("column_migration_entries")?, + column_rename_hints: entries("column_rename_hints")?, + old_name_view_entries: entries("old_name_view_entries")?, + old_name_views: entries("old_name_views")?, + migration_fingerprints: if args.passed("migration_fingerprints")? { + args.value("migration_fingerprints")?.unbind() + } else { + registry::native(py, NativeAttr::MigrationFingerprintCache)? + .call0()? + .unbind() + }, + }) + } +} + +/// One field of a planner warehouse state passed as a Python object. +pub(crate) fn warehouse_field<'py>( + warehouse: &Bound<'py, PyAny>, + field: fn(&PlannerWarehouseState) -> &Py, +) -> PyResult> { + Ok(field(warehouse.cast::()?.get()) + .bind(warehouse.py()) + .clone()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index e06e19c04c..8627463617 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,12 +1,17 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. -use crate::bindings::_helpers::compile_driver::python_calls::keywords; -use crate::bindings::_helpers::planner_entries::relations::build_planner_relations_context; +use crate::bindings::_helpers::planner_entries::models::RelationsRequest; +use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; use crate::bindings::_helpers::planner_migrations::model_planning::plan_model_migrations; +use crate::bindings::_helpers::planner_migrations::models::{ + ColumnMigrationInputs, MigrationPhase, +}; use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; use crate::bindings::_helpers::planner_phases::objects::native; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; use crate::bindings::_helpers::records::ids::RecordId; @@ -16,7 +21,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use std::time::Instant; fn progress(runtime: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { - let on_progress = runtime.getattr("on_progress")?; + let on_progress = runtime_field(runtime, |runtime| &runtime.on_progress)?; if !on_progress.is_none() { on_progress.call1((message,))?; } @@ -33,7 +38,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ) -> PyResult> { let py = runtime.py(); let started = Instant::now(); - let project = runtime.getattr("project")?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; let fingerprints = Py::new( py, MigrationFingerprintCache::open(py, Some(project.getattr("compile_cache_dir")?))?, @@ -43,7 +48,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( progress(runtime, "Inspecting warehouse state...")?; let selected_scope = scopes.getattr("selected_scope")?; let full_refresh: bool = overrides.getattr("full_refresh")?.extract()?; - let config = runtime.getattr("project_config")?; + let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let cursor_scope = native( py, RecordId::CursorSnapshotScope, @@ -53,7 +58,10 @@ pub(crate) fn gather_planner_warehouse_state<'py>( "runtime_producer_keys", selected_scope.getattr("selected_keys")?, ), - ("invocation_time", runtime.getattr("invocation_time")?), + ( + "invocation_time", + runtime_field(runtime, |runtime| &runtime.invocation_time)?, + ), ( "start_cursor_config", if config.is_none() { @@ -65,7 +73,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ("cursor_overrides", overrides.getattr("cursor_overrides")?), ], )?; - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let stale_keys = pyo3::types::PyFrozenSet::new( py, scopes @@ -79,48 +87,37 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let snapshot = warehouse_snapshot(&SnapshotRequest { project: project.clone(), adapter: adapter.clone(), - connection: runtime.getattr("connection")?, + connection: runtime_field(runtime, |runtime| &runtime.connection)?, execute: adapter.getattr("execute")?, selected_keys: stale_keys, full_refresh: false, full_refresh_model_names: effectively_full_refreshed_model_names(&project, full_refresh)?, - on_progress: runtime.getattr("on_progress")?, + on_progress: runtime_field(runtime, |runtime| &runtime.on_progress)?, deferred_locations: deferral.getattr("deferred_locations")?, cursor_scope, })?; - let migrations = plan_model_migrations( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope.clone()), - ("snapshot", snapshot.clone()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ("fingerprints", fingerprints.clone()), - ], - )?), - )?; + let phase = MigrationPhase { + runtime: runtime.clone(), + scope: selected_scope.clone(), + snapshot: snapshot.clone(), + overrides: overrides.clone(), + deferral: deferral.clone(), + }; + let migrations = plan_model_migrations(&phase, &fingerprints)?; fingerprints.call_method0("persist")?; - let inspection_relations = build_planner_relations_context( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", runtime.getattr("connection")?), - ("scope", scopes.getattr("inspection_scope")?), - ("deferral", deferral.clone()), - ("project_config", config), - ("local_config", runtime.getattr("local_config")?), - ], - )?), - )?; + let inspection_relations = planner_relations_context(RelationsRequest { + project: project.clone(), + adapter: adapter.clone(), + connection: runtime_field(runtime, |runtime| &runtime.connection)?, + scope: scopes.getattr("inspection_scope")?, + deferral: deferral.clone(), + project_config: config, + local_config: runtime_field(runtime, |runtime| &runtime.local_config)?, + known_source_columns: py.None().into_bound(py), + relation_keys: py.None().into_bound(py), + })?; let physical = PyDict::new(py); - for entry in migrations.getattr("entries")?.try_iter()? { - let entry = entry?; + for entry in &migrations.entries { if entry .getattr("decision")? .getattr("moves_data")? @@ -139,41 +136,32 @@ pub(crate) fn gather_planner_warehouse_state<'py>( } } let columns = plan_column_migrations( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope.clone()), - ("snapshot", migrations.getattr("snapshot")?), - ( - "full_refresh_model_names", - effectively_full_refreshed_model_names(&project, full_refresh)?, - ), - ("physical_relations", physical.into_any()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ( - "source_columns", - inspection_relations.getattr("source_warehouse_columns")?, - ), - ], - )?), + &MigrationPhase { + snapshot: migrations.snapshot.clone(), + ..phase + }, + &ColumnMigrationInputs { + full_refresh_model_names: effectively_full_refreshed_model_names( + &project, + full_refresh, + )?, + physical_relations: physical.into_any(), + source_columns: inspection_relations.getattr("source_warehouse_columns")?, + }, )?; + let migration_entries = PyTuple::new(py, &migrations.entries)?.into_any(); + let column_entries = PyTuple::new(py, &columns.entries)?.into_any(); let old_names = plan_old_name_migration_views( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope), - ("snapshot", snapshot), - ("migration_entries", migrations.getattr("entries")?), - ("column_entries", columns.getattr("entries")?), - ], - )?), + &MigrationPhase { + runtime: runtime.clone(), + scope: selected_scope, + snapshot, + overrides: overrides.clone(), + deferral: deferral.clone(), + }, + (&migration_entries, &column_entries), )?; - if !runtime.getattr("on_progress")?.is_none() { + if !runtime_field(runtime, |runtime| &runtime.on_progress)?.is_none() { progress( runtime, &format!( @@ -183,25 +171,22 @@ pub(crate) fn gather_planner_warehouse_state<'py>( )?; progress(runtime, "Generating plan...")?; } - let mut warnings: Vec<_> = Vec::new(); - for source in [&migrations, &columns, &old_names] { - for warning in source.getattr("warnings")?.try_iter()? { - warnings.push(warning?); - } - } - native( + let mut warnings: Vec<_> = migrations.warnings; + warnings.extend(columns.warnings); + warnings.extend(old_names.warnings); + Ok(Bound::new( py, - RecordId::PlannerWarehouseState, - &[ - ("snapshot", columns.getattr("snapshot")?), - ("inspection_relations", inspection_relations), - ("migration_entries", migrations.getattr("entries")?), - ("migration_warnings", PyTuple::new(py, warnings)?.into_any()), - ("column_migration_entries", columns.getattr("entries")?), - ("column_rename_hints", columns.getattr("hints")?), - ("old_name_view_entries", old_names.getattr("entries")?), - ("old_name_views", old_names.getattr("views")?), - ("migration_fingerprints", fingerprints), - ], - ) + PlannerWarehouseState { + snapshot: columns.snapshot.unbind(), + inspection_relations: inspection_relations.unbind(), + migration_entries: migration_entries.unbind(), + migration_warnings: PyTuple::new(py, warnings)?.into_any().unbind(), + column_migration_entries: column_entries.unbind(), + column_rename_hints: PyTuple::new(py, columns.hints)?.into_any().unbind(), + old_name_view_entries: old_names.entries.unbind(), + old_name_views: old_names.views.unbind(), + migration_fingerprints: fingerprints.unbind(), + }, + )? + .into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index a77e1e89d3..d419863cd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -1,9 +1,8 @@ //! Layouts of the planner, migration, cursor and fingerprint value records hosted natively. -use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::record_behaviours::planner::forward_only_backfill; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::models::{EnumSpec, FieldDefault, FieldSpec, RecordSpec}; const R_PARSED_SELECTOR: RecordSpec = RecordSpec { @@ -1254,14 +1253,6 @@ const R_DIRECT_MODEL_VERSION_IDENTITIES: RecordSpec = RecordSpec { FieldSpec::required("model_version_hashes"), ], }; -const R_PLANNER_MODEL_ENTRY_RESULTS: RecordSpec = RecordSpec { - name: "PlannerModelEntryResults", - doc: "Model plan-entry phase output.", - fields: &[ - FieldSpec::required("entries"), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - ], -}; const R_SCHEMA_ACTION: RecordSpec = RecordSpec { name: "SchemaAction", doc: "One concrete schema change action to apply to a target table.", @@ -1403,19 +1394,6 @@ const R_PLAN_PROVIDER_USAGE: RecordSpec = RecordSpec { FieldSpec::new("annotation_module", FieldDefault::None), ], }; -const R_PLANNER_RUNTIME: RecordSpec = RecordSpec { - name: "PlannerRuntime", - doc: "Resolved execution environment shared by planner phases.", - fields: &[ - FieldSpec::required("project"), - FieldSpec::required("adapter"), - FieldSpec::required("connection"), - FieldSpec::new("project_config", FieldDefault::None), - FieldSpec::new("local_config", FieldDefault::None), - FieldSpec::new("on_progress", FieldDefault::None), - FieldSpec::new("invocation_time", FieldDefault::Factory(utc_now)), - ], -}; const R_PLANNER_SCOPE_RESOLUTION: RecordSpec = RecordSpec { name: "PlannerScopeResolution", doc: "Resolved planner scopes for one plan build.", @@ -1425,24 +1403,6 @@ const R_PLANNER_SCOPE_RESOLUTION: RecordSpec = RecordSpec { FieldSpec::required("inspection_scope"), ], }; -const R_PLANNER_WAREHOUSE_STATE: RecordSpec = RecordSpec { - name: "PlannerWarehouseState", - doc: "Warehouse snapshot and inspection relations gathered once per plan.", - fields: &[ - FieldSpec::required("snapshot"), - FieldSpec::required("inspection_relations"), - FieldSpec::new("migration_entries", FieldDefault::EmptyTuple), - FieldSpec::new("migration_warnings", FieldDefault::EmptyTuple), - FieldSpec::new("column_migration_entries", FieldDefault::EmptyTuple), - FieldSpec::new("column_rename_hints", FieldDefault::EmptyTuple), - FieldSpec::new("old_name_view_entries", FieldDefault::EmptyTuple), - FieldSpec::new("old_name_views", FieldDefault::EmptyTuple), - FieldSpec::new( - "migration_fingerprints", - FieldDefault::NativeClassCall(NativeAttr::MigrationFingerprintCache), - ), - ], -}; const R_MODEL_MIGRATION_REQUEST: RecordSpec = RecordSpec { name: "ModelMigrationRequest", doc: "One destination model whose data should come from an earlier relation.", @@ -1482,34 +1442,6 @@ const R_MIGRATION_COMPATIBILITY_RESULT: RecordSpec = RecordSpec { FieldSpec::new("findings", FieldDefault::EmptyTuple), ], }; -const R_MODEL_MIGRATION_PLANNING: RecordSpec = RecordSpec { - name: "ModelMigrationPlanning", - doc: "Migration decisions plus the warehouse snapshot as it will look after moves.", - fields: &[ - FieldSpec::required("snapshot"), - FieldSpec::new("entries", FieldDefault::EmptyTuple), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - ], -}; -const R_OLD_NAME_VIEW_PLANNING: RecordSpec = RecordSpec { - name: "OldNameViewPlanning", - doc: "Old-name steps for this build plus every compatibility view still recorded.", - fields: &[ - FieldSpec::new("entries", FieldDefault::EmptyTuple), - FieldSpec::new("views", FieldDefault::EmptyTuple), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - ], -}; -const R_COLUMN_MIGRATION_PLANNING: RecordSpec = RecordSpec { - name: "ColumnMigrationPlanning", - doc: "Column rename decisions plus the warehouse snapshot as it will look after renames.", - fields: &[ - FieldSpec::required("snapshot"), - FieldSpec::new("entries", FieldDefault::EmptyTuple), - FieldSpec::new("hints", FieldDefault::EmptyTuple), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - ], -}; const R_PLANNER_IDENTITY_CONTEXT: RecordSpec = RecordSpec { name: "PlannerIdentityContext", doc: "Expected version identities for inspection and stale-warning scopes.", @@ -1518,30 +1450,6 @@ const R_PLANNER_IDENTITY_CONTEXT: RecordSpec = RecordSpec { FieldSpec::required("stale_warning_identities"), ], }; -const R_PLANNER_SCOPE_PRUNING_RESULT: RecordSpec = RecordSpec { - name: "PlannerScopePruningResult", - doc: "Scope pruning outcome with pruned scopes, actions, and staleness state.", - fields: &[ - FieldSpec::required("inspection_scope"), - FieldSpec::required("execution_scope"), - FieldSpec::required("resolved_actions"), - FieldSpec::required("pruned_direct_model_names"), - FieldSpec::required("direct_identity_stale_model_names"), - ], -}; -const R_PLANNER_CHANGE_RECONCILIATION: RecordSpec = RecordSpec { - name: "PlannerChangeReconciliation", - doc: "Write-hash-honest changes merged back into resolved actions.", - fields: &[ - FieldSpec::required("changes"), - FieldSpec::required("resolved_actions"), - ], -}; -const R_PLANNER_ENTRY_RESULTS: RecordSpec = RecordSpec { - name: "PlannerEntryResults", - doc: "Model plan entries for one plan build.", - fields: &[FieldSpec::required("model_entry_results")], -}; /// Planner value records, in registration order. pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ @@ -1569,7 +1477,6 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_RESOLVED_MODEL_ACTION, R_PLANNER_RESOLVED_ACTIONS, R_DIRECT_MODEL_VERSION_IDENTITIES, - R_PLANNER_MODEL_ENTRY_RESULTS, R_SCHEMA_ACTION, R_PLAN_WARNING, R_COLUMN_RENAME_HINT, @@ -1583,20 +1490,12 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_SCENARIO_ARTIFACT_IDENTITY, R_SCENARIO_ARTIFACT_NAME, R_PLAN_PROVIDER_USAGE, - R_PLANNER_RUNTIME, R_PLANNER_SCOPE_RESOLUTION, - R_PLANNER_WAREHOUSE_STATE, R_MODEL_MIGRATION_REQUEST, R_MODEL_MIGRATION_DECLARATION, R_MODEL_MIGRATION_DISCOVERY, R_MIGRATION_COMPATIBILITY_RESULT, - R_MODEL_MIGRATION_PLANNING, - R_OLD_NAME_VIEW_PLANNING, - R_COLUMN_MIGRATION_PLANNING, R_PLANNER_IDENTITY_CONTEXT, - R_PLANNER_SCOPE_PRUNING_RESULT, - R_PLANNER_CHANGE_RECONCILIATION, - R_PLANNER_ENTRY_RESULTS, ]; /// Planner string enums, in declaration order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 9626282229..6d8260bb67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -1,7 +1,7 @@ //! Model cursor orchestration over live warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::model_config::config_values::{ config_cursor_bound, config_str as authored_config_str, }; @@ -12,12 +12,12 @@ use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ future_cursor_config, start_cursor_config, }; +use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; use crate::bindings::_helpers::planner_resolution::values::string; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::{IntoPyObject, PyErr}; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { let values: Bound<'py, PyAny> = model.getattr("config")?.getattr("values")?; @@ -65,35 +65,16 @@ fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> && target.ne(model_reads::destination(model)?.getattr("qualified_name")?)?) } -#[pyfunction] -#[pyo3(signature = (**kwargs))] +/// One model's cursor bounds from its snapshot, overrides and cursor policies; `None` when unbounded by a full refresh. pub(crate) fn compute_model_cursor_bounds<'py>( - py: Python<'py>, - kwargs: Option>, + request: &ModelSqlRequest<'py>, + context: &Bound<'py, PyAny>, ) -> PyResult> { - let args = KeywordArguments::read( - py, - "compute_model_cursor_bounds", - kwargs, - ( - &[ - "model", - "snapshot", - "backfill", - "full_refresh", - "start_cursor_override", - "end_cursor_override", - "runtime_cursor_producer_names", - "context", - ], - &["replaces_relation"], - ), - )?; - let model = args.value("model")?; + let model = request.model.clone(); + let py = model.py(); if !config(&model, "materialized")?.eq("incremental")? || config(&model, "cursor")?.is_none() { return Ok(py.None().into_bound(py)); } - let context = args.value("context")?; let incremental_mode = config(&model, "incremental_mode")?; let future_config = future_cursor_config(&model, &context.getattr("future_cursor_config")?)?; let start_config = start_cursor_config(&model, &context.getattr("start_cursor_config")?)?; @@ -102,8 +83,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( } let cursor_type = config(&model, "cursor_type")?; let cursor_grain = config(&model, "cursor_grain")?; - let start_override = args.value("start_cursor_override")?; - let end_override = args.value("end_cursor_override")?; + let (start_override, end_override) = request.cursor_overrides.clone(); let bounded = call_with( ®istry::native(py, NativeAttr::ResolveBoundedCursorOverride)?, &[ @@ -116,7 +96,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( if !bounded.is_none() { return Ok(bounded); } - let snapshot = args.value("snapshot")?; + let snapshot = &request.snapshot; let mut cursor_snapshot = snapshot .getattr("cursor_snapshots")? .call_method1("get", (model_reads::name_object(&model)?,))?; @@ -124,7 +104,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, &[("model", model.clone())], )?; - let producers = args.value("runtime_cursor_producer_names")?; + let producers = context.getattr("runtime_cursor_producer_names")?; let mut runtime_owned = false; for name in roles.getattr("watermark_inputs")?.try_iter()? { if producers.contains(name?)? { @@ -132,8 +112,8 @@ pub(crate) fn compute_model_cursor_bounds<'py>( break; } } - let full_refresh = args.flag("full_refresh")?; - let replaces = args.flag("replaces_relation")?; + let full_refresh = request.full_refresh; + let replaces = request.replaces_relation; if !full_refresh && !runtime_owned && !cursor_snapshot.is_none() { let empty = call_with( ®istry::native(py, NativeAttr::CursorInputsWithoutRows)?, @@ -142,7 +122,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( .getattr("input_names")?; if empty.is_truthy()? && replaces - && (destination_exists(&model, &snapshot)? + && (destination_exists(&model, snapshot)? || history_redirected(&model, &cursor_snapshot)?) { let refusal = call_with( @@ -206,7 +186,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( &[("cursor_snapshot", cursor_snapshot)], )?; } - let backfill = args.value("backfill")?; + let backfill = &request.backfill; let duration = if backfill.getattr("action")?.eq("bounded")? { backfill.getattr("duration")? } else { @@ -264,7 +244,3 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ], ) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compute_model_cursor_bounds, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 8c069d8dd4..90173837c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -13,9 +13,9 @@ use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::planner::main::override_kind::override_kind; -#[pyfunction] +#[pyfunction(name = "type_cursor_overrides")] #[pyo3(signature = (**kwargs))] -pub(crate) fn type_cursor_overrides<'py>( +fn type_cursor_overrides_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -25,18 +25,27 @@ pub(crate) fn type_cursor_overrides<'py>( kwargs, (&["cursor_overrides"], &["selected_models"]), )?; - let overrides = args.value("cursor_overrides")?; - if overrides.is_none() || !overrides.getattr("has_untyped")?.is_truthy()? { - return Ok(overrides); - } - let mut models: Vec<(String, String)> = Vec::new(); - let selected = if args.passed("selected_models")? { + let selected: Vec> = if args.passed("selected_models")? { args.value("selected_models")? + .try_iter()? + .collect::>()? } else { - pyo3::types::PyTuple::empty(py).into_any() + Vec::new() }; - for model in selected.try_iter()? { - let model = model?; + type_cursor_overrides(&args.value("cursor_overrides")?, &selected) +} + +/// Untyped cursor overrides typed by the cursor type of the selected incremental models. +pub(crate) fn type_cursor_overrides<'py>( + overrides: &Bound<'py, PyAny>, + selected: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = overrides.py(); + if overrides.is_none() || !overrides.getattr("has_untyped")?.is_truthy()? { + return Ok(overrides.clone()); + } + let mut models: Vec<(String, String)> = Vec::new(); + for model in selected { let values = model.getattr("config")?.getattr("values")?; let config = |key: &str| -> PyResult> { Ok(authored_config_str(&values, key)?.unwrap_or_else(|| py.None().into_bound(py))) @@ -46,10 +55,7 @@ pub(crate) fn type_cursor_overrides<'py>( } let kind = config("cursor_type")?; if kind.eq("integer")? || kind.eq("timestamp")? { - models.push(( - model_reads::name_object(&model)?.extract()?, - kind.extract()?, - )); + models.push((model_reads::name_object(model)?.extract()?, kind.extract()?)); } } let mut integer_literals = true; @@ -75,9 +81,9 @@ pub(crate) fn type_cursor_overrides<'py>( ])) { Ok(error) => PyErr::from_value(error), Err(error) => error } })?; let kind = registry::class(py, RecordId::CursorType)?.call1((kind,))?; - method(&overrides, "typed_as", &[("cursor_type", kind)]) + method(overrides, "typed_as", &[("cursor_type", kind)]) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(type_cursor_overrides, module)?) + module.add_function(wrap_pyfunction!(type_cursor_overrides_of, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index d5f276315b..063cba97d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -2,7 +2,7 @@ mod constants; mod cursor_bounds; -mod cursor_model; +pub(crate) mod cursor_model; pub(crate) mod cursor_overrides; pub(crate) mod cursor_policies; pub(crate) mod cursor_ranges; @@ -12,6 +12,7 @@ mod cursor_windows; mod future_safety; mod locations; mod maximum_safety; +pub(crate) mod models; pub(crate) mod orchestration; mod refs; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs new file mode 100644 index 0000000000..db9d2e26f5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs @@ -0,0 +1,24 @@ +//! Typed requests of planner SQL resolution. + +use pyo3::prelude::{Bound, PyAny}; + +/// The cursor window that filters the cursor inputs one resolution reads. +#[derive(Debug)] +pub(crate) struct CursorFiltering<'py> { + pub(crate) bounds: Bound<'py, PyAny>, + pub(crate) filter_inputs: Bound<'py, PyAny>, + pub(crate) cursor_type: Bound<'py, PyAny>, + pub(crate) lower_bound_inclusive: bool, +} + +/// One model's SQL resolution decisions. +#[derive(Debug)] +pub(crate) struct ModelSqlRequest<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) backfill: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) replaces_relation: bool, + pub(crate) full_rebuild_cause: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index f4f0192106..81d9bbcea8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -5,29 +5,29 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::resolve_cursor_in use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, keywords, + KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_resolution::cursor_model::compute_model_cursor_bounds; +use crate::bindings::_helpers::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; use crate::bindings::_helpers::planner_resolution::refs::{ - resolve_dbt_ref_references, resolve_ref_references, resolve_table_function_references, + resolve_dbt_ref_references, resolve_refs, resolve_table_function_references, resolve_udf_references, }; -use crate::bindings::_helpers::planner_resolution::sources::resolve_source_references; +use crate::bindings::_helpers::planner_resolution::sources::resolve_sources; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{PyErr, pyfunction, wrap_pyfunction}; /// What resolving one model's, hook's or function's references reads. -struct Resolution<'py> { - adapter: Bound<'py, PyAny>, +pub(crate) struct Resolution<'py> { + pub(crate) adapter: Bound<'py, PyAny>, external: Bound<'py, PyAny>, source_map: Bound<'py, PyAny>, source_warehouse_columns: Bound<'py, PyAny>, - star_exclude_keyword: Bound<'py, PyAny>, model_locations: Bound<'py, PyAny>, seed_locations: Bound<'py, PyAny>, function_locations: Bound<'py, PyAny>, @@ -35,7 +35,7 @@ struct Resolution<'py> { impl<'py> Resolution<'py> { /// Resolution against a `ModelPlanContext`'s relations. - fn of_context( + pub(crate) fn of_context( adapter: Bound<'py, PyAny>, context: &Bound<'py, PyAny>, external: Bound<'py, PyAny>, @@ -45,39 +45,26 @@ impl<'py> Resolution<'py> { external, source_map: context.getattr("source_map")?, source_warehouse_columns: context.getattr("source_warehouse_columns")?, - star_exclude_keyword: context.getattr("star_exclude_keyword")?, model_locations: context.getattr("model_locations")?, seed_locations: context.getattr("seed_locations")?, function_locations: context.getattr("function_locations")?, }) } - fn resolve( - &self, - sql: Bound<'py, PyAny>, - cursor: &[(&str, Bound<'py, PyAny>)], - ) -> PyResult { + fn resolve(&self, sql: &str, filtering: &CursorFiltering<'py>) -> PyResult { let py = self.adapter.py(); - let mut source = vec![ - ("query_sql", sql), - ("adapter", self.adapter.clone()), - ("source_map", self.source_map.clone()), - ( - "source_warehouse_columns", - self.source_warehouse_columns.clone(), - ), - ("star_exclude_keyword", self.star_exclude_keyword.clone()), - ]; - source.extend_from_slice(cursor); - let resolved: String = resolve_source_references(py, Some(keywords(py, &source)?))?; - let mut refs = vec![ - ("query_sql", PyString::new(py, &resolved).into_any()), - ("adapter", self.adapter.clone()), - ("model_locations", self.model_locations.clone()), - ("seed_locations", self.seed_locations.clone()), - ]; - refs.extend_from_slice(cursor); - let resolved: String = resolve_ref_references(py, Some(keywords(py, &refs)?))?; + let resolved: String = resolve_sources( + sql, + (&self.source_map, &self.source_warehouse_columns), + &self.adapter, + filtering, + )?; + let resolved: String = resolve_refs( + &resolved, + (&self.model_locations, &self.seed_locations), + &self.adapter, + filtering, + )?; let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; let locations = if self.function_locations.is_truthy()? { self.function_locations.clone() @@ -100,16 +87,13 @@ impl<'py> Resolution<'py> { fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { let py = self.adapter.py(); let resolved = self.resolve( - sql, - &[ - ("cursor_bounds", py.None().into_bound(py)), - ("cursor_filter_inputs", PyDict::new(py).into_any()), - ("cursor_type", py.None().into_bound(py)), - ( - "lower_bound_inclusive", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], + &sql.extract::()?, + &CursorFiltering { + bounds: py.None().into_bound(py), + filter_inputs: PyDict::new(py).into_any(), + cursor_type: py.None().into_bound(py), + lower_bound_inclusive: true, + }, )?; self.assert_resolved(&resolved, label)?; Ok(resolved) @@ -141,9 +125,9 @@ impl<'py> Resolution<'py> { } } -#[pyfunction] +#[pyfunction(name = "resolve_model_sql")] #[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_model_sql<'py>( +fn resolve_model_sql_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -168,79 +152,76 @@ pub(crate) fn resolve_model_sql<'py>( ], ), )?; - let model = args.value("model")?; let context = args.value("context")?; let resolution = Resolution::of_context( args.value("adapter")?, &context, args.value("external_sql_reference_resolver")?, )?; + let overrides = args.value("cursor_overrides")?; + let cause = args.value("full_rebuild_cause")?; + let request = ModelSqlRequest { + model: args.value("model")?, + snapshot: args.value("snapshot")?, + backfill: args.value("backfill")?, + full_refresh: args.flag("full_refresh")?, + cursor_overrides: ( + overrides.getattr("start_cursor_override")?, + overrides.getattr("end_cursor_override")?, + ), + replaces_relation: args.flag("replaces_relation")?, + full_rebuild_cause: if cause.is_truthy()? { + Some(cause.extract()?) + } else { + None + }, + }; + let sql: String = resolve_model_sql(&resolution, &context, &request)?; + Ok(PyString::new(py, &sql).into_any()) +} + +/// One model's planned SQL: references resolved, cursor inputs filtered and intrinsics bound. +pub(crate) fn resolve_model_sql<'py>( + resolution: &Resolution<'py>, + context: &Bound<'py, PyAny>, + request: &ModelSqlRequest<'py>, +) -> PyResult { + let model = &request.model; + let py = model.py(); let cursor_type = authored_config_str(&model.getattr("config")?.getattr("values")?, "cursor_type")? .unwrap_or_else(|| py.None().into_bound(py)); - let overrides = args.value("cursor_overrides")?; - let bounds = compute_model_cursor_bounds( - py, - Some(keywords( - py, - &[ - ("model", model.clone()), - ("snapshot", args.value("snapshot")?), - ("backfill", args.value("backfill")?), - ("full_refresh", args.value("full_refresh")?), - ( - "start_cursor_override", - overrides.getattr("start_cursor_override")?, - ), - ( - "end_cursor_override", - overrides.getattr("end_cursor_override")?, - ), - ( - "runtime_cursor_producer_names", - context.getattr("runtime_cursor_producer_names")?, - ), - ("context", context.clone()), - ( - "replaces_relation", - args.value_or_false("replaces_relation")?, - ), - ], - )?), - )?; - let roles = resolve_cursor_input_roles(&model)?; - let inclusive = append_cursor_inclusive(&model)?; + let bounds = compute_model_cursor_bounds(request, context)?; + let roles = resolve_cursor_input_roles(model)?; + let filtering = CursorFiltering { + bounds: bounds.clone(), + filter_inputs: roles.getattr("filter_inputs")?, + cursor_type: cursor_type.clone(), + lower_bound_inclusive: append_cursor_inclusive(model)?, + }; let mut sql = resolution.resolve( - model_reads::query_sql(&model)?, - &[ - ("cursor_bounds", bounds.clone()), - ("cursor_filter_inputs", roles.getattr("filter_inputs")?), - ("cursor_type", cursor_type.clone()), - ("lower_bound_inclusive", inclusive), - ], + &model_reads::query_sql(model)?.extract::()?, + &filtering, )?; let (_, uses_intrinsics) = resolve_cursor_intrinsics(py, &sql, None, None)?; - let name: String = model_reads::model_name(&model)?; + let name: String = model_reads::model_name(model)?; if uses_intrinsics { if bounds.is_none() { - let cause = args.value("full_rebuild_cause")?; - let cause = if cause.is_truthy()? { - Some(cause.extract::()?) - } else if args.flag("full_refresh")? { - Some("--full-refresh".to_owned()) - } else { - None + let cause = match &request.full_rebuild_cause { + Some(cause) => Some(cause.clone()), + None if request.full_refresh => Some("--full-refresh".to_owned()), + None => None, }; return Err(unresolved_bounds_error(py, &name, cause.as_deref())?); } sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.adapter)?; } resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; - Ok(PyString::new(py, &sql).into_any()) + Ok(sql) } /// Effective append cursor lower-bound inclusivity. -fn append_cursor_inclusive<'py>(model: &Bound<'py, PyAny>) -> PyResult> { +fn append_cursor_inclusive(model: &Bound<'_, PyAny>) -> PyResult { let py = model.py(); let strategy = authored_config_str( &model.getattr("config")?.getattr("values")?, @@ -251,9 +232,7 @@ fn append_cursor_inclusive<'py>(model: &Bound<'py, PyAny>) -> PyResult() || raw.is_truthy()?; - Ok(inclusive.into_pyobject(py)?.to_owned().into_any()) + Ok(!strategy.eq("append")? || !raw.is_instance_of::() || raw.is_truthy()?) } /// `__cursor_start()` / `__cursor_end()` replaced with the adapter's typed bound literals. @@ -325,7 +304,6 @@ fn resolve_function_sql<'py>( external: py.None().into_bound(py), source_map: args.value("source_map")?, source_warehouse_columns: args.value("source_warehouse_columns")?, - star_exclude_keyword: args.value("star_exclude_keyword")?, model_locations: args.value("model_locations")?, seed_locations: args.value("seed_locations")?, function_locations: args.value("function_locations")?, @@ -339,13 +317,21 @@ fn resolve_function_sql<'py>( #[pyfunction] #[pyo3(signature = (*, model, adapter, context, external_sql_reference_resolver))] -pub(crate) fn resolve_model_hook_entries<'py>( +fn resolve_model_hook_entries<'py>( model: &Bound<'py, PyAny>, adapter: Bound<'py, PyAny>, context: Bound<'py, PyAny>, external_sql_reference_resolver: Bound<'py, PyAny>, ) -> PyResult> { let resolution = Resolution::of_context(adapter, &context, external_sql_reference_resolver)?; + model_hook_entries(model, &resolution) +} + +/// One model's pre and post hooks with their SQL resolved. +pub(crate) fn model_hook_entries<'py>( + model: &Bound<'py, PyAny>, + resolution: &Resolution<'py>, +) -> PyResult> { let values = model.getattr("config")?.getattr("values")?; let name: String = model_reads::model_name(model)?; let mut hooks: Vec> = Vec::new(); @@ -359,7 +345,7 @@ pub(crate) fn resolve_model_hook_entries<'py>( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_model_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_sql_of, module)?)?; module.add_function(wrap_pyfunction!(resolve_function_sql, module)?)?; module.add_function(wrap_pyfunction!(resolve_model_hook_entries, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs index 63475c1b12..a614a1269d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs @@ -3,7 +3,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::planner_resolution::values::{cursor_wrap, method, string}; +use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; +use crate::bindings::_helpers::planner_resolution::values::{ + cursor_wrap, filtering_of, method, string, +}; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObject; @@ -14,10 +17,7 @@ use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans:: #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_ref_references( - py: Python<'_>, - kwargs: Option>, -) -> PyResult { +fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> PyResult { let args = KeywordArguments::read( py, "resolve_ref_references", @@ -36,10 +36,27 @@ pub(crate) fn resolve_ref_references( &[], ), )?; - let original: String = args.value("query_sql")?.extract()?; - let mut sql = original.clone(); - for (kind, key) in [("ref", "model_locations"), ("seed", "seed_locations")] { - let locations = args.value(key)?; + resolve_refs( + &args.value("query_sql")?.extract::()?, + ( + &args.value("model_locations")?, + &args.value("seed_locations")?, + ), + &args.value("adapter")?, + &filtering_of(&args)?, + ) +} + +/// `__ref` and `__seed` references replaced with their relations, cursor inputs filtered. +pub(crate) fn resolve_refs( + original: &str, + (model_locations, seed_locations): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + adapter: &Bound<'_, PyAny>, + filtering: &CursorFiltering<'_>, +) -> PyResult { + let py = adapter.py(); + let mut sql = original.to_owned(); + for (kind, locations) in [("ref", model_locations), ("seed", seed_locations)] { let mut output = String::new(); let mut last = 0; for span in reference_spans(&sql, kind) { @@ -50,7 +67,7 @@ pub(crate) fn resolve_ref_references( } else { let relation: String = call_with( &host::entry(py, AdapterHost::ResolveRelationLocationQualifiedName)?, - &[("adapter", args.value("adapter")?), ("location", target)], + &[("adapter", adapter.clone()), ("location", target)], )? .extract()?; let char_end = sql[..span.end].chars().count(); @@ -59,10 +76,10 @@ pub(crate) fn resolve_ref_references( .nth(char_end) .map_or(original.len(), |(offset, _)| offset); output.push_str(&cursor_wrap( - &args, + adapter, + filtering, relation, - &span.name, - !ref_alias(&original, original_end), + (&span.name, !ref_alias(original, original_end)), )?); } last = span.end; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index 94a90504d0..5433bfbe90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -4,7 +4,6 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::cursor_bounds::register(module)?; - super::cursor_model::register(module)?; super::cursor_overrides::register(module)?; super::cursor_policies::register(module)?; super::cursor_ranges::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index d6d44f8596..7e9c530b0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -1,7 +1,10 @@ //! Source contract validation and adapter-controlled projection rendering. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::planner_resolution::values::{cursor_wrap, method, string}; +use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; +use crate::bindings::_helpers::planner_resolution::values::{ + cursor_wrap, filtering_of, method, string, +}; use crate::bindings::_helpers::planner_strategy::values::input_error; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::references::source_relations::render_source_relation; @@ -245,16 +248,19 @@ fn render_source_read_relation( render_source_relation(source_entry, Some(adapter)) } -fn resolve_one(args: &KeywordArguments<'_>, name: &str, original: &str) -> PyResult { - let entry = args.value("source_map")?.call_method1("get", (name,))?; +fn resolve_one( + (source_map, warehouse_columns): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + adapter: &Bound<'_, PyAny>, + filtering: &CursorFiltering<'_>, + (name, original): (&str, &str), +) -> PyResult { + let entry = source_map.call_method1("get", (name,))?; if entry.is_none() { return Ok(original.to_owned()); } - let adapter = args.value("adapter")?; + let adapter = adapter.clone(); let mut relation = render_source_read_relation(&adapter, &entry)?; - let columns = args - .value("source_warehouse_columns")? - .call_method1("get", (name,))?; + let columns = warehouse_columns.call_method1("get", (name,))?; let expression = !entry.getattr("expression")?.is_none(); if columns.is_truthy()? && (!expression @@ -274,12 +280,12 @@ fn resolve_one(args: &KeywordArguments<'_>, name: &str, original: &str) -> PyRes if entry.getattr("type_enforcement")?.is_truthy()? { relation = cast_source(&entry, &columns, &adapter, relation)?; } - cursor_wrap(args, relation, name, false) + cursor_wrap(&adapter, filtering, relation, (name, false)) } #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_source_references( +fn resolve_source_references( py: Python<'_>, kwargs: Option>, ) -> PyResult { @@ -302,19 +308,40 @@ pub(crate) fn resolve_source_references( &[], ), )?; - let sql: String = args.value("query_sql")?.extract()?; - let aliases = args - .value("adapter")? + resolve_sources( + &args.value("query_sql")?.extract::()?, + ( + &args.value("source_map")?, + &args.value("source_warehouse_columns")?, + ), + &args.value("adapter")?, + &filtering_of(&args)?, + ) +} + +/// `__source` references replaced with their read relations, cursor inputs filtered; `lookups` holds the source map and warehouse columns. +pub(crate) fn resolve_sources( + sql: &str, + lookups: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + adapter: &Bound<'_, PyAny>, + filtering: &CursorFiltering<'_>, +) -> PyResult { + let aliases = adapter .call_method0("requires_derived_table_aliases")? .is_truthy()?; let mut output = String::new(); let mut last = 0; - for span in reference_spans(&sql, "source") { - let mut relation = resolve_one(&args, &span.name, &sql[span.start..span.end])?; + for span in reference_spans(sql, "source") { + let mut relation = resolve_one( + lookups, + adapter, + filtering, + (&span.name, &sql[span.start..span.end]), + )?; output.push_str(sql.get(last..span.start).unwrap_or_default()); last = span.end; if aliases && relation.trim_start().starts_with('(') { - let (end, alias) = source_alias(&sql, span.end, &span.name); + let (end, alias) = source_alias(sql, span.end, &span.name); relation.push_str(&format!(" AS {alias}")); last = end; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs index 489e2b8256..0bcd5c6a96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs @@ -1,8 +1,8 @@ //! Boundary calls shared by reference and source resolution. use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; +use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_sqltext::resolution::main::cursor_subquery::cursor_subquery; @@ -21,34 +21,33 @@ pub(super) fn method<'py>( } pub(super) fn cursor_wrap( - arguments: &KeywordArguments<'_>, + adapter: &Bound<'_, PyAny>, + filtering: &CursorFiltering<'_>, relation: String, - name: &str, - inject_alias: bool, + (name, inject_alias): (&str, bool), ) -> PyResult { - let bounds = arguments.value("cursor_bounds")?; + let bounds = &filtering.bounds; if bounds.is_none() { return Ok(relation); } - let column: Option = arguments - .value("cursor_filter_inputs")? + let column: Option = filtering + .filter_inputs .call_method1("get", (name,))? .extract()?; let Some(column) = column else { return Ok(relation); }; - let adapter = arguments.value("adapter")?; let py = adapter.py(); - let cursor_type = arguments.value("cursor_type")?; + let cursor_type = &filtering.cursor_type; let literal = |bound: &str| -> PyResult { - let token = call_with( - ®istry::native(py, NativeAttr::CursorSentinelToToken)?, - &[("sentinel", bounds.getattr(bound)?)], - )?; + let token = sentinel_to_token(&bounds.getattr(bound)?)?; method( - &adapter, + adapter, "render_cursor_bound_literal", - &[("value", token), ("cursor_type", cursor_type.clone())], + &[ + ("value", string(py, &token)), + ("cursor_type", cursor_type.clone()), + ], )? .extract() }; @@ -64,9 +63,21 @@ pub(super) fn cursor_wrap( column: &column, start: &start, end: &end, - lower_inclusive: arguments.value("lower_bound_inclusive")?.is_truthy()?, + lower_inclusive: filtering.lower_bound_inclusive, typed: !cursor_type.is_none(), }, alias, )) } + +/// The cursor filtering of a keyword-argument resolution request. +pub(super) fn filtering_of<'py>( + arguments: &KeywordArguments<'py>, +) -> PyResult> { + Ok(CursorFiltering { + bounds: arguments.value("cursor_bounds")?, + filter_inputs: arguments.value("cursor_filter_inputs")?, + cursor_type: arguments.value("cursor_type")?, + lower_bound_inclusive: arguments.value("lower_bound_inclusive")?.is_truthy()?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 9e7835d0c6..d22cd1591b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; +use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; use crate::bindings::_helpers::planner_scenario::values::string; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; @@ -82,21 +83,14 @@ pub(crate) fn build_cli_scenario_plan<'py>( ], )?; let (database, schema) = namespace(&project, &graph)?; - let relation_args = PyDict::new(py); - for (name, value) in [ - ("project", project.clone()), - ("graph_plan", graph.clone()), - ("relation_map", map), - ( - "render_qualified_name", - adapter.getattr("render_qualified_name")?, - ), - ("database", database), - ("schema", schema), - ] { - relation_args.set_item(name, value)?; - } - let relations = build_scenario_relation_plan(py, Some(relation_args))?; + let relations = build_scenario_relation_plan(&ScenarioRelations { + project: project.clone(), + graph_plan: graph.clone(), + relation_map: map, + render_qualified_name: adapter.getattr("render_qualified_name")?, + database, + schema, + })?; let result = call_with( ®istry::native(py, NativeAttr::BuildScenarioExecutionPlan)?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs index 0c97f32fd3..70ff75b823 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs @@ -9,3 +9,14 @@ pub(crate) struct SqlContext<'py> { pub syntax: Bound<'py, PyAny>, pub label: String, } + +/// One scenario's graph and the relation map and qualification that locate its relations. +#[derive(Debug)] +pub(crate) struct ScenarioRelations<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) graph_plan: Bound<'py, PyAny>, + pub(crate) relation_map: Bound<'py, PyAny>, + pub(crate) render_qualified_name: Bound<'py, PyAny>, + pub(crate) database: Bound<'py, PyAny>, + pub(crate) schema: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs index a6e7cbe60c..9abe33e278 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs @@ -54,7 +54,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { )?)?; module.add_function(wrap_pyfunction!(graph::plan_scenario_graph, module)?)?; module.add_function(wrap_pyfunction!( - relations::build_scenario_relation_plan, + relations::build_scenario_relation_plan_of, module )?)?; module.add_function(wrap_pyfunction!(cli::build_cli_scenario_plan, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs index 07a8992cd9..f26919e2b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -7,15 +7,16 @@ use crate::bindings::_helpers::planner_scenario::constants::{REF, SOURCE}; use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, }; +use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; use crate::bindings::_helpers::planner_scenario::values::error; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; -#[pyfunction] +#[pyfunction(name = "build_scenario_relation_plan")] #[pyo3(signature = (**kwargs))] -pub(crate) fn build_scenario_relation_plan<'py>( +pub(crate) fn build_scenario_relation_plan_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -33,13 +34,24 @@ pub(crate) fn build_scenario_relation_plan<'py>( &["database", "schema"], ), )?; - let graph = args.value("graph_plan")?; + build_scenario_relation_plan(&ScenarioRelations { + project: args.value("project")?, + graph_plan: args.value("graph_plan")?, + relation_map: args.value("relation_map")?, + render_qualified_name: args.value("render_qualified_name")?, + database: args.value("database")?, + schema: args.value("schema")?, + }) +} + +/// The scenario's model, seed, source and fixture locations under its relation map. +pub(crate) fn build_scenario_relation_plan<'py>( + args: &ScenarioRelations<'py>, +) -> PyResult> { + let py = args.project.py(); + let graph = &args.graph_plan; let artifacts = PyDict::new(py); - for artifact in args - .value("relation_map")? - .getattr("artifacts")? - .try_iter()? - { + for artifact in args.relation_map.getattr("artifacts")?.try_iter()? { let artifact = artifact?; let identity = artifact.getattr("identity")?; artifacts.set_item( @@ -48,7 +60,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( )?; } let sources = PyDict::new(py); - for source in args.value("project")?.getattr("sources")?.try_iter()? { + for source in args.project.getattr("sources")?.try_iter()? { let source = source?; sources.set_item(source.getattr("name")?, source.getattr("source_entry")?)?; } @@ -68,7 +80,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( ] { for name in graph.getattr(field)?.try_iter()? { let name: String = name?.extract()?; - let target = target_for_artifact(&args, &artifacts, kind, &name)?; + let target = target_for_artifact(args, &artifacts, kind, &name)?; locations.set_item(&name, &target)?; if kind == REF { models.set_item(&name, &target)?; @@ -105,7 +117,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( ®istry::class(py, RecordId::ScenarioRelationPlan)?, &[ ("scenario_name", graph.getattr("name")?), - ("relation_map", args.value("relation_map")?), + ("relation_map", args.relation_map.clone()), ("model_locations", models.into_any()), ("seed_locations", seeds.into_any()), ("project_source_map", sources.into_any()), @@ -119,7 +131,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( } fn target_for_artifact<'py>( - args: &KeywordArguments<'py>, + args: &ScenarioRelations<'py>, artifacts: &Bound<'py, PyDict>, kind: &str, name: &str, @@ -133,10 +145,10 @@ fn target_for_artifact<'py>( Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), )?); }; - let database = args.value("database")?; - let schema = args.value("schema")?; + let database = args.database.clone(); + let schema = args.schema.clone(); let qualified = call_with( - &args.value("render_qualified_name")?, + &args.render_qualified_name, &[ ("database", database.clone()), ("schema", schema.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 6c190d691c..4e5f95a646 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -15,24 +15,25 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::planner_strategy::constants::{ AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, }; +use crate::bindings::_helpers::planner_strategy::models::AuditPlanning; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; /// Render and validate one audit query with the supplied adapter callbacks. fn render_query<'py>( - request: &Request<'_, 'py>, + request: &AuditPlanning<'py>, sql: Bound<'py, PyAny>, context: &str, ) -> PyResult> { - let py = request.py(); + let py = sql.py(); let resolved = call_with( ®istry::native(py, NativeAttr::RenderAuditSql)?, &[ ("unresolved_sql", sql), - ("model_locations", request.get("model_locations")?), - ("seed_locations", request.get("seed_locations")?), - ("source_map", request.get("source_map")?), - ("adapter", request.get("adapter")?), + ("model_locations", request.model_locations.clone()), + ("seed_locations", request.seed_locations.clone()), + ("source_map", request.source_map.clone()), + ("adapter", request.adapter.clone()), ], )?; call_with( @@ -62,15 +63,36 @@ fn audit_enum<'py>( } /// Build an audit entry while retaining the public Python plan dataclass. -#[pyfunction] +#[pyfunction(name = "plan_audit")] #[pyo3(signature = (**request))] -pub(crate) fn plan_audit<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn plan_audit_of<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("plan_audit", request)?; - let py = request.py(); - let audit = request.get("audit")?; + plan_audit( + &request.get("audit")?, + &AuditPlanning { + model_locations: request.get("model_locations")?, + seed_locations: request.get("seed_locations")?, + source_map: request.get("source_map")?, + adapter: request.get("adapter")?, + model_materializations: request.get("model_materializations")?, + lineage_upstream_deps: request.get("lineage_upstream_deps")?, + lineage_downstream_deps: request.get("lineage_downstream_deps")?, + }, + ) +} + +/// One audit's plan entry, rendered against the planning relations. +fn plan_audit<'py>( + audit: &Bound<'py, PyAny>, + request: &AuditPlanning<'py>, +) -> PyResult> { + let py = audit.py(); + let audit = audit.clone(); let name: String = audit.getattr("name")?.extract()?; let resolved = render_query( - &request, + request, audit.getattr("sql_body")?, &format!("audit '{name}' planned SQL"), )?; @@ -79,19 +101,19 @@ pub(crate) fn plan_audit<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult< py.None().into_bound(py) } else { render_query( - &request, + request, evidence, &format!("audit '{name}' planned evidence SQL"), )? }; let attachment: Bound<'_, PyTuple> = - resolve_attachment_kind(&audit, &request.get("lineage_downstream_deps")?)?; + resolve_attachment_kind(&audit, &request.lineage_downstream_deps)?; let kind = attachment.get_item(0)?; let target = attachment.get_item(1)?; let kind_text: String = kind.extract()?; let materialization = if kind_text == MODEL_ATTACHMENT && !target.is_none() { request - .get("model_materializations")? + .model_materializations .call_method1("get", (&target,))? } else { py.None().into_bound(py) @@ -146,7 +168,7 @@ pub(crate) fn plan_audit<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult< "reads_outside_target_lineage", PyBool::new( py, - reads_outside_target_lineage(&audit, &request.get("lineage_upstream_deps")?)?, + reads_outside_target_lineage(&audit, &request.lineage_upstream_deps)?, ) .to_owned() .into_any(), @@ -210,24 +232,17 @@ pub(crate) fn build_selected_audit_entries<'py>( } let lineage = registry::native(py, NativeAttr::ProjectLineageViews)?.call1((project,))?; let relations = request.get("relations")?; - let arguments = PyDict::new(py); - for field in ["model_locations", "seed_locations"] { - arguments.set_item(field, relations.getattr(field)?)?; - } - arguments.set_item("source_map", relations.getattr("source_read_map")?)?; - arguments.set_item("adapter", request.get("adapter")?)?; - arguments.set_item( - "model_materializations", - request.get("model_materializations")?, - )?; - arguments.set_item("lineage_upstream_deps", lineage.getattr("upstream_deps")?)?; - arguments.set_item( - "lineage_downstream_deps", - lineage.getattr("downstream_deps")?, - )?; + let planning = AuditPlanning { + model_locations: relations.getattr("model_locations")?, + seed_locations: relations.getattr("seed_locations")?, + source_map: relations.getattr("source_read_map")?, + adapter: request.get("adapter")?, + model_materializations: request.get("model_materializations")?, + lineage_upstream_deps: lineage.getattr("upstream_deps")?, + lineage_downstream_deps: lineage.getattr("downstream_deps")?, + }; for audit in selected { - arguments.set_item("audit", audit)?; - entries.append(plan_audit(Some(&arguments))?)?; + entries.append(plan_audit(&audit, &planning)?)?; } Ok(entries) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 2497ab3b8b..aa89cdde7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::planner_strategy::constants::{ SOURCE_REFERENCE, TIMESTAMP_CURSOR, }; use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; +use crate::bindings::_helpers::planner_strategy::models::CursorInputLookups; use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; @@ -58,32 +59,61 @@ fn check_column(check: CursorInputColumnCheck, py: Python<'_>) -> PyResult<()> { } } +/// Cursor input lookups from keyword arguments, empty where the caller left one out. +fn lookups_of<'py>(request: &Request<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.py(); + let lookup = |name: &str| or_empty(request.optional(name)?, py); + Ok(CursorInputLookups { + models_by_name: lookup("models_by_name")?, + functions_by_name: lookup("functions_by_name")?, + source_map: lookup("source_map")?, + source_warehouse_columns: lookup("source_warehouse_columns")?, + model_locations: lookup("model_locations")?, + seed_locations: lookup("seed_locations")?, + runtime_cursor_producer_names: match request.optional("runtime_cursor_producer_names")? { + Some(names) => names, + None => PyFrozenSet::empty(py)?.into_any(), + }, + }) +} + /// Validate authored cursor input columns against contracts, then source warehouse metadata. #[pyfunction(name = "validate_source_cursor_input_columns")] #[pyo3(signature = (**request))] +pub(crate) fn cursor_input_columns_of<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> { + let request = Request::new("validate_source_cursor_input_columns", request)?; + request.get("source_warehouse_columns")?; + reject_unexposed_cursor_input_columns( + &request.get("model")?, + &request.get("cursor_column")?, + &lookups_of(&request)?, + ) +} + +/// Reject authored cursor input columns a contract or the source warehouse metadata does not expose. pub(crate) fn reject_unexposed_cursor_input_columns<'py>( - request: Option<&Bound<'py, PyDict>>, + model: &Bound<'py, PyAny>, + cursor_column: &Bound<'py, PyAny>, + lookups: &CursorInputLookups<'py>, ) -> PyResult<()> { - let request = Request::new("validate_source_cursor_input_columns", request)?; - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; - if !incremental_cursor_flag(&model, &request.get("cursor_column")?)? { + let py: Python<'py> = model.py(); + if !incremental_cursor_flag(model, cursor_column)? { return Ok(()); } - let roles: Bound<'py, PyAny> = resolve_cursor_input_roles(&model)?; - let models_by_name = or_empty(request.optional("models_by_name")?, py)?; - let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; - let source_map = or_empty(request.optional("source_map")?, py)?; - let warehouse_columns: Bound<'py, PyAny> = request.get("source_warehouse_columns")?; - let model_name: String = model_reads::model_name(&model)?; + let model: &Bound<'py, PyAny> = model; + let roles: Bound<'py, PyAny> = resolve_cursor_input_roles(model)?; + let models_by_name: &Bound<'py, PyAny> = &lookups.models_by_name; + let source_map: &Bound<'py, PyAny> = &lookups.source_map; + let warehouse_columns: &Bound<'py, PyAny> = &lookups.source_warehouse_columns; + let model_name: String = model_reads::model_name(model)?; for role in ["filter_inputs", "watermark_inputs"] { for item in roles.getattr(role)?.call_method0("items")?.try_iter()? { let (input_name, column): (String, String) = item?.extract()?; let reference: Bound<'py, PyAny> = resolve_lineage_reference( - &model, + model, &input_name, models_by_name.clone(), - functions_by_name.clone(), + lookups.functions_by_name.clone(), )?; let kind: String = text(&reference.getattr("ref_kind")?)?; let name: String = text(&reference.getattr("ref_name")?)?; @@ -136,34 +166,26 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( /// The qualified relation one cursor input reference reads, when it resolves. fn input_relation<'py>( - request: &Request<'_, 'py>, + (lookups, adapter): (&CursorInputLookups<'py>, &Bound<'py, PyAny>), (kind, name): (&str, &str), ) -> PyResult> { - let py: Python<'py> = request.py(); + let py: Python<'py> = adapter.py(); let location: Bound<'py, PyAny> = match kind { MODEL_REFERENCE => { - let model: Bound<'py, PyAny> = request - .get("model_locations")? - .call_method1("get", (name,))?; + let model: Bound<'py, PyAny> = lookups.model_locations.call_method1("get", (name,))?; if model.is_none() { - request - .get("seed_locations")? - .call_method1("get", (name,))? + lookups.seed_locations.call_method1("get", (name,))? } else { model } } - SEED_REFERENCE => request - .get("seed_locations")? - .call_method1("get", (name,))?, + SEED_REFERENCE => lookups.seed_locations.call_method1("get", (name,))?, SOURCE_REFERENCE => { - let source: Bound<'py, PyAny> = - request.get("source_map")?.call_method1("get", (name,))?; + let source: Bound<'py, PyAny> = lookups.source_map.call_method1("get", (name,))?; if source.is_none() { return Ok(None); } - let adapter: Bound<'py, PyAny> = request.get("adapter")?; - return render_source_relation(&source, Some(&adapter)).map(Some); + return render_source_relation(&source, Some(adapter)).map(Some); } _ => py.None().into_bound(py), }; @@ -203,21 +225,44 @@ fn terminal_bound<'py>( } /// Cursor-bearing input relations of an incremental model, for runtime range discovery. -#[pyfunction] +#[pyfunction(name = "build_cursor_input_relations")] #[pyo3(signature = (**request))] -pub(crate) fn build_cursor_input_relations<'py>( +pub(crate) fn cursor_input_relations_of<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("build_cursor_input_relations", request)?; - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; - if !incremental_cursor_flag(&model, &request.get("cursor_column")?)? { + for name in [ + "adapter", + "model_locations", + "models_by_name", + "seed_locations", + "source_map", + "runtime_cursor_producer_names", + ] { + request.get(name)?; + } + build_cursor_input_relations( + (&request.get("model")?, &request.get("adapter")?), + &request.get("cursor_column")?, + &lookups_of(&request)?, + ) +} + +/// Cursor-bearing input relations of an incremental model, for runtime range discovery. +pub(crate) fn build_cursor_input_relations<'py>( + (model, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + cursor_column: &Bound<'py, PyAny>, + lookups: &CursorInputLookups<'py>, +) -> PyResult> { + let py: Python<'py> = model.py(); + if !incremental_cursor_flag(model, cursor_column)? { return Ok(PyTuple::empty(py)); } - let models_by_name: Bound<'py, PyAny> = request.get("models_by_name")?; - let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; - let model_locations: Bound<'py, PyAny> = request.get("model_locations")?; - let producers: Bound<'py, PyAny> = request.get("runtime_cursor_producer_names")?; + let model: Bound<'py, PyAny> = model.clone(); + let models_by_name: &Bound<'py, PyAny> = &lookups.models_by_name; + let functions_by_name: &Bound<'py, PyAny> = &lookups.functions_by_name; + let model_locations: &Bound<'py, PyAny> = &lookups.model_locations; + let producers: &Bound<'py, PyAny> = &lookups.runtime_cursor_producer_names; let cursor_type: Bound<'py, PyAny> = match config_str(&model_values(&model)?, "cursor_type")? { Some(cursor_type) if cursor_type.is_truthy()? => cursor_type, _ => PyString::new(py, TIMESTAMP_CURSOR).into_any(), @@ -237,7 +282,7 @@ pub(crate) fn build_cursor_input_relations<'py>( let kind: String = text(&reference.getattr("ref_kind")?)?; let name: Bound<'py, PyAny> = reference.getattr("ref_name")?; let name_text: String = text(&name)?; - let Some(relation) = input_relation(&request, (&kind, &name_text))? else { + let Some(relation) = input_relation((lookups, adapter), (&kind, &name_text))? else { continue; }; let producer: Option> = if kind == MODEL_REFERENCE { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index 29fcf254be..ff00ec515e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -69,7 +69,7 @@ fn classify_cursor_sql_type<'py>( /// A `PlanWarning` when the warehouse cursor column type contradicts `cursor_type`. #[pyfunction] #[pyo3(signature = (**options))] -pub(crate) fn check_cursor_type_consistency<'py>( +fn check_cursor_type_consistency<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { @@ -89,28 +89,41 @@ pub(crate) fn check_cursor_type_consistency<'py>( ), )?; let model_name: Bound<'py, PyAny> = arguments.value("model_name")?; - let mut warehouse_columns: Vec<(String, String)> = Vec::new(); - for column in arguments.value("warehouse_columns")?.try_iter()? { - let column: Bound<'py, PyAny> = column?; - warehouse_columns.push(( - text(&column.getattr("name")?)?, - text(&column.getattr("type")?)?, - )); - } let check = CursorTypeCheck { model_name: text(&model_name)?, cursor_column: optional_string(&arguments.value("cursor_column")?)?, cursor_type: optional_string(&arguments.value("cursor_type")?)?, - warehouse_columns, + warehouse_columns: warehouse_column_types(&arguments.value("warehouse_columns")?)?, sql_analysis_enabled: arguments.flag("sql_analysis_enabled")?, }; - let (warning, parse_failure) = cursor_type_warning(&check); + cursor_type_warning_record(&model_name, &check) +} + +/// The `(name, type)` pairs of warehouse `ColumnInfo`s. +pub(crate) fn warehouse_column_types( + columns: &Bound<'_, PyAny>, +) -> PyResult> { + let mut pairs: Vec<(String, String)> = Vec::new(); + for column in columns.try_iter()? { + let column = column?; + pairs.push(( + text(&column.getattr("name")?)?, + text(&column.getattr("type")?)?, + )); + } + Ok(pairs) +} + +/// The `PlanWarning` of one cursor type check, logging an unclassifiable warehouse type. +pub(crate) fn cursor_type_warning_record<'py>( + model_name: &Bound<'py, PyAny>, + check: &CursorTypeCheck, +) -> PyResult>> { + let (warning, parse_failure) = cursor_type_warning(check); if let Some(failure) = parse_failure { - log_parse_failure(py, &failure)?; + log_parse_failure(model_name.py(), &failure)?; } - warning - .map(|row| plan_warning(&model_name, row)) - .transpose() + warning.map(|row| plan_warning(model_name, row)).transpose() } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 9143134c54..d66e0fb1a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -7,11 +7,10 @@ use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_strategy::main::microbatch_lookback::microbatch_lookback; use sqlbuild_analysis::planner_strategy::main::table_type_ddl::table_type_ddl; -use sqlbuild_analysis::planner_strategy::models::LookbackConfig; +use sqlbuild_analysis::planner_strategy::models::{CursorTypeCheck, LookbackConfig}; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::{ config_bool, config_cursor_bound, config_int, config_str, config_string_tuple, @@ -23,13 +22,16 @@ use crate::bindings::_helpers::planner_strategy::constants::{ CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; -use crate::bindings::_helpers::planner_strategy::cursor_types::check_cursor_type_consistency; +use crate::bindings::_helpers::planner_strategy::cursor_types::{ + cursor_type_warning_record, warehouse_column_types, +}; use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; +use crate::bindings::_helpers::planner_strategy::models::ModelEntryDecisions; use crate::bindings::_helpers::planner_strategy::strategy::{ - build_model_warnings, resolve_schema_actions, + model_warning_records, resolve_schema_actions, warning_inputs, }; -use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; +use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; @@ -56,25 +58,6 @@ const STRING_FIELDS: [(&str, &str); 15] = [ ]; /// Model config keys copied as optional strings that complete the snapshot settings. const SNAPSHOT_POLICY_FIELDS: [&str; 2] = ["snapshot_full_refresh", "snapshot_schema_change"]; -/// Plan decisions passed through to the entry unchanged. -const PASSED_FIELDS: [&str; 16] = [ - "action", - "reason", - "materialization_type", - "resolved_sql", - "cursor_column", - "cursor_bounds", - "empty_cursor_inputs", - "waiting_on_empty_inputs", - "cursor_input_relations", - "microbatch_range", - "start_cursor_override", - "end_cursor_override", - "future_cursor_config", - "start_cursor_config", - "invocation_time", - "backfill", -]; /// Change-detection facts copied to the entry. const CHANGE_FIELDS: [&str; 8] = [ "fingerprint_metadata_json", @@ -232,29 +215,29 @@ pub(crate) fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult( - request: &Request<'_, 'py>, + request: &ModelEntryDecisions<'py>, (unique_key, merge_exclude): (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>), (warehouse_columns, cursor_bounds): (&Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult { - let py: Python<'py> = request.py(); - let adapter: Bound<'py, PyAny> = request.get("adapter")?; - let model: Bound<'py, PyAny> = request.get("model")?; - let destination: Bound<'py, PyAny> = model_reads::destination(&model)?; + let py: Python<'py> = request.model.py(); + let adapter: &Bound<'py, PyAny> = &request.adapter; + let model: &Bound<'py, PyAny> = &request.model; + let destination: Bound<'py, PyAny> = model_reads::destination(model)?; let qualified: Bound<'py, PyAny> = destination.getattr("qualified_name")?; let qualified: Bound<'py, PyAny> = if qualified.is_truthy()? { qualified } else { destination.getattr("name")? }; - let sql: Bound<'py, PyAny> = request.get("resolved_sql")?; + let sql: Bound<'py, PyAny> = request.resolved_sql.clone(); let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![("destination", qualified), ("sql", sql)]; - let cursor_column: Bound<'py, PyAny> = request.get("cursor_column")?; - let method: &str = match text(&request.get("action")?)?.as_str() { + let cursor_column: Bound<'py, PyAny> = request.cursor_column.clone(); + let method: &str = match text(&request.action)?.as_str() { "create_view" => "render_create_view_as", "create_table" => "render_create_table_as", "incremental_append" => "render_append", "incremental_delete_insert" if !cursor_column.is_none() && !cursor_bounds.is_none() => { - let values: Bound<'py, PyAny> = model_values(&model)?; + let values: Bound<'py, PyAny> = model_values(model)?; let start = sentinel_to_token(&cursor_bounds.getattr("start")?)?; let end = sentinel_to_token(&cursor_bounds.getattr("end")?)?; pairs.push(("cursor_column", cursor_column)); @@ -331,59 +314,42 @@ fn custom_fields<'py>( /// The model's base warnings followed by its cursor type warning. fn model_warnings<'py>( - request: &Request<'_, 'py>, + request: &ModelEntryDecisions<'py>, (schema_actions, policy): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (enforcement, warehouse_columns): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; - let mut warnings: Vec> = build_model_warnings( - py, - Some(keywords( - py, - &[ - ("model_name", name.clone()), - ("materialization_type", request.get("materialization_type")?), - ("change_result", request.get("change_result")?), - ("schema_actions", schema_actions.clone()), - ("on_schema_change", policy.clone()), - ("type_enforcement", enforcement.clone()), - ], - )?), - )? - .try_iter()? - .collect::>()?; - let cursor_warning: Option> = check_cursor_type_consistency( - py, - Some(keywords( - py, - &[ - ("model_name", name), - ("cursor_column", request.get("cursor_column")?), - ( - "cursor_type", - or_none(py, config_str(&model_values(&model)?, "cursor_type")?), - ), - ("warehouse_columns", warehouse_columns.clone()), - ("sql_analysis_enabled", request.get("sql_analysis_enabled")?), - ], - )?), + let py: Python<'py> = request.model.py(); + let model: &Bound<'py, PyAny> = &request.model; + let name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let inputs = warning_inputs( + &name, + &request.materialization_type, + (schema_actions, policy, enforcement), )?; - if let Some(cursor_warning) = cursor_warning.filter(|warning| !warning.is_none()) { + let mut warnings: Vec> = + model_warning_records(&name, &inputs, &request.change_result)?; + let check = CursorTypeCheck { + model_name: text(&name)?, + cursor_column: optional_string(&request.cursor_column)?, + cursor_type: optional_string(&or_none( + py, + config_str(&model_values(model)?, "cursor_type")?, + ))?, + warehouse_columns: warehouse_column_types(warehouse_columns)?, + sql_analysis_enabled: request.sql_analysis_enabled, + }; + if let Some(cursor_warning) = cursor_type_warning_record(&name, &check)? { warnings.push(cursor_warning); } PyTuple::new(py, warnings) } /// The cursor bounds the logical DDL renders: placeholders for runtime-owned windows, else the batch range or cursor bounds. -fn ddl_cursor_bounds<'py>(request: &Request<'_, 'py>) -> PyResult> { - let py: Python<'py> = request.py(); - let microbatch_range: Bound<'py, PyAny> = request.get("microbatch_range")?; +fn ddl_cursor_bounds<'py>(request: &ModelEntryDecisions<'py>) -> PyResult> { + let py: Python<'py> = request.model.py(); + let microbatch_range: &Bound<'py, PyAny> = &request.microbatch_range; Ok( - if request.get("runtime_owned_cursor_bounds")?.is_truthy()? - && !request.get("cursor_column")?.is_none() - { + if request.runtime_owned_cursor_bounds && !request.cursor_column.is_none() { let sentinel: Bound<'py, PyAny> = host::entry(py, ExecutorHost::BoundSentinel)?; call_with( ®istry::class(py, RecordId::CursorBounds)?, @@ -395,19 +361,19 @@ fn ddl_cursor_bounds<'py>(request: &Request<'_, 'py>) -> PyResult( - request: &Request<'_, 'py>, + request: &ModelEntryDecisions<'py>, values: &Bound<'py, PyAny>, entry: &Bound<'py, PyDict>, ) -> PyResult<()> { - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; + let py: Python<'py> = request.model.py(); + let model: &Bound<'py, PyAny> = &request.model; for (key, field) in STRING_FIELDS { entry.set_item(field, or_none(py, config_str(values, key)?))?; } @@ -433,9 +399,9 @@ fn config_fields<'py>( read("microbatch_strategy")?, read("cursor_grain")?, ), - &request.get("cursor_input_relations")?, + &request.cursor_input_relations, )?; - lookback_of(&model, grain)? + lookback_of(model, grain)? } else { config_str(values, "lookback")? .map(|value| text(&value)) @@ -489,28 +455,62 @@ fn contract_fields<'py>( Ok(()) } -/// Build a model plan entry and its warnings from resolved plan decisions, keeping the Python dataclasses. -#[pyfunction] -#[pyo3(signature = (**request))] +/// Set the plan decisions passed through to the entry unchanged. +fn passed_fields<'py>( + request: &ModelEntryDecisions<'py>, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = request.model.py(); + let flag = |value: bool| PyBool::new(py, value).to_owned().into_any(); + let fields: [(&str, Bound<'py, PyAny>); 16] = [ + ("action", request.action.clone()), + ("reason", request.reason.clone()), + ("materialization_type", request.materialization_type.clone()), + ("resolved_sql", request.resolved_sql.clone()), + ("cursor_column", request.cursor_column.clone()), + ("cursor_bounds", request.cursor_bounds.clone()), + ("empty_cursor_inputs", request.empty_cursor_inputs.clone()), + ( + "waiting_on_empty_inputs", + flag(request.waiting_on_empty_inputs), + ), + ( + "cursor_input_relations", + request.cursor_input_relations.clone(), + ), + ("microbatch_range", request.microbatch_range.clone()), + ("start_cursor_override", request.cursor_overrides.0.clone()), + ("end_cursor_override", request.cursor_overrides.1.clone()), + ("future_cursor_config", request.cursor_configs.0.clone()), + ("start_cursor_config", request.cursor_configs.1.clone()), + ("invocation_time", request.invocation_time.clone()), + ("backfill", request.backfill.clone()), + ]; + for (field, value) in fields { + entry.set_item(field, value)?; + } + Ok(()) +} + +/// Build a model plan entry and its warnings from resolved plan decisions. pub(crate) fn assemble_model_plan_entry<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("assemble_model_plan_entry", request)?; - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; - let values: Bound<'py, PyAny> = model_values(&model)?; - let change: Bound<'py, PyAny> = request.get("change_result")?; - let snapshot: Bound<'py, PyAny> = request.get("snapshot")?; - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; - let policy: Bound<'py, PyAny> = on_schema_change(&model)?; + request: &ModelEntryDecisions<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { + let py: Python<'py> = request.model.py(); + let model: &Bound<'py, PyAny> = &request.model; + let values: Bound<'py, PyAny> = model_values(model)?; + let change: &Bound<'py, PyAny> = &request.change_result; + let snapshot: &Bound<'py, PyAny> = &request.snapshot; + let name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let policy: Bound<'py, PyAny> = on_schema_change(model)?; let schema_actions: Bound<'py, PyAny> = resolve_schema_actions(&change.getattr("schema_findings")?, &policy)?.into_any(); - let enforcement: Bound<'py, PyAny> = type_enforcement(&model)?; + let enforcement: Bound<'py, PyAny> = type_enforcement(model)?; let warehouse_columns: Bound<'py, PyAny> = snapshot .getattr("existing_columns")? .call_method1("get", (&name, PyTuple::empty(py)))?; let warnings: Bound<'py, PyTuple> = model_warnings( - &request, + request, (&schema_actions, &policy), (&enforcement, &warehouse_columns), )?; @@ -523,33 +523,30 @@ pub(crate) fn assemble_model_plan_entry<'py>( .getattr("value")? .getattr("value")?; let ddl: String = logical_ddl( - &request, + request, (&unique_key, &merge_exclude), - (&warehouse_columns, ddl_cursor_bounds(&request)?), + (&warehouse_columns, ddl_cursor_bounds(request)?), )?; let entry: Bound<'py, PyDict> = PyDict::new(py); - entry.set_item("key", model_reads::model_key(&model)?)?; + entry.set_item("key", model_reads::model_key(model)?)?; entry.set_item("name", &name)?; entry.set_item("relative_path", model.getattr("relative_path")?)?; - entry.set_item("destination", model_reads::destination(&model)?)?; - entry.set_item("fingerprint_query_sql", model_reads::query_sql(&model)?)?; + entry.set_item("destination", model_reads::destination(model)?)?; + entry.set_item("fingerprint_query_sql", model_reads::query_sql(model)?)?; entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; - for field in PASSED_FIELDS { - entry.set_item(field, request.get(field)?)?; - } + passed_fields(request, &entry)?; for field in CHANGE_FIELDS { entry.set_item(field, change.getattr(field)?)?; } - config_fields(&request, &values, &entry)?; + config_fields(request, &values, &entry)?; entry.set_item("unique_key", unique_key)?; entry.set_item("merge_exclude_columns", merge_exclude)?; entry.set_item("table_type", table_type)?; entry.set_item("on_schema_change", policy)?; entry.set_item("type_enforcement", enforcement)?; - contract_fields(&model, &values, &entry)?; - for field in ["pre_hooks", "post_hooks"] { - entry.set_item(field, request.get(field)?)?; - } + contract_fields(model, &values, &entry)?; + entry.set_item("pre_hooks", &request.hooks.0)?; + entry.set_item("post_hooks", &request.hooks.1)?; let fingerprint: Bound<'py, PyAny> = snapshot .getattr("fingerprints")? .getattr("models")? @@ -565,10 +562,10 @@ pub(crate) fn assemble_model_plan_entry<'py>( entry.set_item("schema_actions", schema_actions)?; custom_fields( &values, - text(&request.get("materialization_type")?)? == CUSTOM_MATERIALIZATION, + text(&request.materialization_type)? == CUSTOM_MATERIALIZATION, &entry, )?; let plan_entry: Bound<'py, PyAny> = registry::class(py, RecordId::ModelPlanEntry)?.call((), Some(&entry))?; - PyTuple::new(py, [plan_entry, warnings.into_any()]) + Ok((plan_entry, warnings)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs index 6c998ddebe..cc125e3fc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex}; -use pyo3::prelude::Py; +use pyo3::prelude::{Bound, Py}; use pyo3::pyclass; use pyo3::types::{PyAny, PyDict}; use sqlbuild_analysis::planner_fixtures::models::{ @@ -56,3 +56,53 @@ pub(crate) struct ExpectedTypes<'a> { pub(crate) overlay: Vec<(FixtureKey, Vec<(String, String)>)>, pub(crate) base: &'a HashMap>, } + +/// One model's resolved plan decisions, assembled into its `ModelPlanEntry`. +#[derive(Debug)] +pub(crate) struct ModelEntryDecisions<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) change_result: Bound<'py, PyAny>, + pub(crate) action: Bound<'py, PyAny>, + pub(crate) reason: Bound<'py, PyAny>, + pub(crate) materialization_type: Bound<'py, PyAny>, + pub(crate) resolved_sql: Bound<'py, PyAny>, + pub(crate) sql_analysis_enabled: bool, + pub(crate) hooks: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) cursor_column: Bound<'py, PyAny>, + pub(crate) cursor_input_relations: Bound<'py, PyAny>, + pub(crate) runtime_owned_cursor_bounds: bool, + pub(crate) cursor_bounds: Bound<'py, PyAny>, + pub(crate) microbatch_range: Bound<'py, PyAny>, + pub(crate) empty_cursor_inputs: Bound<'py, PyAny>, + pub(crate) waiting_on_empty_inputs: bool, + pub(crate) backfill: Bound<'py, PyAny>, + pub(crate) cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) cursor_configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) invocation_time: Bound<'py, PyAny>, +} + +/// The project lookups that resolve one model's cursor inputs. +#[derive(Debug)] +pub(crate) struct CursorInputLookups<'py> { + pub(crate) models_by_name: Bound<'py, PyAny>, + pub(crate) functions_by_name: Bound<'py, PyAny>, + pub(crate) source_map: Bound<'py, PyAny>, + pub(crate) source_warehouse_columns: Bound<'py, PyAny>, + pub(crate) model_locations: Bound<'py, PyAny>, + pub(crate) seed_locations: Bound<'py, PyAny>, + pub(crate) runtime_cursor_producer_names: Bound<'py, PyAny>, +} + +/// The relations and lineage that render and schedule selected audits. +#[derive(Debug)] +pub(crate) struct AuditPlanning<'py> { + pub(crate) model_locations: Bound<'py, PyAny>, + pub(crate) seed_locations: Bound<'py, PyAny>, + pub(crate) source_map: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) model_materializations: Bound<'py, PyAny>, + pub(crate) lineage_upstream_deps: Bound<'py, PyAny>, + pub(crate) lineage_downstream_deps: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index 89538ed245..26e4c400b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -8,29 +8,28 @@ use crate::bindings::_helpers::planner_strategy::{ }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(super::audit_entries::plan_audit, module)?)?; module.add_function(wrap_pyfunction!( - super::audit_entries::audit_is_selected, + super::audit_entries::plan_audit_of, module )?)?; module.add_function(wrap_pyfunction!( - super::audit_entries::build_selected_audit_entries, + super::audit_entries::audit_is_selected, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::reject_unexposed_cursor_input_columns, + super::audit_entries::build_selected_audit_entries, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::build_cursor_input_relations, + super::cursor_inputs::cursor_input_columns_of, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::runtime_cursor_producer_names, + super::cursor_inputs::cursor_input_relations_of, module )?)?; module.add_function(wrap_pyfunction!( - super::model_entries::assemble_model_plan_entry, + super::cursor_inputs::runtime_cursor_producer_names, module )?)?; module.add_function(wrap_pyfunction!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index ad4e0aa5eb..f3057d8e3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -138,10 +138,41 @@ pub(crate) fn resolve_schema_actions<'py>( PyTuple::new(py, actions) } +/// The warning inputs of one model's plan decisions. +pub(crate) fn warning_inputs( + model_name: &Bound<'_, PyAny>, + materialization: &Bound<'_, PyAny>, + (schema_actions, on_schema_change, type_enforcement): ( + &Bound<'_, PyAny>, + &Bound<'_, PyAny>, + &Bound<'_, PyAny>, + ), +) -> PyResult { + Ok(WarningInputs { + model_name: text(model_name)?, + materialization: materialization_type(Some(&text(materialization)?)), + has_schema_actions: schema_actions.is_truthy()?, + on_schema_change: schema_policy(on_schema_change)?, + type_enforcement: type_enforcement.is_truthy()?, + }) +} + +/// The `PlanWarning`s of one model's warning inputs and change result. +pub(crate) fn model_warning_records<'py>( + model_name: &Bound<'py, PyAny>, + inputs: &WarningInputs, + change_result: &Bound<'py, PyAny>, +) -> PyResult>> { + model_warnings(inputs, &change_facts(change_result)?) + .into_iter() + .map(|row| plan_warning(model_name, row)) + .collect() +} + /// Warnings for one model from its change detection and plan decisions. #[pyfunction] #[pyo3(signature = (**options))] -pub(crate) fn build_model_warnings<'py>( +fn build_model_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -162,21 +193,19 @@ pub(crate) fn build_model_warnings<'py>( ), )?; let model_name: Bound<'py, PyAny> = arguments.value("model_name")?; - let inputs = WarningInputs { - model_name: text(&model_name)?, - materialization: materialization_type(Some(&text( - &arguments.value("materialization_type")?, - )?)), - has_schema_actions: arguments.flag("schema_actions")?, - on_schema_change: schema_policy(&arguments.value("on_schema_change")?)?, - type_enforcement: arguments.flag("type_enforcement")?, - }; - let rows = model_warnings(&inputs, &change_facts(&arguments.value("change_result")?)?); - let warnings: Vec> = rows - .into_iter() - .map(|row| plan_warning(&model_name, row)) - .collect::>()?; - PyTuple::new(py, warnings) + let inputs: WarningInputs = warning_inputs( + &model_name, + &arguments.value("materialization_type")?, + ( + &arguments.value("schema_actions")?, + &arguments.value("on_schema_change")?, + &arguments.value("type_enforcement")?, + ), + )?; + PyTuple::new( + py, + model_warning_records(&model_name, &inputs, &arguments.value("change_result")?)?, + ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs index cbe7d5efed..5d6419d550 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs @@ -54,7 +54,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!( - super::source_freshness::build_planner_source_freshness_result, + super::source_freshness::build_planner_source_freshness_result_of, module )?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index cb56a20b1e..f26f61aa14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -103,9 +103,9 @@ fn state_schemas( Ok(schemas) } -#[pyfunction] +#[pyfunction(name = "build_planner_source_freshness_result")] #[pyo3(signature = (**options))] -pub(crate) fn build_planner_source_freshness_result<'py>( +pub(crate) fn build_planner_source_freshness_result_of<'py>( py: pyo3::Python<'py>, options: Option>, ) -> PyResult> { @@ -118,13 +118,21 @@ pub(crate) fn build_planner_source_freshness_result<'py>( &["freshness_state_schemas"], ), )?; - let project = &args.value("project")?; - let adapter = &args.value("adapter")?; - let connection = &args.value("connection")?; - let scope = &args.value("scope")?; - let relations = &args.value("relations")?; - let freshness_state_schemas = &args.value("freshness_state_schemas")?; + build_planner_source_freshness_result( + (&args.value("project")?, &args.value("scope")?), + (&args.value("adapter")?, &args.value("connection")?), + &args.value("relations")?, + &args.value("freshness_state_schemas")?, + ) +} +/// The source freshness evaluation of one inspection scope against the warehouse. +pub(crate) fn build_planner_source_freshness_result<'py>( + (project, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + relations: &Bound<'py, PyAny>, + freshness_state_schemas: &Bound<'py, PyAny>, +) -> PyResult> { let py = project.py(); let schemas = state_schemas(project, scope)?; let mut lowered_state: BTreeSet = BTreeSet::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index 37cc938dd0..557af6ff50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -72,12 +73,10 @@ fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult = PyDict::new(py); for model in inputs.getattr("model_inputs")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - let entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; - if entry.is_none() { + let Some(schema) = attached_schema(&model)? else { continue; - } - let name: Bound<'py, PyAny> = entry.getattr("name")?; - let columns: Bound<'py, PyAny> = entry.getattr("columns")?; + }; + let (name, columns): (Bound<'py, PyAny>, Bound<'py, PyAny>) = (schema.name, schema.columns); types.set_item(&name, column_types(&columns, false)?)?; nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; let contract: Option> = model @@ -89,7 +88,7 @@ fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult contract.eq(ENFORCED_CONTRACT)?, None => false, }; - if enforced && columns.is_truthy()? && !entry.getattr("dynamic_columns")?.is_truthy()? { + if enforced && columns.is_truthy()? && !schema.dynamic_columns.is_truthy()? { let stem: Bound<'py, PyAny> = model .getattr("model_file")? .getattr("file_path")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 6df67839b2..5958548a0f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -19,6 +19,7 @@ use sqlbuild_model_config::templates::models::Scalar; use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::project_assembly::models::{ NativeProjectResources, NativeResourceFailure, @@ -162,15 +163,10 @@ fn tags(value: &Bound<'_, PyAny>) -> PyResult> { fn model_facts(model_input: &Bound<'_, PyAny>, checks: &Bound<'_, PyAny>) -> PyResult { let model_file: Bound<'_, PyAny> = model_input.getattr("model_file")?; - let mut reference_rows: Vec = Vec::new(); - for reference in model_input.getattr("references")?.try_iter()? { - let reference: Bound<'_, PyAny> = reference?; - reference_rows.push(( - reference.getattr("ref_kind")?.str()?.extract::()?, - reference.getattr("ref_name")?.extract()?, - reference.getattr("ref_package")?.extract()?, - )); - } + let reference_rows: Vec = input_reference_rows(model_input)? + .into_iter() + .map(|(kind, name, package, _)| (kind, name, package)) + .collect(); let mut syntax_checks: Vec = Vec::new(); for check in checks.try_iter()? { let (sql, placeholders): (String, Bound<'_, PyAny>) = check?.extract()?; @@ -449,7 +445,7 @@ fn resources_object<'py>( /// Return the resource facts and the resource Python's assembly raises at, if any. #[pyfunction] #[pyo3(signature = (*, inputs, dialect, syntax_checks))] -fn assemble_native_project_resources<'py>( +pub(crate) fn assemble_native_project_resources<'py>( inputs: &Bound<'py, PyAny>, dialect: Option, syntax_checks: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 3ba20fef2e..b63b199356 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -6,6 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; +use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -90,7 +91,7 @@ fn syntax_checks<'py>( /// Each model's `(sql, placeholders)` that assembly passes to syntax validation. #[pyfunction] #[pyo3(signature = (*, model_inputs, analysis_model_names, analysis_succeeded))] -fn model_syntax_checks<'py>( +pub(crate) fn model_syntax_checks<'py>( model_inputs: &Bound<'py, PyAny>, analysis_model_names: &Bound<'py, PyAny>, analysis_succeeded: &Bound<'py, PyAny>, @@ -105,7 +106,7 @@ fn model_syntax_checks<'py>( /// `(sql, families)` by model index for each pivot model no analysis covered. #[pyfunction] #[pyo3(signature = (*, model_inputs, analysed))] -fn standalone_pivot_models<'py>( +pub(crate) fn standalone_pivot_models<'py>( model_inputs: &Bound<'py, PyAny>, analysed: &Bound<'py, PyAny>, ) -> PyResult> { @@ -113,11 +114,10 @@ fn standalone_pivot_models<'py>( let models: Bound<'py, PyDict> = PyDict::new(py); for (index, model_input) in model_inputs.try_iter()?.enumerate() { let model_input: Bound<'py, PyAny> = model_input?; - let schema_entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; - if schema_entry.is_none() { + let Some(schema) = attached_schema(&model_input)? else { continue; - } - let families: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; + }; + let families: Bound<'py, PyAny> = schema.dynamic_columns; if !families.is_truthy()? || analysed.contains(model_name(&model_input)?)? { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index d37c1c4cda..54b76127e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -118,7 +118,6 @@ fn default_value<'py>( FieldDefault::HostMember(class, member) => host::entry(py, class)?.getattr(member), FieldDefault::NativeMember(class, member) => registry::class(py, class)?.getattr(member), FieldDefault::NativeCall(class) => registry::class(py, class)?.call0(), - FieldDefault::NativeClassCall(class) => registry::native(py, class)?.call0(), FieldDefault::Factory(factory) => factory(py), FieldDefault::CallText(module, name, text) => imported(py, module, name)?.call1((text,)), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index e23a03ebc1..4926d5bdc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -67,7 +67,6 @@ native_ids! { ColumnMigrationDecision => "ColumnMigrationDecision", ColumnMigrationEvent => "ColumnMigrationEvent", ColumnMigrationPlanEntry => "ColumnMigrationPlanEntry", - ColumnMigrationPlanning => "ColumnMigrationPlanning", ColumnRenameHint => "ColumnRenameHint", ColumnTransformKind => "ColumnTransformKind", CommandWarehouseScope => "CommandWarehouseScope", @@ -243,7 +242,6 @@ native_ids! { EnumMetadata => "EnumMetadata", EventExportSeverity => "EventExportSeverity", ExecutionLimitsConfig => "ExecutionLimitsConfig", - ExpandedLintFile => "ExpandedLintFile", ExpansionSpan => "ExpansionSpan", Finding => "Finding", Fingerprint => "Fingerprint", @@ -338,7 +336,6 @@ native_ids! { ModelMigrationDeclaration => "ModelMigrationDeclaration", ModelMigrationDiscovery => "ModelMigrationDiscovery", ModelMigrationPlanEntry => "ModelMigrationPlanEntry", - ModelMigrationPlanning => "ModelMigrationPlanning", ModelMigrationRequest => "ModelMigrationRequest", ModelPlanContext => "ModelPlanContext", ModelPlanEntry => "ModelPlanEntry", @@ -373,7 +370,6 @@ native_ids! { OldNameViewEventType => "OldNameViewEventType", OldNameViewHistory => "OldNameViewHistory", OldNameViewPlanEntry => "OldNameViewPlanEntry", - OldNameViewPlanning => "OldNameViewPlanning", OldNameViewStatus => "OldNameViewStatus", OnSchemaChange => "OnSchemaChange", OwnershipRoot => "OwnershipRoot", @@ -383,7 +379,6 @@ native_ids! { ParsedSelector => "ParsedSelector", PartialBuildPhaseTimings => "PartialBuildPhaseTimings", PathSelector => "PathSelector", - PendingLintBody => "PendingLintBody", PlanAction => "PlanAction", PlanCommandRequest => "PlanCommandRequest", PlanEntryBuildInputs => "PlanEntryBuildInputs", @@ -394,28 +389,19 @@ native_ids! { PlanSafetyGates => "PlanSafetyGates", PlanWarning => "PlanWarning", PlannedSqlTests => "PlannedSqlTests", - PlannerChangeReconciliation => "PlannerChangeReconciliation", PlannerChangeResults => "PlannerChangeResults", - PlannerEntryResults => "PlannerEntryResults", PlannerIdentityContext => "PlannerIdentityContext", - PlannerModelEntryResults => "PlannerModelEntryResults", PlannerOverrides => "PlannerOverrides", PlannerPolicies => "PlannerPolicies", PlannerRelationsContext => "PlannerRelationsContext", PlannerResolvedActions => "PlannerResolvedActions", - PlannerRuntime => "PlannerRuntime", PlannerScope => "PlannerScope", - PlannerScopePruningResult => "PlannerScopePruningResult", PlannerScopeResolution => "PlannerScopeResolution", PlannerSelection => "PlannerSelection", - PlannerWarehouseState => "PlannerWarehouseState", PlaygroundCommandRequest => "PlaygroundCommandRequest", PlaygroundTemplate => "PlaygroundTemplate", PolyglotAnalysisDialect => "PolyglotAnalysisDialect", PolyglotAnalysisResult => "PolyglotAnalysisResult", - PreparedLintFiles => "PreparedLintFiles", - PreparedSqlLint => "PreparedSqlLint", - PreparedSqlLintResult => "PreparedSqlLintResult", Project => "Project", ProjectConfig => "ProjectConfig", ProjectGraph => "ProjectGraph", @@ -558,7 +544,6 @@ native_ids! { SqlBindingDiagnostic => "SqlBindingDiagnostic", SqlBindingResult => "SqlBindingResult", SqlExpansionContext => "SqlExpansionContext", - SqlExpansionReuse => "SqlExpansionReuse", SqlHookEntry => "SqlHookEntry", SqlLexicalSyntax => "SqlLexicalSyntax", SqlLogicalType => "SqlLogicalType", @@ -691,11 +676,9 @@ native_ids! { ApplyCommandWarehouse => "apply_command_warehouse", ApplyConfiguredMicrobatchLimits => "apply_configured_microbatch_limits", ApplyDeferredLocations => "apply_deferred_locations", - ApplyLintSuppressions => "apply_lint_suppressions", ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", ApplyUnitTestSelectors => "apply_unit_test_selectors", - AssembleModelPlanEntry => "assemble_model_plan_entry", AssembleNativeProjectResources => "assemble_native_project_resources", AssembleNativeSqlTests => "assemble_native_sql_tests", AssembleProject => "assemble_project", @@ -738,7 +721,6 @@ native_ids! { BuildFunctionLocations => "build_function_locations", BuildIdentity => "BUILD_IDENTITY", BuildIntermediateSourceMap => "build_intermediate_source_map", - BuildLintExpansionContext => "build_lint_expansion_context", BuildManifest => "build_manifest", BuildMergedDbtExecutionArgv => "build_merged_dbt_execution_argv", BuildMigrationFingerprint => "build_migration_fingerprint", @@ -747,8 +729,6 @@ native_ids! { BuildModelVersionIdentityMetadataJson => "build_model_version_identity_metadata_json", BuildModelWarnings => "build_model_warnings", BuildPhaseTimingTracker => "BuildPhaseTimingTracker", - BuildPlanEntries => "build_plan_entries", - BuildPlannerEntryResults => "build_planner_entry_results", BuildPlannerIdentityContext => "build_planner_identity_context", BuildPlannerRelationsContext => "build_planner_relations_context", BuildPlannerScope => "build_planner_scope", @@ -791,7 +771,6 @@ native_ids! { ClassifyCursorSqlType => "classify_cursor_sql_type", CliStyle => "CliStyle", CollectCompileDiagnostics => "collect_compile_diagnostics", - CollectLintProjectFiles => "collect_lint_project_files", CommandWarehouseScopeContext => "_command_warehouse_scope_context", CompareContracts => "compare_contracts", CompileDbtInteropProject => "compile_dbt_interop_project", @@ -805,10 +784,8 @@ native_ids! { CompiledModelRecord => "compiled_model_record", CompiledProjectFacts => "compiled_project_facts", CompiledProjectRecord => "compiled_project_record", - CompiledRelationKeys => "compiled_relation_keys", CompleteSemanticProject => "complete_semantic_project", ComputeCursorBounds => "compute_cursor_bounds", - ComputeModelCursorBounds => "compute_model_cursor_bounds", ComputeQueryHash => "compute_query_hash", ConnectionProgressReporter => "ConnectionProgressReporter", ContentDigest => "content_digest", @@ -838,7 +815,6 @@ native_ids! { DecideModelMigration => "decide_model_migration", DeclarationRows => "declaration_rows", DeclaredColumnAliases => "declared_column_aliases", - DeclaredRelationKeys => "declared_relation_keys", DeclaredRelations => "declared_relations", DescribeConnectionWarehouse => "describe_connection_warehouse", DescribeFullRebuildCause => "describe_full_rebuild_cause", @@ -862,7 +838,6 @@ native_ids! { DiscoverSourceFiles => "discover_source_files", DiscoverTestFiles => "discover_test_files", DynamicContractProof => "dynamic_contract_proof", - EarlyLintExpansions => "early_lint_expansions", EditContractModel => "edit_contract_model", EditContractSource => "edit_contract_source", EffectiveConfigConnection => "effective_config_connection", @@ -872,7 +847,6 @@ native_ids! { EffectivelyFullRefreshedModelNames => "effectively_full_refreshed_model_names", EmptyCursorWindow => "empty_cursor_window", EmptyInputRebuildRefusal => "empty_input_rebuild_refusal", - EncodeSqlRuleFindings => "encode_sql_rule_findings", EnforceModelExecutionLimit => "enforce_model_execution_limit", EnforcePlanSafetyPolicies => "enforce_plan_safety_policies", EvaluateModelContracts => "evaluate_model_contracts", @@ -887,13 +861,10 @@ native_ids! { ExecuteDbtSqlbuildWork => "execute_dbt_sqlbuild_work", ExecuteDbtWithoutStateTracking => "execute_dbt_without_state_tracking", ExecutionEventWriter => "ExecutionEventWriter", - ExpandFileBodies => "expand_file_bodies", ExpandProjectUpstreamKeys => "expand_project_upstream_keys", ExpandSqlMacros => "expand_sql_macros", - ExpandSqlWithSpans => "expand_sql_with_spans", ExplicitReferenceReport => "explicit_reference_report", ExplicitReferencesHelp => "explicit_references_help", - ExternalIdentifiersForHeaders => "external_identifiers_for_headers", ExternallyDescribedModels => "externally_described_models", ExtractSeedColumns => "extract_seed_columns", FinalizeRuleFindingsRows => "finalize_rule_findings_rows", @@ -949,9 +920,6 @@ native_ids! { LaunchRecordOffered => "_launch_record_offered", LaunchRecordWithheld => "_launch_record_withheld", LineageGraphNames => "lineage_graph_names", - LintNativeHeaders => "lint_native_headers", - LintNativeSqlBodies => "lint_native_sql_bodies", - LintStopScopeFunction => "lint_stop_scope", ListCostRuns => "list_cost_runs", LoadCompiledDbtManifest => "load_compiled_dbt_manifest", LoadDbtManifestIndex => "load_dbt_manifest_index", @@ -960,10 +928,8 @@ native_ids! { LoadProgressReporter => "LoadProgressReporter", LoadProjectConfigFile => "load_project_config_file", LoadRulesConfig => "load_rules_config", - LoadSqlRuleCache => "load_sql_rule_cache", LoaderDestinationPartsFunction => "loader_destination_parts", LocateModelBindingDiagnostics => "locate_model_binding_diagnostics", - LookupSqlRuleCache => "lookup_sql_rule_cache", MachineOutputMode => "machine_output_mode", MacroStoreToken => "macro_store_token", MaintainSqlbuildSkills => "maintain_sqlbuild_skills", @@ -972,7 +938,6 @@ native_ids! { MatchProjectSelector => "match_project_selector", MatchingDynamicFamilies => "matching_dynamic_families", MaximumAllowedStart => "maximum_allowed_start", - MeasurementBodyRanges => "measurement_body_ranges", MergeRecomputedModelChanges => "merge_recomputed_model_changes", MicrobatchGuidance => "microbatch_guidance", MigrationDestinationFingerprints => "migration_destination_fingerprints", @@ -993,7 +958,6 @@ native_ids! { ModelPlaceholders => "model_placeholders", ModelPythonHookNames => "model_python_hook_names", ModelSyntaxChecks => "model_syntax_checks", - NamedDeclarationRoleKind => "named_declaration_role_kind", NativeCompiledModel => "NativeCompiledModel", NativeCompiledProject => "NativeCompiledProject", NativeCompiledSeed => "NativeCompiledSeed", @@ -1005,8 +969,6 @@ native_ids! { NativeProgressProjector => "NativeProgressProjector", NativeProgressProjectorContext => "_native_progress_projector_context", NestedCommandProgressCallbacks => "NestedCommandProgressCallbacks", - NeutralizeLintAuditParameters => "neutralize_lint_audit_parameters", - NeutralizeLintContextInterpolation => "neutralize_lint_context_interpolation", NormalizeAnalysisSql => "normalize_analysis_sql", NormalizeCursorSnapshotGrain => "normalize_cursor_snapshot_grain", NormalizeDbtProfileConnection => "normalize_dbt_profile_connection", @@ -1026,8 +988,6 @@ native_ids! { PlanDbtInteropFromProject => "plan_dbt_interop_from_project", PlanHasExecutableWork => "plan_has_executable_work", PlanModel => "plan_model", - PlanModelMigrations => "plan_model_migrations", - PlanOldNameViews => "plan_old_name_views", PlanReferenceRenames => "plan_reference_renames", PlanRetention => "plan_retention", PlanRuleFixes => "plan_rule_fixes", @@ -1038,19 +998,13 @@ native_ids! { PlanStaticSqlTests => "plan_static_sql_tests", PlanTableTypes => "plan_table_types", PlanningProgressReporter => "PlanningProgressReporter", - PrepareExpandedFiles => "prepare_expanded_files", - PrepareLintSql => "prepare_lint_sql", - PrepareLintSqlBatch => "prepare_lint_sql_batch", PrepareModelHeaderTokens => "prepare_model_header_tokens", - PrepareNativeHeaderCache => "prepare_native_header_cache", - PrepareSqlRules => "prepare_sql_rules", PreparedCompileArtifacts => "PreparedCompileArtifacts", ProjectLineageViews => "project_lineage_views", ProjectRelativePath => "project_relative_path", ProjectWith => "project_with", PromotionConflictDiagnostics => "promotion_conflict_diagnostics", ProveNativeDynamicContracts => "prove_native_dynamic_contracts", - PrunePlannerExecutionScope => "prune_planner_execution_scope", PublicDeclarationIndexes => "public_declaration_indexes", PublicModelSchemaIndex => "public_model_schema_index", PythonReadSourceNamesFor => "python_read_source_names_for", @@ -1060,7 +1014,6 @@ native_ids! { ReadSelectedSeedFingerprints => "read_selected_seed_fingerprints", ReadsOutsideTargetLineage => "reads_outside_target_lineage", RebindDeclarationScope => "rebind_declaration_scope", - ReconcileExecutionChanges => "reconcile_execution_changes", RecordCompileInputReads => "record_compile_input_reads", RecordModelAnalyses => "record_model_analyses", RecordSettingsClassRead => "record_settings_class_read", @@ -1129,7 +1082,6 @@ native_ids! { ResolveSchemaActions => "resolve_schema_actions", ResolveSkippedViewChain => "resolve_skipped_view_chain", ResolveSourceReferences => "resolve_source_references", - ResolveSqlTestModelChains => "resolve_sql_test_model_chains", ResolveSqlbuildExecutionPlanOutput => "resolve_sqlbuild_execution_plan_output", ResolveSqlbuildModelSelectorNames => "resolve_sqlbuild_model_selector_names", ResolveStartCursorConfig => "resolve_start_cursor_config", @@ -1167,7 +1119,6 @@ native_ids! { RunInitCommand => "run_init_command", RunJanitorCommand => "run_janitor_command", RunLineageCommand => "run_lineage_command", - RunLint => "run_lint", RunLoadCommand => "run_load_command", RunPlanCommand => "run_plan_command", RunPlannedSqlRules => "run_planned_sql_rules", @@ -1184,14 +1135,12 @@ native_ids! { RunTestCommand => "run_test_command", RuntimeCursorProducerNames => "runtime_cursor_producer_names", SafeFormatFiles => "safe_format_files", - ScanHeaders => "scan_headers", SchemaValidations => "schema_validations", ScopeMetadataProjection => "scope_metadata_projection", SelectCatalogueRules => "select_catalogue_rules", SelectLatestFingerprints => "select_latest_fingerprints", SelectProjectGraph => "select_project_graph", SelectedRuleCodes => "selected_rule_codes", - SelectedSqlRuleCodes => "selected_sql_rule_codes", SessionOutcomeAnalysis => "session_outcome_analysis", SettingHelp => "setting_help", SharedTransientLineCoordinator => "shared_transient_line_coordinator", @@ -1201,13 +1150,8 @@ native_ids! { SplitProjectUnitTestSelectors => "split_project_unit_test_selectors", SplitSelectorExpansion => "split_selector_expansion", SplitUnitTestSelectors => "split_unit_test_selectors", - SqlBodyRanges => "sql_body_ranges", SqlCodeScanner => "SqlCodeScanner", - SqlFileRuleKey => "sql_file_rule_key", - SqlLintPreparation => "sql_lint_preparation", - SqlModelRuleKeys => "sql_model_rule_keys", SqlRefKey => "sql_ref_key", - SqlRuleLintKey => "sql_rule_lint_key", SqlTestModelChainNamesByKey => "sql_test_model_chain_names_by_key", SqlTestScanCache => "SqlTestScanCache", SqlValueValidationError => "SqlValueValidationError", @@ -1226,7 +1170,6 @@ native_ids! { UnreadableCursorInputsError => "unreadable_cursor_inputs_error", UpdateSqlbuildSkills => "update_sqlbuild_skills", UpstreamLoaderDependencyNames => "upstream_loader_dependency_names", - ValidRulesConfig => "valid_rules_config", ValidateAuthoredSqlTestCtes => "validate_authored_sql_test_ctes", ValidateCappedWatermarkInputs => "validate_capped_watermark_inputs", ValidateCliWarehouse => "validate_cli_warehouse", @@ -1241,7 +1184,6 @@ native_ids! { ValidateSqlSyntax => "validate_sql_syntax", WithBindingDiagnostics => "with_binding_diagnostics", WithDeclaredSourceReads => "with_declared_source_reads", - WithDuplicateLiteralHints => "with_duplicate_literal_hints", WithHonestModelWriteHashes => "with_honest_model_write_hashes", WithMigrationFingerprints => "with_migration_fingerprints", WithTypeRecovery => "with_type_recovery", @@ -1258,7 +1200,6 @@ native_ids! { WriteReusableCompile => "write_reusable_compile", WriteRuntimeTarget => "write_runtime_target", WriteSourceFreshnessRecords => "write_source_freshness_records", - WriteSqlRuleCache => "write_sql_rule_cache", WriteSqlbuildSkipNotice => "write_sqlbuild_skip_notice", YamlEntryLine => "yaml_entry_line", } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 327da1b23d..7430d50770 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::records::fields::{ build_values, cached_value, compared_values, frozen_error, ordered_comparison, record_spec, repr_text, store_cached, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFactory}; /// How a record field is filled when the caller does not pass it. @@ -40,8 +40,6 @@ pub(crate) enum FieldDefault { HostMember(HostId, &'static str), /// A native record class called with no arguments, evaluated for every instance. NativeCall(RecordId), - /// A native Rust class called with no arguments, evaluated for every instance. - NativeClassCall(NativeAttr), /// A native factory, evaluated for every instance. Factory(RecordFactory), /// `getattr(import(module), name)(text)`, evaluated for every instance. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs index f93d22b5f8..4ef70d6c7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs @@ -8,8 +8,9 @@ use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::analysis::relation_names::{ RelationRow, extract_sql_relation_names, }; +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; -const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; const RUNTIME_RELATION_PREFIX: &str = "SELECT 1 FROM "; fn keyword_call<'py>( @@ -31,9 +32,7 @@ fn relation_name<'py>( keywords.set_item("name", name)?; keywords.set_item("schema", schema)?; keywords.set_item("database", database)?; - py.import(REFERENCE_MODELS_MODULE)? - .getattr("RelationName")? - .call((), Some(&keywords)) + host::entry(py, ExecutorHost::RelationName)?.call((), Some(&keywords)) } /// Relations named in `sql` and temporary tables it creates, or None if unparseable. @@ -111,8 +110,7 @@ fn runtime_project_relations<'py>( relations: &Bound<'py, PyAny>, dialect: Option<&str>, ) -> PyResult> { - let models: Bound<'py, PyAny> = py.import(REFERENCE_MODELS_MODULE)?.into_any(); - let relation_type: Bound<'py, PyAny> = models.getattr("ProjectRelation")?; + let relation_type: Bound<'py, PyAny> = host::entry(py, ExecutorHost::ProjectRelation)?; let resolved: Bound<'py, PyDict> = PyDict::new(py); for item in relations.call_method0("items")?.try_iter()? { let (reference, qualified): (Bound<'py, PyAny>, String) = item?.extract()?; @@ -133,7 +131,7 @@ fn runtime_project_relations<'py>( resolved.set_item(relation, py.None())?; } keyword_call( - &models.getattr("ProjectRelationIndex")?, + &host::entry(py, ExecutorHost::ProjectRelationIndex)?, &[("relations", PyTuple::new(py, resolved.keys())?.into_any())], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs index 4aa6770210..fe0ba7ee75 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs @@ -7,12 +7,11 @@ use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_ use sqlbuild_analysis::identifier_fitting::main::fit_auxiliary_relation_name::fit_auxiliary_relation_name as fit_auxiliary; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::imported; - -const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; pub(crate) fn shared_error(py: Python<'_>, message: String) -> PyErr { - imported(py, SHARED_EXCEPTIONS_MODULE, "SharedInputError") + host::entry(py, FacadeHost::SharedInputError) .and_then(|class| class.call1((message,))) .map_or_else(|error| error, PyErr::from_value) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 963b752559..f333f5920c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; @@ -16,12 +17,13 @@ use crate::bindings::_helpers::render::declaration_expansion::{ DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, expand_declarations, span_rows, }; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; -use crate::bindings::_helpers::render::macro_expansion_result::expand_sql_macros; +use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::render::macro_expansion_result::{ExpansionRequest, macro_output}; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; -const FUNCTOOLS_MODULE: &str = "functools"; const DECLARATION_FIELDS: [&str; 4] = [ "enums", "constants", @@ -111,17 +113,10 @@ fn declarations_expanded<'py>( return Ok((sql.to_owned(), Vec::new())); } let context: Bound<'py, PyAny> = declaration_context(py, declarations)?; - let render_constant: Bound<'py, PyAny> = imported(py, FUNCTOOLS_MODULE, "partial")?.call( - (registry::native(py, NativeAttr::RenderConstantDeclaration)?,), - Some(&keywords( - py, - &[ - ("value_renderer", value_renderer.clone()), - ("collection_rendering", collection_rendering.clone()), - ("file_path", file_path.clone()), - ], - )?), - )?; + let rendering: ConstantRendering = ConstantRendering::new( + value_renderer.clone().unbind(), + collection_rendering.clone().unbind(), + ); let classes = ExpansionClasses { usage_record: registry::class(py, RecordId::UsageRecord)?, compile_error: registry::error(py, ErrorId::CompileInput)?, @@ -137,60 +132,136 @@ fn declarations_expanded<'py>( relationship_visibility: false, path: &path, }, - render_constant: &render_constant, + rendering: &rendering, classes: &classes, })?; let spans: Vec = span_rows(&expanded.spliced); Ok((expanded.spliced.sql, spans)) } -/// Expand authored SQL, returning each pass's substitution spans in order. -fn authored_sql_with_spans<'py>(request: &Request<'_, 'py>) -> PyResult> { - let py: Python<'py> = request.py(); - let none = || py.None().into_bound(py); - let file_path: Bound<'py, PyAny> = request.get("file_path")?; +/// Authored SQL expanded through vars, declarations and macros, with each pass's spans. +pub(crate) struct AuthoredExpansion { + pub(crate) sql: String, + /// Interpolation, declaration and macro spans, in pass order. + pub(crate) passes: [Vec; 3], +} + +/// What one authored SQL expansion reads. +struct AuthoredSources<'a, 'py> { + sql: &'a Bound<'py, PyAny>, + file_path: &'a Bound<'py, PyAny>, + effective_vars: &'a Bound<'py, PyAny>, + context_values: &'a Bound<'py, PyAny>, + declarations: &'a Bound<'py, PyAny>, + value_renderer: &'a Bound<'py, PyAny>, + collection_rendering: &'a Bound<'py, PyAny>, + loaded_macros: &'a Bound<'py, PyAny>, + macro_context: &'a Bound<'py, PyAny>, + declaration_resolver: Option>, +} + +fn authored_expansion(sources: AuthoredSources<'_, '_>) -> PyResult { + let py: Python<'_> = sources.sql.py(); let (interpolated, interpolation_spans) = substituted( - &request.get("sql")?, - &file_path, - ( - &request.get("effective_vars")?, - &request.optional("context_values")?.unwrap_or_else(none), - ), + sources.sql, + sources.file_path, + (sources.effective_vars, sources.context_values), )?; let (declared, declaration_spans) = declarations_expanded( &interpolated, - &file_path, + sources.file_path, ( - &request.optional("declarations")?.unwrap_or_else(none), - &request.get("value_renderer")?, - &request.get("collection_rendering")?, + sources.declarations, + sources.value_renderer, + sources.collection_rendering, ), )?; - let macro_options: Bound<'py, PyDict> = keywords( - py, - &[ - ("sql", PyString::new(py, &declared).into_any()), - ("file_path", file_path), - ("loaded_macros", request.get("loaded_macros")?), - ("macro_context", request.get("macro_context")?), - ( - "declaration_resolver", - request - .optional("declaration_resolver")? - .unwrap_or_else(none), - ), - ], - )?; - let expansion: Bound<'py, PyAny> = expand_sql_macros(py, Some(macro_options))?; - let passes: Bound<'py, PyTuple> = PyTuple::new( + let request: ExpansionRequest<'_> = ExpansionRequest { + sql: PyString::new(py, &declared), + file_path: sources.file_path.clone(), + loaded_macros: sources.loaded_macros.downcast()?.clone(), + macro_overrides: None, + macro_context: sources.macro_context.clone(), + declaration_resolver: sources.declaration_resolver, + declarations: None, + consumer: None, + }; + Ok(match macro_output(request)? { + Some(output) => AuthoredExpansion { + sql: output.sql, + passes: [ + interpolation_spans, + declaration_spans, + output.spans.into_iter().map(span_row).collect(), + ], + }, + None => AuthoredExpansion { + sql: declared, + passes: [interpolation_spans, declaration_spans, Vec::new()], + }, + }) +} + +/// `(sql, passes)` with each pass's `ExpansionSpan` objects. +fn expansion_tuple<'py>( + py: Python<'py>, + expansion: &AuthoredExpansion, +) -> PyResult> { + let passes: Vec> = expansion + .passes + .iter() + .map(|spans| span_objects(py, spans)) + .collect::>()?; + PyTuple::new( py, [ - span_objects(py, &interpolation_spans)?.into_any(), - span_objects(py, &declaration_spans)?.into_any(), - expansion.getattr("spans")?, + PyString::new(py, &expansion.sql).into_any(), + PyTuple::new(py, passes)?.into_any(), ], - )?; - PyTuple::new(py, [expansion.getattr("sql")?, passes.into_any()]) + ) +} + +/// Expand authored SQL, returning each pass's substitution spans in order. +fn authored_sql_with_spans<'py>(request: &Request<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.py(); + let none = || py.None().into_bound(py); + let expansion: AuthoredExpansion = authored_expansion(AuthoredSources { + sql: &request.get("sql")?, + file_path: &request.get("file_path")?, + effective_vars: &request.get("effective_vars")?, + context_values: &request.optional("context_values")?.unwrap_or_else(none), + declarations: &request.optional("declarations")?.unwrap_or_else(none), + value_renderer: &request.get("value_renderer")?, + collection_rendering: &request.get("collection_rendering")?, + loaded_macros: &request.get("loaded_macros")?, + macro_context: &request.get("macro_context")?, + declaration_resolver: request + .optional("declaration_resolver")? + .filter(|resolver| !resolver.is_none()), + })?; + expansion_tuple(py, &expansion) +} + +/// `sql` expanded with lint's expansion inputs, as `file_path`'s declarations see them. +pub(crate) fn expand_with_inputs( + inputs: &SqlExpansionInputs, + sql: &str, + file_path: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = file_path.py(); + let resolver: Bound<'_, PyAny> = inputs.declaration_resolver.bind(py).clone(); + authored_expansion(AuthoredSources { + sql: &PyString::new(py, sql).into_any(), + file_path, + effective_vars: inputs.effective_vars.bind(py), + context_values: &py.None().into_bound(py), + declarations: &input_declarations(inputs, file_path)?, + value_renderer: inputs.value_renderer.bind(py), + collection_rendering: inputs.collection_rendering.bind(py), + loaded_macros: inputs.loaded_macros.bind(py), + macro_context: inputs.macro_context.bind(py), + declaration_resolver: (!resolver.is_none()).then_some(resolver), + }) } /// Expand authored SQL, returning each pass's substitution spans in order. @@ -202,29 +273,29 @@ fn expand_authored_sql_with_spans<'py>( authored_sql_with_spans(&Request::new("expand_authored_sql_with_spans", request)?) } -/// The declarations visible to `file_path`: the resolver's, else the context's, plus model-local. -fn path_declarations<'py>( - context: &Bound<'py, PyAny>, +/// The declarations visible to `file_path`: the resolver's, else the inputs', plus model-local. +fn input_declarations<'py>( + inputs: &SqlExpansionInputs, file_path: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = context.py(); + let py: Python<'py> = file_path.py(); let resolution_class: Bound<'py, PyAny> = registry::class(py, RecordId::DeclarationResolutionContext)?; - let resolver: Bound<'py, PyAny> = context.getattr("declaration_resolver")?; + let resolver: &Bound<'py, PyAny> = inputs.declaration_resolver.bind(py); let declarations: Bound<'py, PyAny> = if resolver.is_none() { call_with( &resolution_class, &[ - ("enums", context.getattr("enums")?), - ("constants", context.getattr("constants")?), + ("enums", inputs.enums.bind(py).clone()), + ("constants", inputs.constants.bind(py).clone()), ], )? } else { - resolved_declaration_context(&resolver, file_path, None)? + resolved_declaration_context(resolver, file_path, None)? + }; + let Some(local) = inputs.local_declarations.bind(py).get_item(file_path)? else { + return Ok(declarations); }; - let local: Bound<'py, PyAny> = context - .getattr("local_declarations")? - .call_method1("get", (file_path,))?; if local.is_none() { return Ok(declarations); } @@ -252,32 +323,15 @@ fn path_declarations<'py>( #[pyfunction] #[pyo3(signature = (*, sql, file_path, context))] fn expand_sql_with_spans<'py>( - sql: &Bound<'py, PyAny>, + sql: &str, file_path: &Bound<'py, PyAny>, context: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = sql.py(); - let arguments: Bound<'py, PyDict> = keywords( - py, - &[ - ("sql", sql.clone()), - ("file_path", file_path.clone()), - ("effective_vars", context.getattr("effective_vars")?), - ("loaded_macros", context.getattr("loaded_macros")?), - ("macro_context", context.getattr("macro_context")?), - ("declarations", path_declarations(context, file_path)?), - ( - "declaration_resolver", - context.getattr("declaration_resolver")?, - ), - ("value_renderer", context.getattr("value_renderer")?), - ( - "collection_rendering", - context.getattr("collection_rendering")?, - ), - ], - )?; - authored_sql_with_spans(&Request::new("expand_sql_with_spans", Some(&arguments))?) + let inputs: SqlExpansionInputs = SqlExpansionInputs::from_record(context)?; + expansion_tuple( + file_path.py(), + &expand_with_inputs(&inputs, sql, file_path)?, + ) } /// Replace SQL interpolation tokens, returning the span of every substitution. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 599a082a4d..57f1fa829d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -11,6 +11,8 @@ use sqlbuild_analysis::compiled_project::models::{ DeclarationSplice, InaccessibleDeclaration, SplicedSql, }; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; + const ENUM_REFERENCE_KIND: u8 = 0; const ENUM_KIND: &str = "enum"; const CONSTANT_KIND: &str = "constant"; @@ -57,14 +59,14 @@ pub(crate) struct ExpansionConsumer<'a, 'py> { pub(crate) path: &'a str, } -/// One expansion: the SQL, its scan, the visible declarations and the adapter's constant renderer. +/// One expansion: the SQL, its scan, the visible declarations and the adapter's constant rendering. pub(crate) struct DeclarationExpansion<'a, 'py> { pub(crate) sql: &'a str, pub(crate) references: &'a [ScannedReference], pub(crate) stop: Option, pub(crate) context: &'a Bound<'py, PyAny>, pub(crate) consumer: ExpansionConsumer<'a, 'py>, - pub(crate) render_constant: &'a Bound<'py, PyAny>, + pub(crate) rendering: &'a ConstantRendering, pub(crate) classes: &'a ExpansionClasses<'py>, } @@ -145,12 +147,9 @@ fn constant_text<'py>( name: &str, ) -> PyResult<(String, Bound<'py, PyAny>)> { let declaration: Bound<'py, PyAny> = visible_declaration(request, CONSTANT_KIND, name)?; - let arguments: Bound<'py, PyDict> = PyDict::new(declaration.py()); - arguments.set_item(DECLARATION, declaration)?; let rendered: String = request - .render_constant - .call((), Some(&arguments))? - .extract()?; + .rendering + .render(&declaration, request.consumer.path)?; Ok((rendered, visibility(request, CONSTANT_VISIBILITY, name)?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs index 6bfd127f0e..2f2f6c537c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs @@ -54,7 +54,7 @@ fn index_declarations<'py>( /// Every public enum and constant by name, collision-checked. #[pyfunction] -fn public_declaration_indexes<'py>( +pub(crate) fn public_declaration_indexes<'py>( inputs: Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { let py: Python<'py> = inputs.py(); @@ -166,7 +166,9 @@ fn lowered_name(column: &Bound<'_, PyAny>) -> PyResult { /// Every public model schema by name, with its inherited columns first, collision-checked. #[pyfunction] -fn public_model_schema_index<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn public_model_schema_index<'py>( + inputs: Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = inputs.py(); let mut authored: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); let mut by_name: HashMap = HashMap::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 13b5c6e3a5..58f0c57528 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -1,10 +1,10 @@ //! Declaration scope resolvers, per-path declaration contexts and adapter-rendered constants. use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyclass, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -371,7 +371,7 @@ fn rendered_constant<'py>( pub(crate) fn constant_declaration_sql<'py>( declaration: &Bound<'py, PyAny>, value_renderer: &Bound<'py, PyAny>, - (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), + (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&str>), ) -> PyResult { let py: Python<'py> = declaration.py(); let render_as: Bound<'py, PyAny> = declaration.getattr("render_as")?; @@ -392,16 +392,16 @@ pub(crate) fn constant_declaration_sql<'py>( return Err(error); } let relative_path: Bound<'py, PyAny> = declaration.getattr("relative_path")?; - let location: Bound<'py, PyAny> = match file_path { - Some(file_path) if !file_path.is_none() => file_path.clone(), - _ => relative_path.clone(), + let location: String = match file_path { + Some(file_path) => file_path.to_owned(), + None => relative_path.str()?.to_string(), }; let compile_error: Bound<'py, PyAny> = registry::error(py, ErrorId::CompileInput)?.call1((format!( "{} constant '{}' could not be rendered in '{}' by adapter '{}' as {}: {}", relative_path.str()?, declaration.getattr("name")?.str()?, - location.str()?, + location, value_renderer.getattr("adapter_name")?.str()?, selected.getattr("value")?.str()?, raised.str()?, @@ -419,13 +419,48 @@ fn render_constant_declaration<'py>( collection_rendering: &Bound<'py, PyAny>, file_path: Option>, ) -> PyResult { + let location: Option = match file_path { + Some(file_path) => Some(file_path.str()?.to_string()), + None => None, + }; constant_declaration_sql( declaration, value_renderer, - (collection_rendering, file_path.as_ref()), + (collection_rendering, location.as_deref()), ) } +/// The adapter's constant rendering a compile's declaration expansions share. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ConstantRendering { + value_renderer: Py, + collection_rendering: Py, +} + +impl ConstantRendering { + /// Rendering with the adapter's value renderer and the project's collection rendering. + pub(crate) fn new(value_renderer: Py, collection_rendering: Py) -> Self { + Self { + value_renderer, + collection_rendering, + } + } + + /// One visible constant as SQL, its failures located at `file_path`. + pub(crate) fn render( + &self, + declaration: &Bound<'_, PyAny>, + file_path: &str, + ) -> PyResult { + let py: Python<'_> = declaration.py(); + constant_declaration_sql( + declaration, + self.value_renderer.bind(py), + (self.collection_rendering.bind(py), Some(file_path)), + ) + } +} + /// One enum member value as a SQL scalar literal. #[pyfunction] #[pyo3(signature = (*, value))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index ee59c533ca..4b14f89002 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -41,15 +41,15 @@ const GENERATED_ALLOWED: &str = "macro-generated references"; static EXPANSION_CLASSES: PyOnceLock> = PyOnceLock::new(); /// One expansion's inputs, as Python's `expand_sql_macros_result` takes them. -struct ExpansionRequest<'py> { - sql: Bound<'py, PyString>, - file_path: Bound<'py, PyAny>, - loaded_macros: Bound<'py, PyDict>, - macro_overrides: Option>, - macro_context: Bound<'py, PyAny>, - declaration_resolver: Option>, - declarations: Option>, - consumer: Option>, +pub(crate) struct ExpansionRequest<'py> { + pub(crate) sql: Bound<'py, PyString>, + pub(crate) file_path: Bound<'py, PyAny>, + pub(crate) loaded_macros: Bound<'py, PyDict>, + pub(crate) macro_overrides: Option>, + pub(crate) macro_context: Bound<'py, PyAny>, + pub(crate) declaration_resolver: Option>, + pub(crate) declarations: Option>, + pub(crate) consumer: Option>, } fn generated_classes(py: Python<'_>) -> PyResult> { @@ -190,12 +190,13 @@ fn result_model<'py>( call_with(®istry::class(py, RecordId::MacroExpansionResult)?, pairs) } -fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult> { - let py: Python<'py> = request.sql.py(); +/// The macro expansion of `request`'s SQL, or `None` when it calls no macro. +pub(crate) fn macro_output(request: ExpansionRequest<'_>) -> PyResult> { + let py: Python<'_> = request.sql.py(); if !request.sql.to_str()?.contains(MACRO_TOKEN) { - return result_model(py, &[("sql", request.sql.into_any())]); + return Ok(None); } - let declarations: Bound<'py, PyAny> = match (request.declarations, request.declaration_resolver) + let declarations: Bound<'_, PyAny> = match (request.declarations, request.declaration_resolver) { (Some(declarations), _) => declarations, (None, Some(resolver)) => { @@ -221,9 +222,17 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult = Bound::new(py, expansion)?; - let bridge: Bound<'py, PyAny> = host::entry(py, UserCodeHost::ActiveMacroBridge)?.call0()?; - let output: ExpansionOutput = expanded(&expansion, request.sql.to_str()?, &bridge)?; + let expansion: Bound<'_, NativeMacroExpansion> = Bound::new(py, expansion)?; + let bridge: Bound<'_, PyAny> = host::entry(py, UserCodeHost::ActiveMacroBridge)?.call0()?; + expanded(&expansion, request.sql.to_str()?, &bridge).map(Some) +} + +fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult> { + let py: Python<'py> = request.sql.py(); + let sql: Bound<'py, PyString> = request.sql.clone(); + let Some(output) = macro_output(request)? else { + return result_model(py, &[("sql", sql.into_any())]); + }; let span_class: Bound<'py, PyAny> = registry::class(py, RecordId::ExpansionSpan)?; let mut spans: Vec> = Vec::with_capacity(output.spans.len()); for span in output.spans { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs index 670af04e79..94767d13ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -67,31 +67,6 @@ const R_SQL_TEST_RULES_CONFIG: RecordSpec = RecordSpec { FieldDefault::Text("pipelines"), )], }; -const R_PREPARED_SQL_LINT_RESULT: RecordSpec = RecordSpec { - name: "PreparedSqlLintResult", - doc: "Reusable expansion context and completed preliminary SQL checks.", - fields: &[ - FieldSpec::required("result"), - FieldSpec::required("context"), - ], -}; -const R_SQL_EXPANSION_REUSE: RecordSpec = RecordSpec { - name: "SqlExpansionReuse", - doc: "Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.", - fields: &[ - FieldSpec::required("discovered_inputs"), - FieldSpec::required("declaration_scope"), - ], -}; -const R_PREPARED_SQL_LINT: RecordSpec = RecordSpec { - name: "PreparedSqlLint", - doc: "Process-local SQL checks started after compiler expansion has completed.", - fields: &[ - FieldSpec::required("codes"), - FieldSpec::required("future"), - FieldSpec::new("project_dir", FieldDefault::None), - ], -}; const R_SQL_RULES_EVALUATION: RecordSpec = RecordSpec { name: "SqlRulesEvaluation", doc: "SQL rule findings of one run, their cache accounting and elapsed time.", @@ -397,9 +372,6 @@ pub(crate) const RULE_RECORDS: &[RecordSpec] = &[ R_THRESHOLD_OVERRIDE, R_RULES_CACHE_CONFIG, R_SQL_TEST_RULES_CONFIG, - R_PREPARED_SQL_LINT_RESULT, - R_SQL_EXPANSION_REUSE, - R_PREPARED_SQL_LINT, R_SQL_RULES_EVALUATION, R_RULE_RUN_PLAN, R_RULES_RUN_RESULT, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs index 41f064c542..674fc5886f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs @@ -1,182 +1,43 @@ //! DSL header scanning, header lint rules and header formatting fixes over Python lint models. -use std::cell::RefCell; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::dsl_headers::constants::RULE_HEADER_PARSE; use sqlbuild_rules::dsl_headers::main::format_headers::format_headers; -use sqlbuild_rules::dsl_headers::main::inner_header_text::inner_header_text; use sqlbuild_rules::dsl_headers::main::lint_headers::lint_headers; use sqlbuild_rules::dsl_headers::main::measurement_body_ranges::measurement_body_ranges as measurement_ranges; use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers as scan; use sqlbuild_rules::dsl_headers::main::sql_body_ranges::sql_body_ranges as body_ranges; -use sqlbuild_rules::dsl_headers::models::{ - HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, ParseFailure, ParseOutcome, -}; -use sqlbuild_rules::dsl_headers::types::HeaderParser; +use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderSpan, HeaderViolation}; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; +use crate::bindings::_helpers::rules::native_header_parser::NativeHeaderParser; -const PATHLIB_MODULE: &str = "pathlib"; -const SENTINEL_PATH: &str = ""; -const ENGINE: &str = "sqlbuild"; const LINE_FEED: &str = "\n"; const CARRIAGE_RETURN_LINE_FEED: &str = "\r\n"; -const FUNCTION_KIND: &str = "FUNCTION"; - -/// Header values parsed by `parse_header_values`, with Python's error classes. -struct PyHeaderParser<'py> { - parse: Bound<'py, PyAny>, - path: Bound<'py, PyAny>, - syntax_error: Bound<'py, PyAny>, - discovery_error: Bound<'py, PyAny>, - /// The header syntax error of the last rejected parse. - rejected: RefCell>, -} - -impl<'py> PyHeaderParser<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - parse: registry::native(py, NativeAttr::ParseHeaderValues)?, - path: imported(py, PATHLIB_MODULE, "Path")?.call1((SENTINEL_PATH,))?, - syntax_error: registry::error(py, ErrorId::ModelSqlParse)?, - discovery_error: registry::error(py, ErrorId::Discovery)?, - rejected: RefCell::new(None), - }) - } - - fn failure(&self, error: &PyErr) -> PyResult { - let py: Python<'py> = self.parse.py(); - let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); - let help: Option = if value.is_instance(&self.discovery_error)? { - value.getattr("help")?.extract()? - } else { - None - }; - Ok(ParseFailure { - message: value.str()?.extract()?, - help, - }) - } -} - -impl<'py> HeaderParser for PyHeaderParser<'py> { - type Values = Bound<'py, PyAny>; - type Error = PyErr; - - fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { - let py: Python<'py> = self.parse.py(); - let parsed: PyResult> = call_with( - &self.parse, - &[ - ("header", PyString::new(py, inner).into_any()), - ("file_path", self.path.clone()), - ("statement_name", PyString::new(py, kind).into_any()), - ("header_line", py.None().into_bound(py)), - ], - ); - let error: PyErr = match parsed { - Ok(values) => return ParseOutcome::Parsed(values), - Err(error) => error, - }; - let failure: ParseFailure = match self.failure(&error) { - Ok(failure) => failure, - Err(failed) => { - return ParseOutcome::Raised( - failed, - ParseFailure { - message: String::new(), - help: None, - }, - ); - } - }; - if error.is_instance(py, &self.syntax_error) { - self.rejected.replace(Some(error)); - ParseOutcome::Rejected(failure) - } else { - ParseOutcome::Raised(error, failure) - } - } - - fn description(&self, values: &Self::Values) -> PyResult> { - let description: Bound<'py, PyAny> = values.call_method1("get", ("description",))?; - if description.is_instance_of::() { - return description.extract().map(Some); - } - Ok(None) - } - - fn same(&self, left: &Self::Values, right: &Self::Values) -> PyResult { - left.eq(right) - } -} - -/// The models and enums header results are built as. -struct LintClasses<'py> { - violation: Bound<'py, PyAny>, - severity: Bound<'py, PyAny>, -} - -impl<'py> LintClasses<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - violation: registry::class(py, RecordId::LintViolation)?, - severity: registry::class(py, RecordId::LintSeverity)?, - }) - } - - fn violation( - &self, - file_path: &Bound<'py, PyAny>, - violation: HeaderViolation, - ) -> PyResult> { - let py: Python<'py> = file_path.py(); - let severity: &str = match violation.severity { - HeaderSeverity::Fault => "FAULT", - HeaderSeverity::Warning => "WARNING", - }; - call_with( - &self.violation, - &[ - ("file_path", file_path.clone()), - ("line", violation.line.into_pyobject(py)?.into_any()), - ("column", violation.column.into_pyobject(py)?.into_any()), - ("code", PyString::new(py, violation.code).into_any()), - ("message", PyString::new(py, &violation.message).into_any()), - ("severity", self.severity.getattr(severity)?), - ("engine", PyString::new(py, ENGINE).into_any()), - ( - "remediation", - PyString::new(py, &violation.remediation).into_any(), - ), - ], - ) - } - fn violations( - &self, - file_path: &Bound<'py, PyAny>, - violations: Vec, - ) -> PyResult>> { - violations - .into_iter() - .map(|violation| self.violation(file_path, violation)) - .collect() - } +/// Header violations against `file_path` as `LintViolation` records. +fn violation_objects<'py>( + file_path: &Bound<'py, PyAny>, + violations: Vec, +) -> PyResult>> { + let classes: ViolationClasses<'py> = ViolationClasses::new(file_path.py())?; + violations + .into_iter() + .map(|violation| classes.violation(file_path, &LintViolationRow::header(0, violation))) + .collect() } fn text(contents: &Bound<'_, PyAny>) -> PyResult> { Ok(contents.extract::()?.chars().collect()) } -fn rules_config(config: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn rules_config(config: &Bound<'_, PyAny>) -> PyResult { Ok(HeaderRulesConfig { line_width: config.getattr("line_width")?.extract()?, max_description_lines: config.getattr("max_description_lines")?.extract()?, @@ -246,13 +107,13 @@ fn measurement_body_ranges<'py>(contents: &Bound<'py, PyAny>) -> PyResult( +/// The header rule violations of `headers` in `contents`. +pub(crate) fn header_violations( contents: &[char], headers: &[HeaderSpan], - config: &Bound<'py, PyAny>, -) -> PyResult> { - let parser: PyHeaderParser<'py> = PyHeaderParser::new(config.py())?; - lint_headers(contents, headers, rules_config(config)?, &parser) + config: HeaderRulesConfig, +) -> Result, String> { + lint_headers(contents, headers, config, &NativeHeaderParser) } /// Every header rule violation of `headers` in `contents`. @@ -265,9 +126,9 @@ fn lint_native_headers<'py>( config: &Bound<'py, PyAny>, ) -> PyResult> { let violations: Vec = - header_violations(&text(contents)?, &spans(headers)?, config)?; - let classes: LintClasses<'py> = LintClasses::new(contents.py())?; - PyTuple::new(contents.py(), classes.violations(file_path, violations)?) + header_violations(&text(contents)?, &spans(headers)?, rules_config(config)?) + .map_err(value_error)?; + PyTuple::new(contents.py(), violation_objects(file_path, violations)?) } /// `contents` with header formatting fixes applied, and the faults relocation reported. @@ -279,47 +140,28 @@ fn format_native_headers<'py>( config: &Bound<'py, PyAny>, ) -> PyResult<(String, Bound<'py, PyTuple>)> { let py: Python<'py> = contents.py(); - let parser: PyHeaderParser<'py> = PyHeaderParser::new(py)?; - let (formatted, faults) = format_headers(&text(contents)?, rules_config(config)?, &parser)?; - let classes: LintClasses<'py> = LintClasses::new(py)?; + let (formatted, faults) = + format_headers(&text(contents)?, rules_config(config)?, &NativeHeaderParser) + .map_err(value_error)?; Ok(( formatted.into_iter().collect(), - PyTuple::new(py, classes.violations(file_path, faults)?)?, + PyTuple::new(py, violation_objects(file_path, faults)?)?, )) } /// The header-parse faults of one file. fn parse_faults<'py>( - classes: &LintClasses<'py>, (file_path, contents): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), config: &Bound<'py, PyAny>, ) -> PyResult>> { let contents: Vec = text(contents)?; let violations: Vec = - header_violations(&contents, &scan(&contents, false), config)? + header_violations(&contents, &scan(&contents, false), rules_config(config)?) + .map_err(value_error)? .into_iter() .filter(|violation| violation.code == RULE_HEADER_PARSE) .collect(); - classes.violations(file_path, violations) -} - -/// Batch parse formatter-owned headers before per-file formatting and linting. -#[pyfunction] -#[pyo3(signature = (*, files))] -fn prepare_native_header_cache(files: &Bound<'_, PyDict>) -> PyResult<()> { - let mut headers: Vec = Vec::new(); - for contents in files.values() { - let contents: Vec = text(&contents)?; - for header in scan(&contents, false) { - headers.push( - inner_header_text(&contents[header.start..header.end]) - .iter() - .collect(), - ); - } - } - registry::native(files.py(), NativeAttr::PrepareModelHeaderTokens)?.call1((headers,))?; - Ok(()) + violation_objects(file_path, violations) } /// Parse-safe files, and every file's authored newline convention. @@ -330,11 +172,10 @@ fn safe_format_files<'py>( config: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { let py: Python<'py> = files.py(); - let classes: LintClasses<'py> = LintClasses::new(py)?; let safe: Bound<'py, PyDict> = PyDict::new(py); let newlines: Bound<'py, PyDict> = PyDict::new(py); for (file_path, contents) in files.iter() { - if parse_faults(&classes, (&file_path, &contents), config)?.is_empty() { + if parse_faults((&file_path, &contents), config)?.is_empty() { safe.set_item(&file_path, &contents)?; } } @@ -362,11 +203,9 @@ fn reject_unparseable_header_rewrites<'py>( config: &Bound<'py, PyAny>, faults: &Bound<'py, PyList>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyList>)> { - let classes: LintClasses<'py> = LintClasses::new(updated.py())?; let accepted: Bound<'py, PyDict> = updated.copy()?; for (file_path, contents) in updated.iter() { - let parse: Vec> = - parse_faults(&classes, (&file_path, &contents), config)?; + let parse: Vec> = parse_faults((&file_path, &contents), config)?; if parse.is_empty() { continue; } @@ -380,57 +219,16 @@ fn reject_unparseable_header_rewrites<'py>( Ok((accepted, faults.clone())) } -/// The SQL function argument names a FUNCTION header declares, lowercased and sorted. -#[pyfunction] -#[pyo3(signature = (*, contents, headers))] -fn external_identifiers_for_headers<'py>( - contents: &Bound<'py, PyAny>, - headers: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = contents.py(); - let text: Vec = text(contents)?; - let parser: PyHeaderParser<'py> = PyHeaderParser::new(py)?; - for header in spans(headers)? { - if header.kind != FUNCTION_KIND { - continue; - } - let inner: String = inner_header_text(&text[header.start..header.end]) - .iter() - .collect(); - let values: Bound<'py, PyAny> = match parser.parse(&header.kind, &inner) { - ParseOutcome::Parsed(values) => values, - ParseOutcome::Rejected(_) => { - return Err(parser.rejected.take().unwrap_or_else(|| PyErr::fetch(py))); - } - ParseOutcome::Raised(error, _) => return Err(error), - }; - let arguments: Bound<'py, PyAny> = values.call_method1("get", ("arguments",))?; - let Ok(arguments) = arguments.downcast::() else { - continue; - }; - let mut names: Vec = arguments - .keys() - .iter() - .map(|name| name.str()?.call_method0("lower")?.extract()) - .collect::>()?; - names.sort_unstable(); - return PyTuple::new(py, names); - } - Ok(PyTuple::empty(py)) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(scan_headers, module)?)?; module.add_function(wrap_pyfunction!(sql_body_ranges, module)?)?; module.add_function(wrap_pyfunction!(measurement_body_ranges, module)?)?; module.add_function(wrap_pyfunction!(lint_native_headers, module)?)?; module.add_function(wrap_pyfunction!(format_native_headers, module)?)?; - module.add_function(wrap_pyfunction!(prepare_native_header_cache, module)?)?; module.add_function(wrap_pyfunction!(safe_format_files, module)?)?; module.add_function(wrap_pyfunction!( reject_unparseable_header_rewrites, module )?)?; - module.add_function(wrap_pyfunction!(external_identifiers_for_headers, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index a44077bd9b..6ea9a18e7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -3,187 +3,282 @@ use std::path::PathBuf; use std::time::Instant; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; +use std::thread::JoinHandle; + +use pyo3::exceptions::{PyException, PyRuntimeError}; use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, + Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRefMut, + PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::types::PyTuple; +use pyo3::{pyclass, pymethods}; +use sqlbuild_rules::lint_text::models::LintSqlOptions; use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path; +use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ + SqlExpansionInputs, present, +}; +use crate::bindings::_helpers::native_threads::interruptible_join::{Joined, join_interruptibly}; +use crate::bindings::_helpers::rules::lint_expansion::{ + LintExpansionSources, lint_expansion_inputs, +}; +use crate::bindings::_helpers::rules::lint_inputs::{model_key_row, model_lint_expansion}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, +}; +use crate::bindings::_helpers::rules::relation_keys::{ + RelationKeyRow, RelationKeys, relation_keys, +}; +use crate::bindings::_helpers::rules::rule_cache::stored_entries; +use crate::bindings::_helpers::rules::rules_config::rules_config_of; /// The lint thresholds used when the project leaves them unset. const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; -const FUTURES_MODULE: &str = "concurrent.futures"; -const THREADING_MODULE: &str = "threading"; -const LINT_THREAD_PREFIX: &str = "sqlbuild-lint"; +const MAX_LITERAL_LENGTH: &str = "max_literal_length"; +const MAX_RANKING_ORDER_BY: &str = "max_ranking_order_by"; +const LINT_THREAD: &str = "sqlbuild-lint"; const SQL_RULES_CACHE_FILE: &str = "sql.json"; /// The SQL rule result cache file of `project_dir`. -pub(crate) fn sql_rule_cache_path<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let root: PathBuf = project_dir.extract()?; - Ok(rules_bulk_cache_path(&root, SQL_RULES_CACHE_FILE) - .into_pyobject(project_dir.py())? - .into_any()) +pub(crate) fn sql_rule_cache_path(project_dir: &Path) -> PathBuf { + rules_bulk_cache_path(project_dir, SQL_RULES_CACHE_FILE) } -/// The native SQL lint configuration that SQL rules run with. -pub(crate) fn sql_lint_config<'py>( - dialect: &Bound<'py, PyAny>, +/// The native SQL lint settings that SQL rules run with. +pub(crate) fn sql_lint_settings( + dialect: &str, codes: &[String], - thresholds: &Bound<'py, PyAny>, - (relation_keys, allow_suppressions): (Bound<'py, PyAny>, Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = dialect.py(); - let threshold = |key: &str, default: i64| -> PyResult> { - thresholds.call_method1("get", (key, default)) - }; - call_with( - ®istry::class(py, RecordId::LintConfig)?, - &[ - ("dialect", dialect.clone()), - ("enabled_native_rules", PyTuple::new(py, codes)?.into_any()), - ("header_rules_enabled", false.into_bound_py_any(py)?), - ( - "max_literal_length", - threshold("max_literal_length", DEFAULT_MAX_LITERAL_LENGTH)?, - ), - ( - "max_ranking_order_by", - threshold("max_ranking_order_by", DEFAULT_MAX_RANKING_ORDER_BY)?, - ), - ("relation_keys", relation_keys), - ("allow_suppressions", allow_suppressions), - ], - ) + threshold: impl Fn(&str) -> PyResult>, + (relation_keys, allow_suppressions): (RelationKeys, bool), +) -> PyResult { + let defaults: LintSettings = LintSettings::default(); + Ok(LintSettings { + header_rules_enabled: false, + sql: LintSqlOptions { + dialect: dialect.to_owned(), + enabled_rules: Some(codes.to_vec()), + max_literal_length: threshold(MAX_LITERAL_LENGTH)? + .unwrap_or(DEFAULT_MAX_LITERAL_LENGTH), + max_ranking_order_by: threshold(MAX_RANKING_ORDER_BY)? + .unwrap_or(DEFAULT_MAX_RANKING_ORDER_BY), + ..defaults.sql + }, + allow_suppressions, + relation_keys, + ..defaults + }) +} + +/// What early SQL lint runs with: the project, its selected SQL rule codes and lint settings. +struct EarlyLintPlan { + project_dir: PathBuf, + codes: Vec, + settings: LintSettings, } -/// The project, SQL rule codes and lint settings early SQL lint runs with, if it should run. -#[pyfunction] -#[pyo3(signature = (*, inputs, dialect))] -fn sql_lint_preparation<'py>( - inputs: &Bound<'py, PyAny>, - dialect: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = inputs.py(); - let native = |name: NativeAttr| registry::native(py, name); - let project_dir: Bound<'py, PyAny> = inputs +/// The plan early SQL lint runs with, when the project selects SQL rules without a warm cache. +fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult> { + let py: Python<'_> = inputs.py(); + let project_dir: Bound<'_, PyAny> = inputs .getattr("discovered_inputs")? .getattr("project_dir")?; if project_dir.is_none() { return Ok(None); } + let project_dir: PathBuf = project_dir.extract()?; for diagnostic in inputs.getattr("diagnostics")?.try_iter()? { if diagnostic?.getattr("is_error")?.is_truthy()? { return Ok(None); } } - let config: Bound<'py, PyAny> = call_with( - &native(NativeAttr::ValidRulesConfig)?, - &[("project_dir", project_dir.clone())], - )?; - if config.is_none() || !config.getattr("select")?.is_truthy()? { + let Ok(config) = rules_config_of(&project_dir)? else { + return Ok(None); + }; + if config.select.is_empty() { return Ok(None); } - if config.getattr("cache")?.getattr("enabled")?.is_truthy()? - && call_with( - &native(NativeAttr::LoadSqlRuleCache)?, - &[("path", sql_rule_cache_path(&project_dir)?)], - )? - .is_truthy()? - { + if config.cache.enabled && !stored_entries(&sql_rule_cache_path(&project_dir))?.is_empty() { return Ok(None); } - let codes: Vec = call_with( - &native(NativeAttr::SelectedSqlRuleCodes)?, - &[ - ( - "select", - PyList::new( - py, - config - .getattr("select")? - .try_iter()? - .collect::>>()?, - )? - .into_any(), - ), - ( - "ignore", - PyList::new( - py, - config - .getattr("ignore")? - .try_iter()? - .collect::>>()?, - )? - .into_any(), - ), - ], - )? - .extract()?; + let codes: Vec = + compiler_guard(|| Ok(selected_sql_codes(&config.select, &config.ignore)))?; if codes.is_empty() { return Ok(None); } - let mut model_rows: Vec> = Vec::new(); + let mut rows: Vec = Vec::new(); for model in inputs.getattr("model_inputs")?.try_iter()? { - model_rows.push(model?.getattr("relation_key_row")?); + rows.push(model_key_row(&model?)?); } - let mut seeds: Vec> = Vec::new(); + let mut seeds: Vec> = Vec::new(); for seed in inputs.getattr("seed_inputs")?.try_iter()? { seeds.push(seed?.getattr("schema_entry")?); } - let mut sources: Vec> = Vec::new(); + let mut sources: Vec> = Vec::new(); for source in inputs.getattr("source_inputs")?.try_iter()? { sources.push(source?.getattr("source_entry")?); } - let relation_keys: Bound<'py, PyAny> = call_with( - &native(NativeAttr::DeclaredRelationKeys)?, - &[ - ("model_rows", PyList::new(py, model_rows)?.into_any()), - ("seeds", PyList::new(py, seeds)?.into_any()), - ("sources", PyList::new(py, sources)?.into_any()), - ], + let keys: RelationKeys = relation_keys( + py, + rows, + ( + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ), )?; - let lint_config: Bound<'py, PyAny> = sql_lint_config( + let settings: LintSettings = sql_lint_settings( dialect, &codes, - &config.getattr("thresholds")?, - (relation_keys, config.getattr("allow_exceptions")?), + |key| Ok(config.thresholds.get(key).map(|value| i64::from(*value))), + (keys, config.allow_exceptions), )?; - Ok(Some(PyTuple::new( - py, - [ - project_dir, - PyTuple::new(py, codes)?.into_any(), - lint_config, - ], - )?)) + Ok(Some(EarlyLintPlan { + project_dir, + codes, + settings, + })) } -/// Each model's compile expansion for early lint, whole when the compile handed one over. -#[pyfunction] -fn early_lint_expansions<'py>(model_inputs: &Bound<'py, PyAny>) -> PyResult> { - let expansions: Bound<'py, PyDict> = PyDict::new(model_inputs.py()); - for model in model_inputs.try_iter()? { - let model: Bound<'py, PyAny> = model?; - let mut expansion: Bound<'py, PyAny> = model.getattr("lint_expansion")?; - if !expansion.is_truthy()? { - expansion = model.getattr("sql_expansion")?; - } - if !expansion.is_none() { - expansions.set_item( - model.getattr("model_file")?.getattr("file_path")?, - expansion, - )?; +/// Early SQL lint's outcome and the expansion inputs it linted with. +pub(crate) struct EarlyLintResult { + pub(crate) outcome: LintOutcome, + pub(crate) context: SqlExpansionInputs, +} + +/// One early SQL lint running on a native thread, for one project and SQL rule codes. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct PreparedSqlLint { + codes: Vec, + project_dir: PathBuf, + thread: Mutex>>>, + result: OnceLock>, + stop: Arc, +} + +impl PreparedSqlLint { + /// Whether this lint ran for `project_dir` with exactly `codes`. + pub(crate) fn matches(&self, project_dir: &Path, codes: &[String]) -> bool { + self.project_dir == project_dir && same_codes(&self.codes, codes) + } + + /// Whether this lint ran with exactly `codes`. + pub(crate) fn has_codes(&self, codes: &[String]) -> bool { + same_codes(&self.codes, codes) + } + + /// Whether the lint's result is already known. + fn finished(&self) -> bool { + self.result.get().is_some() + } + + /// The lint's result, waiting without the GIL until it finishes or a signal handler raises. + pub(crate) fn result(&self, py: Python<'_>) -> &Result { + self.result.get_or_init(|| { + let thread = self + .thread + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + let Some(thread) = thread else { + return Err(PyRuntimeError::new_err("early SQL lint was already joined")); + }; + match join_interruptibly(py, thread, || self.stop.store(true, Ordering::Release)) { + Joined::Finished(joined) => joined.unwrap_or_else(|_| { + Err(PyRuntimeError::new_err("early SQL lint thread panicked")) + }), + Joined::Interrupted(signal) => Err(signal), + } + }) + } +} + +fn same_codes(left: &[String], right: &[String]) -> bool { + left.iter().collect::>() == right.iter().collect::>() +} + +/// Start early SQL lint on a native thread when the project selects SQL rules without a cache. +fn start_early_lint( + inputs: &Bound<'_, PyAny>, + dialect: &str, + stop: &Arc, +) -> PyResult> { + let Some(plan) = early_lint_plan(inputs, dialect)? else { + return Ok(None); + }; + let mut compiled: HashMap = HashMap::new(); + for model in inputs.getattr("model_inputs")?.try_iter()? { + if let Some((path, expansion)) = model_lint_expansion(&model?)? { + compiled.insert(path, expansion); } } - Ok(expansions) + let work: EarlyLintWork = EarlyLintWork { + project_dir: plan.project_dir.clone(), + settings: plan.settings, + discovered: inputs.getattr("discovered_inputs")?.unbind(), + static_scope: inputs.getattr("declaration_scope")?.unbind(), + compiled, + stop: Arc::clone(stop), + }; + let thread: JoinHandle> = std::thread::Builder::new() + .name(LINT_THREAD.to_owned()) + .spawn(move || Python::attach(|py| early_lint(py, &work))) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + Ok(Some(PreparedSqlLint { + codes: plan.codes, + project_dir: plan.project_dir, + thread: Mutex::new(Some(thread)), + result: OnceLock::new(), + stop: Arc::clone(stop), + })) +} + +/// What one early SQL lint thread lints: the project, its settings and the compile's expansions. +struct EarlyLintWork { + project_dir: PathBuf, + settings: LintSettings, + discovered: Py, + static_scope: Py, + compiled: HashMap, + stop: Arc, +} + +/// Expand and lint the project's SQL early, stopping between files once `stop` is set. +fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult { + let project_dir: &Path = &work.project_dir; + let context: SqlExpansionInputs = lint_expansion_inputs(LintExpansionSources { + project_dir: project_dir.into_pyobject(py)?.into_any(), + value_renderer: None, + discovered_inputs: Some(work.discovered.bind(py).clone()), + declaration_scope: None, + static_declaration_scope: present(work.static_scope.bind(py).clone()), + })?; + let files: Vec = lint_sources(project_dir, None, None)?; + let outcome: LintOutcome = lint( + py, + &LintRequest { + project_dir, + settings: &work.settings, + dynamic_output_paths: &HashSet::new(), + compiled_expansions: &work.compiled, + context: Some(&context), + skip_unexpandable: false, + stop: Some(&work.stop), + }, + files, + )?; + Ok(EarlyLintResult { outcome, context }) +} + +/// The compiler's discovery and declaration scope, which lint expansion reuses for the same discovery. +pub(crate) struct ExpansionReuse { + pub(crate) discovered_inputs: Py, + pub(crate) declaration_scope: Py, } /// Own speculative SQL checks for one compilation invocation. @@ -191,34 +286,47 @@ fn early_lint_expansions<'py>(model_inputs: &Bound<'py, PyAny>) -> PyResult, - #[pyo3(get)] - expansion_reuse: Py, - stop_requested: Py, - executor: Py, + preparation: Option>, + expansion_reuse: Option, + stop_requested: Arc, } impl EarlySqlLint { /// A manager that starts speculative checks only when `enabled`. - pub(crate) fn new(py: Python<'_>, enabled: bool) -> PyResult { - let executor: Bound<'_, PyAny> = call_with( - &imported(py, FUTURES_MODULE, "ThreadPoolExecutor")?, - &[ - ("max_workers", 1.into_bound_py_any(py)?), - ( - "thread_name_prefix", - LINT_THREAD_PREFIX.into_bound_py_any(py)?, - ), - ], - )?; - Ok(Self { + pub(crate) fn new(enabled: bool) -> Self { + Self { enabled, - preparation: py.None(), - expansion_reuse: py.None(), - stop_requested: imported(py, THREADING_MODULE, "Event")?.call0()?.unbind(), - executor: executor.unbind(), - }) + preparation: None, + expansion_reuse: None, + stop_requested: Arc::new(AtomicBool::new(false)), + } + } + + /// The running early SQL lint, if one started. + pub(crate) fn preparation(&self) -> Option<&Py> { + self.preparation.as_ref() + } + + /// The compile's discovery and declaration scope, if it had one. + pub(crate) fn expansion_reuse(&self) -> Option<&ExpansionReuse> { + self.expansion_reuse.as_ref() + } + + /// Abandon speculative work, return the milliseconds spent joining it, and raise Ctrl+C caught while waiting. + pub(crate) fn stop(&self, py: Python<'_>) -> PyResult { + let started: Instant = Instant::now(); + self.stop_requested.store(true, Ordering::Release); + if let Some(preparation) = &self.preparation { + let preparation: &PreparedSqlLint = preparation.get(); + let waiting: bool = !preparation.finished(); + if let Err(error) = preparation.result(py) + && waiting + && !error.is_instance_of::(py) + { + return Err(error.clone_ref(py)); + } + } + Ok(u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)) } } @@ -226,8 +334,8 @@ impl EarlySqlLint { impl EarlySqlLint { #[new] #[pyo3(signature = (*, enabled))] - fn py_new(py: Python<'_>, enabled: bool) -> PyResult { - Self::new(py, enabled) + fn py_new(enabled: bool) -> Self { + Self::new(enabled) } fn __enter__(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { @@ -235,9 +343,8 @@ impl EarlySqlLint { } #[pyo3(signature = (*_exception))] - fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult<()> { - let _ = self.stop(py)?; - Ok(()) + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) { + let _ = self.stop(py); } /// Start early SQL lint for `inputs` when enabled and the project selects SQL rules. @@ -245,60 +352,37 @@ impl EarlySqlLint { fn start( mut slf: PyRefMut<'_, Self>, inputs: &Bound<'_, PyAny>, - dialect: &Bound<'_, PyAny>, + dialect: &str, ) -> PyResult<()> { if !slf.enabled { return Ok(()); } let py: Python<'_> = inputs.py(); - let preparation: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::PrepareSqlRules)?, - &[ - ("inputs", inputs.clone()), - ("executor", slf.executor.bind(py).clone()), - ("dialect", dialect.clone()), - ("stop", slf.stop_requested.bind(py).clone()), - ], - )?; + let stop: Arc = Arc::clone(&slf.stop_requested); + slf.preparation = start_early_lint(inputs, dialect, &stop)? + .map(|preparation| Py::new(py, preparation)) + .transpose()?; let declaration_scope: Bound<'_, PyAny> = inputs.getattr("declaration_scope")?; - let expansion_reuse: Bound<'_, PyAny> = if declaration_scope.is_none() { - declaration_scope + slf.expansion_reuse = if declaration_scope.is_none() { + None } else { - call_with( - ®istry::class(py, RecordId::SqlExpansionReuse)?, - &[ - ("discovered_inputs", inputs.getattr("discovered_inputs")?), - ("declaration_scope", declaration_scope), - ], - )? + Some(ExpansionReuse { + discovered_inputs: inputs.getattr("discovered_inputs")?.unbind(), + declaration_scope: declaration_scope.unbind(), + }) }; - slf.preparation = preparation.unbind(); - slf.expansion_reuse = expansion_reuse.unbind(); Ok(()) } /// Abandon unconsumed speculative work and return the milliseconds spent joining it. - fn stop(&self, py: Python<'_>) -> PyResult { - let started: Instant = Instant::now(); - let _ = self.stop_requested.bind(py).call_method0("set")?; - let _ = self.executor.bind(py).call_method( - "shutdown", - (), - Some(&keywords( - py, - &[ - ("wait", true.into_bound_py_any(py)?), - ("cancel_futures", true.into_bound_py_any(py)?), - ], - )?), - )?; - Ok(u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)) + #[pyo3(name = "stop")] + fn py_stop(&self, py: Python<'_>) -> PyResult { + self.stop(py) } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - module.add_function(wrap_pyfunction!(sql_lint_preparation, module)?)?; - module.add_function(wrap_pyfunction!(early_lint_expansions, module)?)?; + module.add_class::()?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 0c5cd5437c..e8e1ecdba5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -1,6 +1,6 @@ //! Compiler-verified fixed-point repair planning, entirely in memory. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::PathBuf; use pyo3::exceptions::PyException; @@ -11,6 +11,7 @@ use sqlbuild_rules::fix_verification::main::refusal::refusal; use sqlbuild_rules::fix_verification::main::with_edits::with_edits; use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -19,11 +20,20 @@ use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::fix_facts::compile_facts; use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; +use crate::bindings::_helpers::rules::lint_expansion::{ + LintExpansionSources, lint_expansion_inputs, +}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + LintFile, LintOutcome, LintRequest, LintSettings, ViolationClasses, lint, lint_sources, +}; +use crate::bindings::_helpers::rules::relation_keys::{RelationKeys, project_relation_keys}; +use crate::bindings::_helpers::rules::run_lint::given_files; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; const APPLIED_REASON: &str = "Meaning-preserving rewrite; compiler invariants verified"; const MAX_PASSES: usize = 128; +const SQL_RULE_PREFIX: &str = "SQBRSQL"; struct FixSession<'py> { project_dir: Bound<'py, PyAny>, @@ -141,42 +151,59 @@ impl<'py> FixSession<'py> { model_paths: &Bound<'py, PyAny>, ) -> PyResult> { let py = files.py(); - let relation_keys = self.relation_keys()?; - let config = replace( - &self.config, - &[ - ( - "enabled_native_rules", - PyTuple::new(py, ["SQBRSQL"])?.into_any(), - ), - ("relation_keys", relation_keys), - ], - )?; - let lint = call_with( - ®istry::native(py, NativeAttr::RunLint)?, - &[ - ("project_dir", self.project_dir.clone()), - ("config", config), - ("value_renderer", self.adapter.clone()), - ("discovered_inputs", updated_inputs(&self.inputs, files)?), - ("source_files", files.clone().into_any()), - ( - "selected_paths", - PyFrozenSet::new(py, files.keys())?.into_any(), - ), - ( - "skip_unexpandable", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], + let mut settings: LintSettings = LintSettings::from_record(&self.config)?; + settings.sql.enabled_rules = Some(vec![SQL_RULE_PREFIX.to_owned()]); + settings.relation_keys = self.relation_keys()?; + let project_dir: PathBuf = self.project_dir.extract()?; + let given: Option> = given_files(files.as_any())?; + let selected: BTreeSet = given + .iter() + .flatten() + .map(|file| file.path.clone()) + .collect(); + let sources: Vec = lint_sources(&project_dir, Some(&selected), given)?; + let context: Option = if settings.native_enabled { + Some(lint_expansion_inputs(LintExpansionSources { + project_dir: self.project_dir.clone(), + value_renderer: self.adapter.is_truthy()?.then(|| self.adapter.clone()), + discovered_inputs: Some(updated_inputs(&self.inputs, files)?), + declaration_scope: None, + static_declaration_scope: None, + })?) + } else { + None + }; + let lint: LintOutcome = lint( + py, + &LintRequest { + project_dir: &project_dir, + settings: &settings, + dynamic_output_paths: &HashSet::new(), + compiled_expansions: &HashMap::new(), + context: context.as_ref(), + skip_unexpandable: true, + stop: None, + }, + sources, )?; - self.skipped - .call_method1("update", (lint.getattr("unexpandable")?,))?; + let paths: Vec> = lint + .files + .iter() + .map(|file| Ok(file.path.clone().into_pyobject(py)?.into_any())) + .collect::>()?; + for (file, message) in &lint.unexpandable { + self.skipped.set_item(&paths[*file], message)?; + } + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; + let violations: Vec> = lint + .violations + .iter() + .map(|row| classes.violation(&paths[row.file], row)) + .collect::>()?; let edits = PyDict::new(py); let mut pending: Vec> = Vec::new(); let mut remaining: Vec> = Vec::new(); - for violation in lint.getattr("violations")?.try_iter()? { - let violation = violation?; + for violation in violations { if !violation .getattr("code")? .extract::()? @@ -353,7 +380,7 @@ impl<'py> FixSession<'py> { Ok(baseline) } - fn relation_keys(&mut self) -> PyResult> { + fn relation_keys(&mut self) -> PyResult { let py = self.inputs.py(); let baseline = match self.baseline() { Ok(value) => value, @@ -362,11 +389,11 @@ impl<'py> FixSession<'py> { .value(py) .is_instance(®istry::error(py, ErrorId::CompileInput)?)? => { - return Ok(PyDict::new(py).into_any()); + return Ok(RelationKeys::new()); } Err(error) => return Err(error), }; - registry::native(py, NativeAttr::CompiledRelationKeys)?.call1((baseline,)) + project_relation_keys(&baseline) } fn verdict(&mut self, files: &Bound<'py, PyDict>) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 6fdc2696b1..54e7413b52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -39,31 +39,6 @@ fn prepare_lint_sql( .map_err(value_error) } -/// Prepare many bodies in one detached call; `false` marks a body to prepare again individually. -#[pyfunction] -fn prepare_lint_sql_batch( - py: Python<'_>, - requests: Vec, -) -> PyResult)>> { - py.compiler_detach(|| { - Ok(requests - .iter() - .map(|request| { - sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic(|| { - sqlbuild_rules::sql_lint::main::preparation::prepare( - &request.expanded, - &request.before_expansion, - &request.prior_sites, - &request.dialect, - ) - }) - .map_or((false, None), |prepared| (true, prepared)) - }) - .collect()) - }) - .map_err(value_error) -} - #[pyfunction] fn lint_backtick_identifiers(dialect: &str) -> PyResult { compiler_guard(|| { @@ -96,7 +71,6 @@ fn format_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; - module.add_function(wrap_pyfunction!(prepare_lint_sql_batch, module)?)?; module.add_function(wrap_pyfunction!(lint_backtick_identifiers, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index 6ab29ec336..33db3aa639 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -6,10 +6,11 @@ use pyo3::prelude::{ use pyo3::types::{PyBool, PyDict}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::build_sql_expansion_context; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ + ExpansionSources, SqlExpansionInputs, present, sql_expansion_inputs, }; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; @@ -39,59 +40,75 @@ pub(crate) fn build_lint_expansion_context<'py>( ], ), )?; - expansion_context(py, &arguments).or_else(|error| Err(compile_first_error(py, error)?)) + let renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; + lint_expansion_inputs(LintExpansionSources { + project_dir: arguments.value("project_dir")?, + value_renderer: renderer.is_truthy()?.then_some(renderer), + discovered_inputs: present(arguments.value("discovered_inputs")?), + declaration_scope: present(arguments.value("declaration_scope")?), + static_declaration_scope: present(arguments.value("static_declaration_scope")?), + })? + .record(py) } -fn expansion_context<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, -) -> PyResult> { - let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; - let mut discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - if discovered.is_none() { - let no: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); - discovered = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("sql_analysis_enabled_override", no.clone()), - ("extract_output_column_locations", no), - ], - )?; - } - let mut renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; - if !renderer.is_truthy()? { - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ], - )?; - renderer = call_with( - &host::entry(py, AdapterHost::ResolveAdapter)?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - )?; - } - build_sql_expansion_context( - py, - Some(keywords( - py, - &[ - ("project_dir", project_dir), - ("discovered_inputs", discovered), - ("value_renderer", renderer), - ("declaration_scope", arguments.value("declaration_scope")?), - ( - "static_declaration_scope", - arguments.value("static_declaration_scope")?, - ), - ], - )?), - ) +/// Where lint's expansion inputs come from; a missing renderer is the project's adapter. +pub(crate) struct LintExpansionSources<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) value_renderer: Option>, + pub(crate) discovered_inputs: Option>, + pub(crate) declaration_scope: Option>, + pub(crate) static_declaration_scope: Option>, +} + +/// Lint's expansion inputs, reporting compile failures as `ProjectCompileError`. +pub(crate) fn lint_expansion_inputs( + sources: LintExpansionSources<'_>, +) -> PyResult { + let py: Python<'_> = sources.project_dir.py(); + expansion_inputs(sources).or_else(|error| Err(compile_first_error(py, error)?)) +} + +fn expansion_inputs(sources: LintExpansionSources<'_>) -> PyResult { + let py: Python<'_> = sources.project_dir.py(); + let project_dir: Bound<'_, PyAny> = sources.project_dir; + let discovered: Bound<'_, PyAny> = match sources.discovered_inputs { + Some(discovered) => discovered, + None => { + let no: Bound<'_, PyAny> = PyBool::new(py, false).to_owned().into_any(); + call_with( + ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", no.clone()), + ("extract_output_column_locations", no), + ], + )? + } + }; + let renderer: Bound<'_, PyAny> = match sources.value_renderer { + Some(renderer) => renderer, + None => { + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + )?; + call_with( + &host::entry(py, AdapterHost::ResolveAdapter)?, + &[ + ("adapter_name", adapter_name), + ("project_dir", project_dir.clone()), + ], + )? + } + }; + sql_expansion_inputs(ExpansionSources { + project_dir, + value_renderer: renderer, + cli_vars: None, + discovered_inputs: Some(discovered), + declaration_scope: sources.declaration_scope, + static_declaration_scope: sources.static_declaration_scope, + }) } /// Adapter, compile and discovery errors as `ProjectCompileError`; any other error unchanged. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs index 0d34892772..05251751e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs @@ -1,13 +1,16 @@ -//! Route lint bodies through the existing native declaration and header scanners. +//! The path-derived role of one lint input. -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; -use pyo3::types::PyDict; +use pyo3::types::PyBool; +use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +const HOOKS_FOLDER: &str = "hooks"; + /// Classify a lint path, retaining the caller's supplied relative path when available. #[pyfunction] #[pyo3(signature = (*, file_path, project_dir, relative_path=None))] @@ -17,73 +20,34 @@ pub(crate) fn lint_file_role<'py>( relative_path: Option>, ) -> PyResult> { let py = file_path.py(); - let relative = if let Some(relative) = relative_path { - relative - } else { - if !file_path - .call_method1("is_relative_to", (project_dir,))? - .is_truthy()? - { - return registry::class(py, RecordId::LintFileRole)?.call0(); + let relative: Bound<'py, PyAny> = match relative_path { + Some(relative) => relative, + None => { + if !file_path + .call_method1("is_relative_to", (project_dir,))? + .is_truthy()? + { + return registry::class(py, RecordId::LintFileRole)?.call0(); + } + file_path.call_method1("relative_to", (project_dir,))? } - file_path.call_method1("relative_to", (project_dir,))? + }; + let parts: Vec = relative.getattr("parts")?.extract()?; + let kind: Bound<'py, PyAny> = match named_declaration_role_kind(&parts) { + Some(kind) => registry::class(py, RecordId::DeclarationKind)?.call1((kind,))?, + None => py.None().into_bound(py), }; call_with( ®istry::class(py, RecordId::LintFileRole)?, &[ - ( - "declaration_kind", - call_with( - ®istry::native(py, NativeAttr::NamedDeclarationRoleKind)?, - &[("relative_path", relative.clone())], - )?, - ), - ("in_project", true.into_pyobject(py)?.to_owned().into_any()), + ("declaration_kind", kind), + ("in_project", PyBool::new(py, true).to_owned().into_any()), ( "in_hook_directory", - relative.call_method1("is_relative_to", ("hooks",))?, + PyBool::new(py, parts.first().is_some_and(|part| part == HOOKS_FOLDER)) + .to_owned() + .into_any(), ), ], ) } - -/// Use audit measurement ranges when present, otherwise the native SQL header boundaries. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn lint_body_ranges<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "lint_body_ranges", - options, - ( - &["contents", "headers", "file_path", "project_dir"], - &["role"], - ), - )?; - let mut role = args.value("role")?; - if role.is_none() { - role = lint_file_role(&args.value("file_path")?, &args.value("project_dir")?, None)?; - } - let kind = role.getattr("declaration_kind")?; - if role.getattr("in_project")?.is_truthy()? - && (kind.eq("audit")? || kind.eq("singular_audit")?) - { - let ranges = call_with( - ®istry::native(py, NativeAttr::MeasurementBodyRanges)?, - &[("contents", args.value("contents")?)], - )?; - if ranges.is_truthy()? { - return Ok(ranges); - } - } - call_with( - ®istry::native(py, NativeAttr::SqlBodyRanges)?, - &[ - ("contents", args.value("contents")?), - ("headers", args.value("headers")?), - ], - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs new file mode 100644 index 0000000000..70589cf566 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -0,0 +1,115 @@ +//! Key rows and compiled expansions lint reads natively from compile inputs and compiled projects. + +use std::collections::HashMap; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyStringMethods; + +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + ExpansionRecord, ModelInputRecord, NativeModelInput, model_relation_key_row, +}; +use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; +use crate::bindings::_helpers::rules::lint_pipeline::{CompiledExpansion, span_pass}; +use crate::bindings::_helpers::rules::relation_keys::RelationKeyRow; + +const FILE_PATH_ATTRIBUTE: &str = "file_path"; + +/// The declared key facts of a model input or compiled model. +pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + if let Ok(input) = model.downcast::() { + return input.get().read(py, model_relation_key_row); + } + if let Ok(compiled) = model.downcast::() { + return compiled.get().read(py, model_relation_key_row); + } + model.getattr("relation_key_row")?.extract() +} + +/// The model file path of a native input record. +fn record_path(record: &ModelInputRecord, py: Python<'_>) -> PyResult { + record + .model_file + .bind(py) + .getattr(FILE_PATH_ATTRIBUTE)? + .extract() +} + +/// An expansion record with the given passes. +fn expansion( + expansion: &ExpansionRecord, + passes: &[&Vec<(usize, usize, usize, usize)>], + py: Python<'_>, +) -> PyResult { + Ok(CompiledExpansion { + authored_sql: expansion.authored_sql.bind(py).to_str()?.to_owned(), + expanded_sql: expansion.expanded_sql.bind(py).to_str()?.to_owned(), + passes: passes.iter().map(|rows| span_pass(rows)).collect(), + }) +} + +/// A model input's whole compile expansion, variables included, keyed by its file path. +pub(crate) fn model_lint_expansion( + model: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = model.py(); + if let Ok(input) = model.downcast::() { + return input.get().read(py, |record, py| { + let whole: &ExpansionRecord = &record.expansion; + Ok(Some(( + record_path(record, py)?, + expansion( + whole, + &[ + &whole.interpolation_spans, + &whole.declaration_spans, + &whole.macro_spans, + ], + py, + )?, + ))) + }); + } + let mut record: Bound<'_, PyAny> = model.getattr("lint_expansion")?; + if !record.is_truthy()? { + record = model.getattr("sql_expansion")?; + } + if record.is_none() { + return Ok(None); + } + let path: PathBuf = model + .getattr("model_file")? + .getattr(FILE_PATH_ATTRIBUTE)? + .extract()?; + Ok(Some((path, CompiledExpansion::from_record(&record)?))) +} + +/// A compiled project's authored expansions: models whose interpolation left the query unchanged. +pub(crate) fn project_sql_expansions( + project: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = project.py(); + let mut expansions: HashMap = HashMap::new(); + if let Ok(view) = project.downcast::() { + let owner = view.get().owner().bind(py).borrow(); + for record in owner.inputs() { + let authored: &ExpansionRecord = &record.expansion; + if authored.authored { + expansions.insert( + record_path(record, py)?, + expansion( + authored, + &[&authored.declaration_spans, &authored.macro_spans], + py, + )?, + ); + } + } + return Ok(expansions); + } + crate::bindings::_helpers::rules::run_lint::compiled_expansion_records( + &project.getattr("sql_expansions")?, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs new file mode 100644 index 0000000000..e4a14efaad --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -0,0 +1,1107 @@ +//! The lint pipeline over native data: header rules, expanded bodies, native SQL lint and suppressions. + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_rules::dsl_headers::main::measurement_body_ranges::measurement_body_ranges; +use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers; +use sqlbuild_rules::dsl_headers::main::sql_body_ranges::sql_body_ranges; +use sqlbuild_rules::dsl_headers::models::{ + HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, +}; +use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; +use sqlbuild_rules::lint_text::main::apply_suppressions::apply_suppressions; +use sqlbuild_rules::lint_text::main::dependency_sites::dependency_sites; +use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; +use sqlbuild_rules::lint_text::main::lint_sql_bodies::lint_sql_bodies; +use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind; +use sqlbuild_rules::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; +use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; +use sqlbuild_rules::lint_text::models::{ + AuthoredDiagnostic, ExpansionSpan, InterpolationSite, LintSqlBody, LintSqlOptions, Neutralized, + RetainedViolation, SuppressibleViolation, SuppressionFinding, +}; +use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; +use sqlbuild_rules::sql_lint::main::preparation::prepare; +use sqlbuild_rules::sql_lint::types::PreparedSql; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; +use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::render::authored_sql::{AuthoredExpansion, expand_with_inputs}; +use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::rules::dsl_headers::header_violations; +use crate::bindings::_helpers::rules::native_header_parser::{ + HeaderFailure, NativeHeaderParser, external_identifiers, +}; +use crate::bindings::types::CompilerDetach; + +pub(crate) const NATIVE_ENGINE: &str = "native"; +const HEADER_ENGINE: &str = "sqlbuild"; +const HOOKS_FOLDER: &str = "hooks"; +const AUDIT_KIND: &str = "audit"; +const SINGULAR_AUDIT_KIND: &str = "singular_audit"; +const SQL_HOOK_KIND: &str = "sql_hook"; +const CEREMONIAL_KINDS: [&str; 2] = ["TEST", "SCENARIO"]; +const INTERPOLATION_MARKERS: [&str; 2] = ["@", "${"]; +const FAULT_SEVERITY: &str = "FAULT"; +const WARNING_SEVERITY: &str = "WARNING"; + +/// Resolved lint settings; the defaults are `LintConfig`'s. +#[derive(Clone, Debug)] +pub(crate) struct LintSettings { + pub(crate) native_enabled: bool, + pub(crate) header_rules_enabled: bool, + pub(crate) header_rules: HeaderRulesConfig, + pub(crate) sql: LintSqlOptions, + pub(crate) allow_suppressions: bool, + /// Declared keys of each relation, in declaration order. + pub(crate) relation_keys: Vec<(String, Vec>)>, +} + +impl Default for LintSettings { + fn default() -> Self { + Self { + native_enabled: true, + header_rules_enabled: true, + header_rules: HeaderRulesConfig { + line_width: 100, + max_description_lines: 10, + }, + sql: LintSqlOptions { + dialect: "generic".to_owned(), + enabled_rules: None, + ignored_rules: Vec::new(), + max_literal_length: 100, + max_ranking_order_by: 5, + }, + allow_suppressions: true, + relation_keys: Vec::new(), + } + } +} + +impl LintSettings { + /// The settings a `LintConfig` record holds. + pub(crate) fn from_record(config: &Bound<'_, PyAny>) -> PyResult { + let mut relation_keys: Vec<(String, Vec>)> = Vec::new(); + for item in config + .getattr("relation_keys")? + .call_method0("items")? + .try_iter()? + { + relation_keys.push(item?.extract()?); + } + Ok(Self { + native_enabled: config.getattr("native_enabled")?.is_truthy()?, + header_rules_enabled: config.getattr("header_rules_enabled")?.is_truthy()?, + header_rules: HeaderRulesConfig { + line_width: config.getattr("line_width")?.extract()?, + max_description_lines: config.getattr("max_description_lines")?.extract()?, + }, + sql: LintSqlOptions { + dialect: config.getattr("dialect")?.extract()?, + enabled_rules: config.getattr("enabled_native_rules")?.extract()?, + ignored_rules: config.getattr("ignored_native_rules")?.extract()?, + max_literal_length: config.getattr("max_literal_length")?.extract()?, + max_ranking_order_by: config.getattr("max_ranking_order_by")?.extract()?, + }, + allow_suppressions: config.getattr("allow_suppressions")?.is_truthy()?, + relation_keys, + }) + } + + /// The declared keys of `relation`, when it declares any. + fn keys_of(&self, relation: &str) -> Option<&Vec>> { + self.relation_keys + .iter() + .find(|(name, _)| name == relation) + .map(|(_, keys)| keys) + .filter(|keys| !keys.is_empty()) + } +} + +/// One authored SQL file and its text. +#[derive(Clone, Debug)] +pub(crate) struct LintFile { + pub(crate) path: PathBuf, + pub(crate) contents: String, +} + +/// A compile's authored-to-expanded SQL for one file, reused instead of expanding again. +#[derive(Clone, Debug)] +pub(crate) struct CompiledExpansion { + pub(crate) authored_sql: String, + pub(crate) expanded_sql: String, + pub(crate) passes: Vec>, +} + +impl CompiledExpansion { + /// The expansion a `CompiledSqlExpansion` record holds. + pub(crate) fn from_record(record: &Bound<'_, PyAny>) -> PyResult { + let mut passes: Vec> = Vec::new(); + for pass in record.getattr("passes")?.try_iter()? { + let mut spans: Vec = Vec::new(); + for span in pass?.try_iter()? { + let span: Bound<'_, PyAny> = span?; + spans.push(ExpansionSpan { + source_start: span.getattr("source_start")?.extract()?, + source_end: span.getattr("source_end")?.extract()?, + output_start: span.getattr("output_start")?.extract()?, + output_end: span.getattr("output_end")?.extract()?, + }); + } + passes.push(spans); + } + Ok(Self { + authored_sql: record.getattr("authored_sql")?.extract()?, + expanded_sql: record.getattr("expanded_sql")?.extract()?, + passes, + }) + } +} + +/// Expansion span rows as lint spans. +pub(crate) fn span_pass(rows: &[SpanRow]) -> Vec { + rows.iter() + .map( + |&(source_start, source_end, output_start, output_end)| ExpansionSpan { + source_start: offset(source_start), + source_end: offset(source_end), + output_start: offset(output_start), + output_end: offset(output_end), + }, + ) + .collect() +} + +fn offset(value: usize) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) +} + +/// A proposed authored-source replacement in the violation's file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct LintEditRow { + pub(crate) start: i64, + pub(crate) end: i64, + pub(crate) replacement: String, +} + +/// One lint violation against the file at index `file` of its run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct LintViolationRow { + pub(crate) file: usize, + pub(crate) line: i64, + pub(crate) column: i64, + pub(crate) code: String, + pub(crate) message: String, + pub(crate) fault: bool, + pub(crate) engine: &'static str, + pub(crate) end: Option<(i64, i64)>, + pub(crate) remediation: Option, + pub(crate) fix: Option, + pub(crate) fix_unavailable_reason: Option, +} + +impl LintViolationRow { + pub(crate) fn header(file: usize, violation: HeaderViolation) -> Self { + Self { + file, + line: offset(violation.line), + column: offset(violation.column), + code: violation.code.to_owned(), + message: violation.message, + fault: violation.severity == HeaderSeverity::Fault, + engine: HEADER_ENGINE, + end: None, + remediation: Some(violation.remediation), + fix: None, + fix_unavailable_reason: None, + } + } + + pub(crate) fn diagnostic(diagnostic: AuthoredDiagnostic) -> Self { + Self { + file: diagnostic.file, + line: diagnostic.line, + column: diagnostic.column, + code: diagnostic.code, + message: diagnostic.message, + fault: diagnostic.fault, + engine: NATIVE_ENGINE, + end: diagnostic.end, + remediation: Some(diagnostic.remediation), + fix: diagnostic.fix.map(|edit| LintEditRow { + start: edit.start, + end: edit.end, + replacement: edit.replacement, + }), + fix_unavailable_reason: diagnostic.fix_unavailable_reason, + } + } + + pub(crate) fn suppression(finding: SuppressionFinding) -> Self { + Self { + file: finding.file, + line: offset(finding.line), + column: 1, + code: SUPPRESSION_CODE.to_owned(), + message: finding.message, + fault: finding.fault, + engine: NATIVE_ENGINE, + end: None, + remediation: Some(finding.remediation.to_owned()), + fix: finding.fix.map(|(start, end)| LintEditRow { + start: offset(start), + end: offset(end), + replacement: String::new(), + }), + fix_unavailable_reason: finding.fix_unavailable_reason.map(str::to_owned), + } + } +} + +/// The lint model classes violations are projected as. +pub(crate) struct ViolationClasses<'py> { + violation: Bound<'py, PyAny>, + edit: Bound<'py, PyAny>, + severity: Bound<'py, PyAny>, +} + +impl<'py> ViolationClasses<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + violation: registry::class(py, RecordId::LintViolation)?, + edit: registry::class(py, RecordId::LintEdit)?, + severity: registry::class(py, RecordId::LintSeverity)?, + }) + } + + /// `row` as a `LintViolation` against `file_path`. + pub(crate) fn violation( + &self, + file_path: &Bound<'py, PyAny>, + row: &LintViolationRow, + ) -> PyResult> { + let py: Python<'py> = file_path.py(); + let none = || py.None().into_bound(py); + let text = |value: &Option| { + value + .as_deref() + .map_or_else(none, |text| PyString::new(py, text).into_any()) + }; + let code: Bound<'py, PyAny> = PyString::new(py, &row.code).into_any(); + let fix: Bound<'py, PyAny> = match &row.fix { + Some(edit) => call_with( + &self.edit, + &[ + ("file_path", file_path.clone()), + ("code", code.clone()), + ("start", edit.start.into_pyobject(py)?.into_any()), + ("end", edit.end.into_pyobject(py)?.into_any()), + ( + "replacement", + PyString::new(py, &edit.replacement).into_any(), + ), + ], + )?, + None => none(), + }; + let (end_line, end_column) = match row.end { + Some((line, column)) => ( + line.into_pyobject(py)?.into_any(), + column.into_pyobject(py)?.into_any(), + ), + None => (none(), none()), + }; + let severity: &str = if row.fault { + FAULT_SEVERITY + } else { + WARNING_SEVERITY + }; + call_with( + &self.violation, + &[ + ("file_path", file_path.clone()), + ("line", row.line.into_pyobject(py)?.into_any()), + ("column", row.column.into_pyobject(py)?.into_any()), + ("code", code), + ("message", PyString::new(py, &row.message).into_any()), + ("severity", self.severity.getattr(severity)?), + ("engine", PyString::new(py, row.engine).into_any()), + ("end_line", end_line), + ("end_column", end_column), + ("remediation", text(&row.remediation)), + ("fix", fix), + ("fix_unavailable_reason", text(&row.fix_unavailable_reason)), + ], + ) + } +} + +/// Every linted file, its sorted violations and the files expansion skipped. +pub(crate) struct LintOutcome { + /// The linted files in the order they were given. + pub(crate) files: Vec, + pub(crate) violations: Vec, + /// Skipped files by index, with why they could not be expanded. + pub(crate) unexpandable: Vec<(usize, String)>, +} + +impl LintOutcome { + /// This outcome as a `LintRunResult`. + pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { + let paths: Vec> = self + .files + .iter() + .map(|file| Ok(file.path.clone().into_pyobject(py)?.into_any())) + .collect::>()?; + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; + let violations: Vec> = self + .violations + .iter() + .map(|row| classes.violation(&paths[row.file], row)) + .collect::>()?; + let texts: Bound<'py, PyDict> = PyDict::new(py); + for (path, file) in paths.iter().zip(&self.files) { + texts.set_item(path, &file.contents)?; + } + let unexpandable: Bound<'py, PyDict> = PyDict::new(py); + for (file, message) in &self.unexpandable { + unexpandable.set_item(&paths[*file], message)?; + } + call_with( + ®istry::class(py, RecordId::LintRunResult)?, + &[ + ( + "files_checked", + self.files.len().into_pyobject(py)?.into_any(), + ), + ("violations", PyTuple::new(py, violations)?.into_any()), + ("formatted_files", PyTuple::empty(py).into_any()), + ("source_texts", texts.into_any()), + ("unexpandable", unexpandable.into_any()), + ], + ) + } +} + +/// What one lint run reads besides its files. +pub(crate) struct LintRequest<'a> { + pub(crate) project_dir: &'a Path, + pub(crate) settings: &'a LintSettings, + /// Resolved paths of models whose output columns a dynamic `*` proves. + pub(crate) dynamic_output_paths: &'a HashSet, + pub(crate) compiled_expansions: &'a HashMap, + /// The expansion inputs; without them bodies are not expanded or SQL-linted. + pub(crate) context: Option<&'a SqlExpansionInputs>, + pub(crate) skip_unexpandable: bool, + /// Set once the run's result is no longer needed. + pub(crate) stop: Option<&'a AtomicBool>, +} + +/// `path` resolved through the filesystem, or unchanged when it cannot be. +pub(crate) fn resolved(path: &Path) -> PathBuf { + project_real_path(path) +} + +/// `given` filtered to the selection, or the project's lint files read from disk. +pub(crate) fn lint_sources( + project_dir: &Path, + selected: Option<&BTreeSet>, + given: Option>, +) -> PyResult> { + if let Some(given) = given { + return Ok(given + .into_iter() + .filter(|file| selected.is_none_or(|paths| paths.contains(&resolved(&file.path)))) + .collect()); + } + let collected: Vec<(PathBuf, String)> = + sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files( + project_dir, + selected, + ) + .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.to_string()))?; + Ok(collected + .into_iter() + .map(|(path, contents)| LintFile { path, contents }) + .collect()) +} + +/// Raise `LintStoppedError` once `stop` is set. +fn check_stop(py: Python<'_>, stop: Option<&AtomicBool>) -> PyResult<()> { + if stop.is_some_and(|stop| stop.load(Ordering::Acquire)) { + return Err(PyErr::from_value( + registry::error(py, ErrorId::LintStopped)?.call0()?, + )); + } + Ok(()) +} + +/// Where one body's expanded SQL comes from. +enum BodySource { + /// The compile's expansion of the same authored SQL. + Compiled(String, Vec>), + /// Text without interpolation, linted as written. + Plain, + /// Text that user macros expand. + Expand, +} + +/// One authored body, neutralized and ready to expand. +struct PlannedBody { + start: usize, + input: String, + sites: Vec, + source: BodySource, +} + +/// How a file's bodies expand, shared by each of its bodies. +struct BodyTraits { + external_identifiers: Vec, + ceremonial: bool, + dynamic_star: bool, +} + +/// One file's header violations and, when expanding, its planned bodies. +struct FilePlan { + file: usize, + header_violations: Vec, + expansion: Option), HeaderFailure>>, +} + +/// Why a file could not be planned. +enum PlanFailure { + OutsideProject { path: PathBuf }, + Native(String), +} + +/// One expanded body awaiting native preparation. +struct PendingBody { + start: usize, + input: String, + expanded: String, + passes: Vec>, + sites: Vec, +} + +/// One file's expanded bodies and the expansion failure that stopped it, if any. +struct ExpandedFile { + file: usize, + traits: BodyTraits, + bodies: Vec, + failure: Option, +} + +fn path_parts(relative: &Path) -> Vec { + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect() +} + +/// Lint `files`: header rules, then expanded bodies through the native SQL engine. +pub(crate) fn lint( + py: Python<'_>, + request: &LintRequest<'_>, + files: Vec, +) -> PyResult { + let mut order: Vec = (0..files.len()).collect(); + order.sort_by(|left, right| files[*left].path.cmp(&files[*right].path)); + let expanding: bool = request.context.is_some(); + let plans: Vec> = py + .compiler_detach(|| Ok(plan_files(request, &files, &order, expanding))) + .map_err(value_error)?; + let mut header_rows: Vec = Vec::new(); + let mut expanded_files: Vec = Vec::new(); + let mut scan_failure: Option = None; + for plan in plans { + let plan: FilePlan = match plan { + Ok(plan) => plan, + Err(failure) => { + scan_failure = Some(plan_error(request.project_dir, failure)); + break; + } + }; + header_rows.extend( + plan.header_violations + .into_iter() + .map(|violation| LintViolationRow::header(plan.file, violation)), + ); + let (Some(expansion), Some(context)) = (plan.expansion, request.context) else { + continue; + }; + check_stop(py, request.stop)?; + let (traits, bodies) = match expansion { + Ok(planned) => planned, + Err(failure) => { + scan_failure = Some(failure.into_pyerr(py)); + break; + } + }; + let file: &LintFile = &files[plan.file]; + let expanded: ExpandedFile = + match expand_bodies(py, context, file, (plan.file, traits, bodies)) { + Ok(expanded) => expanded, + Err(error) if error.is_instance_of::(py) => { + scan_failure = Some(error); + break; + } + Err(error) => return Err(error), + }; + let failed: bool = expanded.failure.is_some(); + expanded_files.push(expanded); + if failed && !request.skip_unexpandable { + break; + } + } + check_stop(py, request.stop)?; + let (bodies, unexpandable) = prepared_bodies(py, request, &files, expanded_files)?; + if let Some(error) = scan_failure { + return Err(error); + } + let mut rows: Vec = header_rows; + if !bodies.is_empty() && request.settings.native_enabled { + check_stop(py, request.stop)?; + let options: &LintSqlOptions = &request.settings.sql; + let diagnostics: Vec = py + .compiler_detach(|| { + let texts: Vec<&str> = files.iter().map(|file| file.contents.as_str()).collect(); + Ok(lint_sql_bodies(&bodies, &texts, options)) + }) + .map_err(value_error)?; + rows.extend(diagnostics.into_iter().map(LintViolationRow::diagnostic)); + check_stop(py, request.stop)?; + } + let violations: Vec = + suppressed(&files, rows, request.settings.allow_suppressions)?; + Ok(LintOutcome { + files, + violations, + unexpandable, + }) +} + +/// Each file's headers, header violations and bodies in path order, up to the first failure. +fn plan_files( + request: &LintRequest<'_>, + files: &[LintFile], + order: &[usize], + expanding: bool, +) -> Vec> { + let mut plans: Vec> = Vec::new(); + for &index in order { + let plan: Result = + plan_file(request, &files[index], index, expanding); + let failed: bool = plan.is_err(); + plans.push(plan); + if failed { + break; + } + } + plans +} + +fn plan_file( + request: &LintRequest<'_>, + file: &LintFile, + index: usize, + expanding: bool, +) -> Result { + let Ok(relative) = file.path.strip_prefix(request.project_dir) else { + return Err(PlanFailure::OutsideProject { + path: file.path.clone(), + }); + }; + let parts: Vec = path_parts(relative); + let text: Vec = file.contents.chars().collect(); + let headers: Vec = scan_headers(&text, first_header_only(&parts)); + let header_violations: Vec = if request.settings.header_rules_enabled { + header_violations(&text, &headers, request.settings.header_rules) + .map_err(PlanFailure::Native)? + } else { + Vec::new() + }; + let expansion = expanding.then(|| plan_bodies(request, file, (&text, &headers), &parts)); + Ok(FilePlan { + file: index, + header_violations, + expansion, + }) +} + +/// A file's bodies, neutralized and matched against the compile's expansion. +fn plan_bodies( + request: &LintRequest<'_>, + file: &LintFile, + (text, headers): (&[char], &[HeaderSpan]), + parts: &[String], +) -> Result<(BodyTraits, Vec), HeaderFailure> { + let kind: Option<&str> = named_declaration_role_kind(parts); + let in_hook_directory: bool = parts.first().is_some_and(|part| part == HOOKS_FOLDER); + let dynamic_star: bool = !request.dynamic_output_paths.is_empty() + && request.dynamic_output_paths.contains(&resolved(&file.path)); + let compiled: Option<&CompiledExpansion> = request.compiled_expansions.get(&file.path); + let external: Vec = external_identifiers(text, headers, &NativeHeaderParser)?; + let ceremonial: bool = headers + .iter() + .any(|header| CEREMONIAL_KINDS.contains(&header.kind.as_str())); + let mut ranges: Vec<(usize, usize)> = Vec::new(); + if matches!(kind, Some(AUDIT_KIND | SINGULAR_AUDIT_KIND)) { + ranges = measurement_body_ranges(text); + } + if ranges.is_empty() { + ranges = sql_body_ranges(text, headers); + } + let neutralize: Option Neutralized> = neutralizer(&BodyRole { + kind, + in_project: true, + in_hook_directory, + }); + let bodies: Vec = ranges + .into_iter() + .map(|range| plan_body(text, range, neutralize, compiled)) + .collect(); + Ok(( + BodyTraits { + external_identifiers: external, + ceremonial, + dynamic_star, + }, + bodies, + )) +} + +/// A lint file's path classification. +pub(crate) struct BodyRole<'a> { + pub(crate) kind: Option<&'a str>, + pub(crate) in_project: bool, + pub(crate) in_hook_directory: bool, +} + +/// How a body's interpolation is neutralized before expansion: audit parameters, or hook context. +fn neutralizer(role: &BodyRole<'_>) -> Option Neutralized> { + if role.in_project && role.kind == Some(AUDIT_KIND) { + Some(neutralize_audit_parameters) + } else if role.in_hook_directory || (role.in_project && role.kind == Some(SQL_HOOK_KIND)) { + Some(neutralize_context_interpolation) + } else { + None + } +} + +/// One authored body expanded and prepared for native SQL lint, outside a lint run. +pub(crate) struct SingleBody { + pub(crate) body: LintSqlBody, + pub(crate) body_end: usize, + pub(crate) dependency_relations: Vec<(String, String)>, +} + +/// One body to expand and prepare outside a lint run. +pub(crate) struct SingleBodyRequest<'a, 'py> { + pub(crate) context: &'a SqlExpansionInputs, + pub(crate) file_path: &'a Bound<'py, PyAny>, + pub(crate) contents: &'a str, + pub(crate) range: (usize, usize), + pub(crate) role: BodyRole<'a>, + pub(crate) dialect: &'a str, +} + +/// Expand and prepare the body at `range` of `contents` as `role` reads it. +pub(crate) fn prepare_single_body(request: &SingleBodyRequest<'_, '_>) -> PyResult { + let SingleBodyRequest { + context, + file_path, + contents, + range, + role, + dialect, + } = request; + let (context, file_path, range, dialect): ( + &SqlExpansionInputs, + &Bound<'_, PyAny>, + (usize, usize), + &str, + ) = (context, file_path, *range, dialect); + let py: Python<'_> = file_path.py(); + let text: Vec = contents.chars().collect(); + let planned: PlannedBody = plan_body(&text, range, neutralizer(role), None); + let (expanded, passes) = match planned.source { + BodySource::Expand => { + let expansion: AuthoredExpansion = + expand_with_inputs(context, &planned.input, file_path) + .map_err(|error| expansion_error(file_path, error))?; + ( + expansion.sql, + expansion + .passes + .iter() + .map(|rows| span_pass(rows)) + .collect(), + ) + } + BodySource::Compiled(expanded, passes) => (expanded, passes), + BodySource::Plain => (planned.input.clone(), Vec::new()), + }; + let pending: PendingBody = PendingBody { + start: planned.start, + input: planned.input, + expanded, + passes, + sites: planned.sites, + }; + let prior: Vec = prior_sites(&pending.sites); + let prepared: Option = py + .compiler_detach(|| prepare(&pending.expanded, &pending.input, &prior, dialect)) + .map_err(value_error)?; + let Some(prepared) = prepared else { + return Err(not_prepared(py, &file_path.extract::()?)?); + }; + let traits: BodyTraits = BodyTraits { + external_identifiers: Vec::new(), + ceremonial: false, + dynamic_star: false, + }; + let (body, dependency_relations) = lint_body((0, &traits), pending, prepared)?; + Ok(SingleBody { + body, + body_end: range.1, + dependency_relations, + }) +} + +/// The `NativeLintError` of a body native preparation declined. +fn not_prepared(py: Python<'_>, path: &Path) -> PyResult { + Ok(PyErr::from_value( + registry::error(py, ErrorId::NativeLint)?.call1((format!( + "{} could not be prepared for native SQL lint", + path.display() + ),))?, + )) +} + +fn plan_body( + text: &[char], + (mut start, end): (usize, usize), + neutralize: Option Neutralized>, + compiled: Option<&CompiledExpansion>, +) -> PlannedBody { + let body: String = text[start..end].iter().collect(); + let (mut input, sites): (String, Vec) = match neutralize { + Some(neutralize) => { + let neutralized: Neutralized = neutralize(&body); + (neutralized.text, neutralized.sites) + } + None => (body.clone(), Vec::new()), + }; + let source: BodySource = match compiled { + Some(compiled) if python_strip(&body) == compiled.authored_sql => { + let leading: usize = + body.chars().count() - body.trim_start_matches(python_isspace).chars().count(); + start += leading; + input.clone_from(&compiled.authored_sql); + BodySource::Compiled(compiled.expanded_sql.clone(), compiled.passes.clone()) + } + _ if INTERPOLATION_MARKERS + .iter() + .all(|marker| !input.contains(marker)) => + { + BodySource::Plain + } + _ => BodySource::Expand, + }; + PlannedBody { + start, + input, + sites, + source, + } +} + +/// The `ValueError` `PurePath.relative_to` raises for a path outside `project_dir`. +pub(crate) fn outside_project(path: &Path, project_dir: &Path) -> PyErr { + PyValueError::new_err(format!( + "{} is not in the subpath of {}", + python_str_repr(&path.to_string_lossy()), + python_str_repr(&project_dir.to_string_lossy()), + )) +} + +fn plan_error(project_dir: &Path, failure: PlanFailure) -> PyErr { + match failure { + PlanFailure::OutsideProject { path } => outside_project(&path, project_dir), + PlanFailure::Native(message) => value_error(message), + } +} + +/// Expand a file's bodies in order; a compile failure stops the file with its partial work. +fn expand_bodies( + py: Python<'_>, + context: &SqlExpansionInputs, + file: &LintFile, + (index, traits, planned): (usize, BodyTraits, Vec), +) -> PyResult { + let mut bodies: Vec = Vec::with_capacity(planned.len()); + let mut failure: Option = None; + let mut path: Option> = None; + for body in planned { + let (expanded, passes) = match body.source { + BodySource::Compiled(expanded, passes) => (expanded, passes), + BodySource::Plain => (body.input.clone(), Vec::new()), + BodySource::Expand => { + let file_path: &Bound<'_, PyAny> = match &path { + Some(path) => path, + None => path.insert(file.path.clone().into_pyobject(py)?.into_any()), + }; + match expand_with_inputs(context, &body.input, file_path) { + Ok(AuthoredExpansion { sql, passes }) => { + (sql, passes.iter().map(|rows| span_pass(rows)).collect()) + } + Err(error) => { + let error: PyErr = expansion_error(file_path, error); + if !error + .value(py) + .is_instance(®istry::error(py, ErrorId::ProjectCompile)?)? + { + return Err(error); + } + failure = Some(error); + break; + } + } + } + }; + bodies.push(PendingBody { + start: body.start, + input: body.input, + expanded, + passes, + sites: body.sites, + }); + } + Ok(ExpandedFile { + file: index, + traits, + bodies, + failure, + }) +} + +/// A compile-input error while expanding as `ProjectCompileError`; any other error unchanged. +fn expansion_error(file_path: &Bound<'_, PyAny>, error: PyErr) -> PyErr { + let py: Python<'_> = file_path.py(); + let wrapped = (|| -> PyResult> { + if !error + .value(py) + .is_instance(®istry::error(py, ErrorId::CompileInput)?)? + { + return Ok(None); + } + let message: String = format!( + "{} could not be expanded, so its SQL cannot be linted: {}", + file_path.str()?, + error.value(py).str()? + ); + Ok(Some(PyErr::from_value( + registry::error(py, ErrorId::ProjectCompile)?.call1((message,))?, + ))) + })(); + match wrapped { + Ok(Some(wrapped)) => { + wrapped.set_cause(py, Some(error)); + wrapped + } + Ok(None) => error, + Err(wrapping_error) => wrapping_error, + } +} + +/// Prepared bodies to lint, and skipped files by index with why they could not expand. +type PreparedBodies = (Vec, Vec<(usize, String)>); + +/// Prepare every expanded body natively, then publish bodies or failures in file order. +fn prepared_bodies( + py: Python<'_>, + request: &LintRequest<'_>, + files: &[LintFile], + expanded_files: Vec, +) -> PyResult { + let dialect: &str = &request.settings.sql.dialect; + let prepared: Vec, String>> = py + .compiler_detach(|| { + Ok(expanded_files + .iter() + .flat_map(|file| &file.bodies) + .map(|body| { + let prior: Vec = prior_sites(&body.sites); + catch_compiler_panic(|| prepare(&body.expanded, &body.input, &prior, dialect)) + }) + .collect()) + }) + .map_err(value_error)?; + let mut prepared: Vec> = prepared + .into_iter() + .collect::>() + .map_err(value_error)?; + let mut bodies: Vec = Vec::new(); + let mut unexpandable: Vec<(usize, String)> = Vec::new(); + let mut position: usize = 0; + for file in expanded_files { + let mut file_bodies: Vec = Vec::with_capacity(file.bodies.len()); + for body in file.bodies { + let Some(result) = prepared[position].take() else { + return Err(not_prepared(py, &files[file.file].path)?); + }; + position += 1; + let (mut lint_body, relations) = lint_body((file.file, &file.traits), body, result)?; + lint_body.relation_keys = relation_keys(request.settings, relations); + file_bodies.push(lint_body); + } + match file.failure { + None => bodies.extend(file_bodies), + Some(failure) if request.skip_unexpandable => { + let message: String = failure.value(py).str()?.extract()?; + let prefix: String = format!("{} ", files[file.file].path.display()); + unexpandable.push(( + file.file, + message.strip_prefix(&prefix).unwrap_or(&message).to_owned(), + )); + } + Some(failure) => return Err(failure), + } + } + Ok((bodies, unexpandable)) +} + +/// One prepared body as the native SQL engine lints it. +fn lint_body( + (file, traits): (usize, &BodyTraits), + body: PendingBody, + (lint_text, prepared_sites, externally_referenced_ctes): PreparedSql, +) -> PyResult<(LintSqlBody, Vec<(String, String)>)> { + let prepared_sites: Vec = prepared_sites + .into_iter() + .map( + |( + sentinel, + neutralized_start, + neutralized_end, + original_start, + original_end, + original_text, + )| { + InterpolationSite { + sentinel, + neutralized_start, + neutralized_end, + original_start, + original_end, + original_text, + } + }, + ) + .collect(); + let dependencies = dependency_sites(&prepared_sites).map_err(value_error)?; + let mut passes: Vec> = Vec::with_capacity(body.passes.len() + 2); + passes.push(sentinel_spans(&body.sites)); + passes.extend(body.passes); + passes.push(sentinel_spans(&prepared_sites)); + let body = LintSqlBody { + file, + body_start: offset(body.start), + lint_text, + passes, + external_identifiers: traits.external_identifiers.clone(), + dependency_identifiers: dependencies.identifiers, + externally_referenced_ctes, + allows_ceremonial_select: traits.ceremonial, + allows_dynamic_output_star: traits.dynamic_star, + allows_empty_fixture_star: traits.ceremonial, + relation_keys: Vec::new(), + }; + Ok((body, dependencies.relations)) +} + +/// The declared keys of each dependency relation that declares any, by its sentinel. +pub(crate) fn relation_keys( + settings: &LintSettings, + relations: Vec<(String, String)>, +) -> Vec<(String, Vec>)> { + let mut keyed: Vec<(String, Vec>)> = Vec::new(); + for (sentinel, name) in relations { + if let Some(keys) = settings.keys_of(&name) { + keyed.push((sentinel, keys.clone())); + } + } + keyed +} + +/// Where each pre-expansion site's sentinel starts in the neutralized body. +fn prior_sites(sites: &[InterpolationSite]) -> Vec { + sites.iter().map(|site| site.neutralized_start).collect() +} + +fn sentinel_spans(sites: &[InterpolationSite]) -> Vec { + sites + .iter() + .map(|site| ExpansionSpan { + source_start: offset(site.original_start), + source_end: offset(site.original_end), + output_start: offset(site.neutralized_start), + output_end: offset(site.neutralized_end), + }) + .collect() +} + +/// `rows` without suppressed violations, with suppression findings, sorted by path then place. +fn suppressed( + files: &[LintFile], + rows: Vec, + allow_suppressions: bool, +) -> PyResult> { + let keys: Vec = rows + .iter() + .map(|row| SuppressibleViolation { + file: row.file, + line: usize::try_from(row.line).unwrap_or(usize::MAX), + code: row.code.clone(), + native: row.engine == NATIVE_ENGINE, + }) + .collect(); + let texts: Vec<&str> = files.iter().map(|file| file.contents.as_str()).collect(); + let retained: Vec = catch_compiler_panic(|| { + apply_suppressions(&texts, &keys, allow_suppressions).map_err(|error| error.to_string()) + }) + .map_err(value_error)?; + let mut violations: Vec = retained + .into_iter() + .map(|item| match item { + RetainedViolation::Existing(index) => rows[index].clone(), + RetainedViolation::Added(finding) => LintViolationRow::suppression(finding), + }) + .collect(); + violations.sort_by(|left, right| { + ( + files[left.file].path.to_string_lossy(), + left.line, + left.column, + ) + .cmp(&( + files[right.file].path.to_string_lossy(), + right.line, + right.column, + )) + }); + Ok(violations) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs index d571ba353b..10597b7345 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -1,42 +1,58 @@ -//! Native lint expansion orchestration around the existing user-macro boundary. +//! One authored body expanded and prepared for native SQL lint, as a `LintBody`. -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use sqlbuild_rules::lint_text::main::dependency_sites::dependency_sites; -use sqlbuild_rules::lint_text::models::InterpolationSite; +use pyo3::types::{PyBool, PyDict, PyTuple}; +use sqlbuild_rules::lint_text::models::{ExpansionSpan, LintSqlBody}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::rules::lint_headers::{lint_body_ranges, lint_file_role}; -use crate::bindings::_helpers::rules::lint_stop::check_stop; +use crate::bindings::_helpers::rules::lint_pipeline::{ + BodyRole, SingleBody, SingleBodyRequest, prepare_single_body, +}; -/// Expand a single body and finish its native SQL preparation. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn prepare_lint_body<'py>( +/// `ExpansionSpan` objects of one pass. +fn span_objects<'py>( + class: &Bound<'py, PyAny>, + spans: &[ExpansionSpan], +) -> PyResult> { + let mut objects: Vec> = Vec::with_capacity(spans.len()); + for span in spans { + objects.push(class.call1(( + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ))?); + } + PyTuple::new(class.py(), objects) +} + +/// `ExpansionSpan` tuples for each pass. +fn pass_objects<'py>( py: Python<'py>, - options: Option>, -) -> PyResult> { - let pending = expand_lint_body(py, options)?; - let prepared = - registry::native(py, NativeAttr::PrepareLintSql)?.call1((pending.getattr("request")?,))?; - finish_body(&pending, &prepared) + passes: &[Vec], +) -> PyResult> { + let class: Bound<'py, PyAny> = registry::class(py, RecordId::ExpansionSpan)?; + let passes: Vec> = passes + .iter() + .map(|spans| span_objects(&class, spans)) + .collect::>()?; + PyTuple::new(py, passes) } -/// Expand authored SQL while retaining source offsets and pre-expansion substitutions. +/// Expand a single body and finish its native SQL preparation. #[pyfunction] #[pyo3(signature = (**options))] -pub(crate) fn expand_lint_body<'py>( +pub(crate) fn prepare_lint_body<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - let args = KeywordArguments::read( + let arguments: KeywordArguments<'py> = KeywordArguments::read( py, - "expand_lint_body", + "prepare_lint_body", options, ( &[ @@ -47,451 +63,67 @@ pub(crate) fn expand_lint_body<'py>( "context", "dialect", ], - &[ - "external_identifiers", - "allows_ceremonial_select", - "allows_dynamic_output_star", - "compiled_expansion", - ], + &[], ), )?; - let (mut start, mut end): (usize, usize) = args.value("body_range")?.extract()?; - let body = args.value("contents")?.get_item(pyo3::types::PySlice::new( - py, - start as isize, - end as isize, - 1, - ))?; - let mut input = body.clone(); - let mut sites = PyTuple::empty(py).into_any(); - let role = args.value("role")?; - let kind = role.getattr("declaration_kind")?; - let in_project = role.getattr("in_project")?.is_truthy()?; - let neutralizer = if in_project && kind.eq("audit")? { - Some(NativeAttr::NeutralizeLintAuditParameters) - } else if role.getattr("in_hook_directory")?.is_truthy()? - || (in_project && kind.eq("sql_hook")?) - { - Some(NativeAttr::NeutralizeLintContextInterpolation) - } else { - None - }; - if let Some(neutralizer) = neutralizer { - let neutralized = registry::native(py, neutralizer)?.call1((&body,))?; - input = neutralized.get_item(0)?; - sites = site_objects(&neutralized.get_item(1)?)?; - } - let compiled = args.value("compiled_expansion")?; - let expanded; - let passes; - if !compiled.is_none() - && body - .call_method0("strip")? - .eq(compiled.getattr("authored_sql")?)? - { - start += body.len()? - body.call_method0("lstrip")?.len()?; - input = compiled.getattr("authored_sql")?; - end = start + input.len()?; - expanded = compiled.getattr("expanded_sql")?; - passes = compiled.getattr("passes")?; - } else if !input.extract::()?.contains('@') - && !input.extract::()?.contains("${") - { - expanded = input.clone(); - passes = PyTuple::empty(py).into_any(); - } else { - let value = call_with( - ®istry::native(py, NativeAttr::ExpandSqlWithSpans)?, - &[ - ("sql", input.clone()), - ("file_path", args.value("file_path")?), - ("context", args.value("context")?), - ], - ) - .map_err(|error| expansion_error(py, &args, error))?; - expanded = value.get_item(0)?; - passes = value.get_item(1)?; - } - let mut prior_sites: Vec> = Vec::new(); - for site in sites.try_iter()? { - prior_sites.push(site?.getattr("neutralized_start")?); - } - let request = keywords( - py, - &[ - ("expanded", expanded.clone()), - ("before_expansion", input.clone()), - ("prior_sites", PyList::new(py, prior_sites)?.into_any()), - ("dialect", args.value("dialect")?), - ], - )?; - let external = if args.passed("external_identifiers")? { - args.value("external_identifiers")? - } else { - PyTuple::empty(py).into_any() - }; + let role: Bound<'py, PyAny> = arguments.value("role")?; + let file_path: Bound<'py, PyAny> = arguments.value("file_path")?; + let contents: String = arguments.value("contents")?.extract()?; + let dialect: String = arguments.value("dialect")?.extract()?; + let kind: Option = role.getattr("declaration_kind")?.extract()?; + let prepared: SingleBody = prepare_single_body(&SingleBodyRequest { + context: &SqlExpansionInputs::from_record(&arguments.value("context")?)?, + file_path: &file_path, + contents: &contents, + range: arguments.value("body_range")?.extract()?, + role: BodyRole { + kind: kind.as_deref(), + in_project: role.getattr("in_project")?.is_truthy()?, + in_hook_directory: role.getattr("in_hook_directory")?.is_truthy()?, + }, + dialect: &dialect, + })?; + let body: &LintSqlBody = &prepared.body; + let flag = |value: bool| PyBool::new(py, value).to_owned().into_any(); call_with( - ®istry::class(py, RecordId::PendingLintBody)?, + ®istry::class(py, RecordId::LintBody)?, &[ - ("file_path", args.value("file_path")?), - ("body_start", start.into_pyobject(py)?.into_any()), - ("body_end", end.into_pyobject(py)?.into_any()), - ("expansion_input", input), - ("expanded", expanded), - ("expansion_passes", passes), - ("pre_expansion_sites", sites), - ("external_identifiers", external), + ("file_path", file_path.clone()), + ("body_start", body.body_start.into_pyobject(py)?.into_any()), + ("body_end", prepared.body_end.into_pyobject(py)?.into_any()), ( - "allows_ceremonial_select", - args.value_or_false("allows_ceremonial_select")?, + "lint_text", + body.lint_text.clone().into_pyobject(py)?.into_any(), ), + ("passes", pass_objects(py, &body.passes)?.into_any()), ( - "allows_dynamic_output_star", - args.value_or_false("allows_dynamic_output_star")?, - ), - ( - "allows_empty_fixture_star", - false.into_pyobject(py)?.to_owned().into_any(), + "external_identifiers", + PyTuple::new(py, &body.external_identifiers)?.into_any(), ), - ("request", request.into_any()), - ], - ) -} - -fn expansion_error(py: Python<'_>, args: &KeywordArguments<'_>, error: PyErr) -> PyErr { - let wrapped = (|| -> PyResult> { - if !error - .value(py) - .is_instance(®istry::error(py, ErrorId::CompileInput)?)? - { - return Ok(None); - } - let message = format!( - "{} could not be expanded, so its SQL cannot be linted: {}", - args.value("file_path")?.str()?, - error.value(py).str()? - ); - Ok(Some(PyErr::from_value( - registry::error(py, ErrorId::ProjectCompile)?.call1((message,))?, - ))) - })(); - match wrapped { - Ok(Some(wrapped)) => { - wrapped.set_cause(py, Some(error)); - wrapped - } - Ok(None) => error, - Err(wrapping_error) => wrapping_error, - } -} - -fn site_objects<'py>(sites: &Bound<'py, PyAny>) -> PyResult> { - let class = registry::class(sites.py(), RecordId::InterpolationSite)?; - let values: Vec> = sites - .try_iter()? - .map(|site| class.call1(site?.cast_into::()?)) - .collect::>()?; - Ok(PyTuple::new(sites.py(), values)?.into_any()) -} - -fn sentinel_spans<'py>(sites: &Bound<'py, PyAny>) -> PyResult> { - let py = sites.py(); - let mut spans: Vec> = Vec::new(); - for site in sites.try_iter()? { - let site = site?; - spans.push(call_with( - ®istry::class(py, RecordId::ExpansionSpan)?, - &[ - ("source_start", site.getattr("original_start")?), - ("source_end", site.getattr("original_end")?), - ("output_start", site.getattr("neutralized_start")?), - ("output_end", site.getattr("neutralized_end")?), - ], - )?); - } - Ok(PyTuple::new(py, spans)?.into_any()) -} - -fn finish_body<'py>( - pending: &Bound<'py, PyAny>, - prepared: &Bound<'py, PyAny>, -) -> PyResult> { - let py = pending.py(); - if prepared.is_none() { - return Err(PyErr::from_value( - registry::error(py, ErrorId::NativeLint)?.call1((format!( - "{} could not be prepared for native SQL lint", - pending.getattr("file_path")?.str()? - ),))?, - )); - } - let sites = site_objects(&prepared.get_item(1)?)?; - let native_sites: Vec = sites - .try_iter()? - .map(|site| { - let site = site?; - Ok(InterpolationSite { - sentinel: site.getattr("sentinel")?.extract()?, - original_text: site.getattr("original_text")?.extract()?, - original_start: site.getattr("original_start")?.extract()?, - original_end: site.getattr("original_end")?.extract()?, - neutralized_start: site.getattr("neutralized_start")?.extract()?, - neutralized_end: site.getattr("neutralized_end")?.extract()?, - }) - }) - .collect::>()?; - let dependencies = dependency_sites(&native_sites) - .map_err(crate::bindings::_helpers::boundary::panics::value_error)?; - let mut passes: Vec> = - vec![sentinel_spans(&pending.getattr("pre_expansion_sites")?)?]; - passes.extend( - pending - .getattr("expansion_passes")? - .try_iter()? - .collect::>>()?, - ); - passes.push(sentinel_spans(&sites)?); - let values = keywords( - py, - &[ - ("lint_text", prepared.get_item(0)?), - ("passes", PyTuple::new(py, passes)?.into_any()), ( "dependency_identifiers", - PyTuple::new(py, dependencies.identifiers)?.into_any(), + PyTuple::new(py, &body.dependency_identifiers)?.into_any(), ), ( "externally_referenced_ctes", - PyTuple::new(py, prepared.get_item(2)?.extract::>()?)?.into_any(), + PyTuple::new(py, &body.externally_referenced_ctes)?.into_any(), ), ( "dependency_relations", - PyTuple::new(py, dependencies.relations)?.into_any(), + PyTuple::new(py, &prepared.dependency_relations)?.into_any(), + ), + ( + "allows_ceremonial_select", + flag(body.allows_ceremonial_select), + ), + ( + "allows_dynamic_output_star", + flag(body.allows_dynamic_output_star), + ), + ( + "allows_empty_fixture_star", + flag(body.allows_empty_fixture_star), ), - ], - )?; - for name in [ - "file_path", - "body_start", - "body_end", - "external_identifiers", - "allows_ceremonial_select", - "allows_dynamic_output_star", - "allows_empty_fixture_star", - ] { - values.set_item(name, pending.getattr(name)?)?; - } - registry::class(py, RecordId::LintBody)?.call((), Some(&values)) -} - -fn prepare_batch<'py>( - py: Python<'py>, - pending: &[Bound<'py, PyAny>], -) -> PyResult>> { - if pending.is_empty() { - return Ok(Vec::new()); - } - let requests: Vec> = pending - .iter() - .map(|body| body.getattr("request")) - .collect::>()?; - let results = registry::native(py, NativeAttr::PrepareLintSqlBatch)? - .call1((PyList::new(py, requests)?,))?; - let mut prepared: Vec> = Vec::new(); - for (body, result) in pending.iter().zip(results.try_iter()?) { - let result = result?; - prepared.push(if result.get_item(0)?.is_truthy()? { - result.get_item(1)? - } else { - registry::native(py, NativeAttr::PrepareLintSql)?.call1((body.getattr("request")?,))? - }); - } - if prepared.len() != pending.len() || results.len()? != pending.len() { - return Err(pyo3::exceptions::PyValueError::new_err( - "native preparation batch length mismatch", - )); - } - Ok(prepared) -} - -/// Expand each file body in order, preserving the first expansion failure with its partial work. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn expand_file_bodies<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "expand_file_bodies", - options, - ( - &[ - "file_path", - "contents", - "headers", - "context", - "dialect", - "project_dir", - "relative_path", - "dynamic_output_paths", - "compiled_expansions", - ], - &[], - ), - )?; - check_stop(py)?; - let path = args.value("file_path")?; - let role = lint_file_role( - &path, - &args.value("project_dir")?, - Some(args.value("relative_path")?), - )?; - let dynamic = args.value("dynamic_output_paths")?; - let dynamic_star = dynamic.is_truthy()? && dynamic.contains(path.call_method0("resolve")?)?; - let expansions = args.value("compiled_expansions")?; - let compiled = if expansions.is_truthy()? { - expansions.call_method1("get", (&path,))? - } else { - py.None().into_bound(py) - }; - let external = call_with( - ®istry::native(py, NativeAttr::ExternalIdentifiersForHeaders)?, - &[ - ("contents", args.value("contents")?), - ("headers", args.value("headers")?), - ], - )?; - let mut ceremonial = false; - for header in args.value("headers")?.try_iter()? { - let kind = header?.getattr("kind")?; - if kind.eq("TEST")? || kind.eq("SCENARIO")? { - ceremonial = true; - break; - } - } - let mut bodies: Vec> = Vec::new(); - let mut failure = py.None().into_bound(py); - let expansion = (|| -> PyResult<()> { - let ranges = lint_body_ranges( - py, - Some(keywords( - py, - &[ - ("contents", args.value("contents")?), - ("headers", args.value("headers")?), - ("file_path", path.clone()), - ("project_dir", args.value("project_dir")?), - ("role", role.clone()), - ], - )?), - )?; - for range in ranges.try_iter()? { - let pending = expand_lint_body( - py, - Some(keywords( - py, - &[ - ("role", role.clone()), - ("file_path", path.clone()), - ("contents", args.value("contents")?), - ("body_range", range?), - ("context", args.value("context")?), - ("dialect", args.value("dialect")?), - ("external_identifiers", external.clone()), - ( - "allows_ceremonial_select", - ceremonial.into_pyobject(py)?.to_owned().into_any(), - ), - ( - "allows_dynamic_output_star", - dynamic_star.into_pyobject(py)?.to_owned().into_any(), - ), - ("compiled_expansion", compiled.clone()), - ], - )?), - )?; - let changes = keywords( - py, - &[( - "allows_empty_fixture_star", - ceremonial.into_pyobject(py)?.to_owned().into_any(), - )], - )?; - bodies.push( - py.import("dataclasses")? - .getattr("replace")? - .call((pending,), Some(&changes))?, - ); - } - Ok(()) - })(); - if let Err(error) = expansion { - if !error - .value(py) - .is_instance(®istry::error(py, ErrorId::ProjectCompile)?)? - { - return Err(error); - } - failure = error.value(py).clone().into_any(); - } - call_with( - ®istry::class(py, RecordId::ExpandedLintFile)?, - &[ - ("file_path", path), - ("bodies", PyTuple::new(py, bodies)?.into_any()), - ("failure", failure), - ], - ) -} - -/// Batch SQL preparation, then publish bodies or failures in the original file order. -#[pyfunction] -#[pyo3(signature = (*, expanded_files, dialect, skip_unexpandable))] -pub(crate) fn prepare_expanded_files<'py>( - expanded_files: &Bound<'py, PyAny>, - dialect: &str, - skip_unexpandable: bool, -) -> PyResult> { - let _ = dialect; - let py = expanded_files.py(); - check_stop(py)?; - let files: Vec> = expanded_files.try_iter()?.collect::>()?; - let mut pending: Vec> = Vec::new(); - for file in &files { - pending.extend( - file.getattr("bodies")? - .try_iter()? - .collect::>>()?, - ); - } - let prepared = prepare_batch(py, &pending)?; - let mut bodies: Vec> = Vec::new(); - let unexpandable = PyDict::new(py); - let mut position = 0; - for file in files { - let mut file_bodies: Vec> = Vec::new(); - for body in file.getattr("bodies")?.try_iter()? { - file_bodies.push(finish_body(&body?, &prepared[position])?); - position += 1; - } - let failure = file.getattr("failure")?; - if failure.is_none() { - bodies.extend(file_bodies); - } else if skip_unexpandable { - let path = file.getattr("file_path")?; - let message: String = failure.str()?.extract()?; - let prefix = format!("{} ", path.str()?); - unexpandable.set_item(path, message.strip_prefix(&prefix).unwrap_or(&message))?; - } else { - return Err(PyErr::from_value(failure)); - } - } - call_with( - ®istry::class(py, RecordId::PreparedLintFiles)?, - &[ - ("bodies", PyTuple::new(py, bodies)?.into_any()), - ("unexpandable", unexpandable.into_any()), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs index 3020f23ec0..5e8c656119 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs @@ -1,21 +1,17 @@ //! Native SQL lint of prepared lint bodies, as Python lint violations. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::main::lint_sql_bodies::lint_sql_bodies; use sqlbuild_rules::lint_text::models::{ AuthoredDiagnostic, ExpansionSpan, LintSqlBody, LintSqlOptions, }; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; -use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; use crate::bindings::types::CompilerDetach; -const NATIVE_ENGINE: &str = "native"; - fn spans(passes: &Bound<'_, PyAny>) -> PyResult>> { passes.try_iter()?.map(|pass| pass_spans(&pass?)).collect() } @@ -84,82 +80,6 @@ fn options(config: &Bound<'_, PyAny>) -> PyResult { }) } -/// The lint model classes native SQL diagnostics are built as. -struct ViolationClasses<'py> { - violation: Bound<'py, PyAny>, - edit: Bound<'py, PyAny>, - severity: Bound<'py, PyAny>, -} - -impl<'py> ViolationClasses<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - violation: registry::class(py, RecordId::LintViolation)?, - edit: registry::class(py, RecordId::LintEdit)?, - severity: registry::class(py, RecordId::LintSeverity)?, - }) - } - - fn violation( - &self, - file_path: &Bound<'py, PyAny>, - diagnostic: AuthoredDiagnostic, - ) -> PyResult> { - let py: Python<'py> = file_path.py(); - let none = || py.None().into_bound(py); - let code: Bound<'py, PyAny> = PyString::new(py, &diagnostic.code).into_any(); - let fix: Bound<'py, PyAny> = match diagnostic.fix { - Some(edit) => call_with( - &self.edit, - &[ - ("file_path", file_path.clone()), - ("code", code.clone()), - ("start", edit.start.into_pyobject(py)?.into_any()), - ("end", edit.end.into_pyobject(py)?.into_any()), - ( - "replacement", - PyString::new(py, &edit.replacement).into_any(), - ), - ], - )?, - None => none(), - }; - let (end_line, end_column) = match diagnostic.end { - Some((line, column)) => ( - line.into_pyobject(py)?.into_any(), - column.into_pyobject(py)?.into_any(), - ), - None => (none(), none()), - }; - let severity: &str = if diagnostic.fault { "FAULT" } else { "WARNING" }; - call_with( - &self.violation, - &[ - ("file_path", file_path.clone()), - ("line", diagnostic.line.into_pyobject(py)?.into_any()), - ("column", diagnostic.column.into_pyobject(py)?.into_any()), - ("code", code), - ("message", PyString::new(py, &diagnostic.message).into_any()), - ("severity", self.severity.getattr(severity)?), - ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), - ("end_line", end_line), - ("end_column", end_column), - ( - "remediation", - PyString::new(py, &diagnostic.remediation).into_any(), - ), - ("fix", fix), - ( - "fix_unavailable_reason", - diagnostic - .fix_unavailable_reason - .map_or_else(none, |reason| PyString::new(py, &reason).into_any()), - ), - ], - ) - } -} - /// Lint expanded bodies natively and map every diagnostic onto authored source. #[pyfunction] fn lint_native_sql_bodies<'py>( @@ -191,7 +111,10 @@ fn lint_native_sql_bodies<'py>( let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; diagnostics .into_iter() - .map(|diagnostic| classes.violation(&paths[diagnostic.file], diagnostic)) + .map(|diagnostic| { + let row: LintViolationRow = LintViolationRow::diagnostic(diagnostic); + classes.violation(&paths[row.file], &row) + }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs deleted file mode 100644 index 3f0801a4dd..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! The stop signal that lets a speculative lint run be abandoned between files. - -use std::sync::{Mutex, PoisonError}; - -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::sync::PyOnceLock; -use pyo3::types::PyTuple; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; - -use crate::bindings::_helpers::compile_driver::python_calls::context_variable; -use crate::bindings::_helpers::records::ids::ErrorId; -use crate::bindings::_helpers::records::registry; - -const STOP_VARIABLE: &str = "active_lint_stop"; - -static ACTIVE_STOP: PyOnceLock> = PyOnceLock::new(); - -/// The context variable holding the active lint run's stop event, if any. -fn active_stop_variable(py: Python<'_>) -> PyResult> { - context_variable(py, &ACTIVE_STOP, STOP_VARIABLE) -} - -/// Abandon a speculative lint run on this thread once its owner no longer needs it. -pub(crate) fn check_stop(py: Python<'_>) -> PyResult<()> { - let stop: Bound<'_, PyAny> = active_stop_variable(py)?.call_method0("get")?; - if !stop.is_none() && stop.call_method0("is_set")?.is_truthy()? { - return Err(PyErr::from_value( - registry::error(py, ErrorId::LintStopped)?.call0()?, - )); - } - Ok(()) -} - -/// Makes `stop` the active lint stop event while entered. -#[pyclass(frozen, module = "sqlbuild._native")] -struct LintStopScope { - stop: Py, - token: Mutex>>, -} - -#[pymethods] -impl LintStopScope { - fn __enter__(&self, py: Python<'_>) -> PyResult<()> { - let token: Bound<'_, PyAny> = - active_stop_variable(py)?.call_method1("set", (self.stop.bind(py),))?; - *self.token.lock().unwrap_or_else(PoisonError::into_inner) = Some(token.unbind()); - Ok(()) - } - - #[pyo3(signature = (*_exception))] - fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { - let token: Option> = self - .token - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take(); - if let Some(token) = token { - active_stop_variable(py)?.call_method1("reset", (token,))?; - } - Ok(false) - } -} - -/// A scope that makes `stop` the active lint stop event while entered. -#[pyfunction] -fn lint_stop_scope(stop: &Bound<'_, PyAny>) -> LintStopScope { - LintStopScope { - stop: stop.clone().unbind(), - token: Mutex::new(None), - } -} - -/// The active lint run's stop event, or `None` outside a stop scope. -#[pyfunction] -fn active_lint_stop(py: Python<'_>) -> PyResult> { - active_stop_variable(py)?.call_method0("get") -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(lint_stop_scope, module)?)?; - module.add_function(wrap_pyfunction!(active_lint_stop, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index 6ae4847acc..9de19afb41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; use sqlbuild_rules::lint_text::errors::LintTextError; use sqlbuild_rules::lint_text::main::apply_suppressions::apply_suppressions; use sqlbuild_rules::lint_text::main::format_sql_bodies::format_sql_bodies; @@ -19,16 +18,18 @@ use sqlbuild_rules::lint_text::main::restore_interpolation::restore_interpolatio use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use sqlbuild_rules::lint_text::models::{ FormatFile, FormattedBodies, InterpolationSite, Neutralized, RetainedViolation, - SuppressibleViolation, SuppressionFinding, + SuppressibleViolation, }; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::lint_pipeline::{ + LintViolationRow, NATIVE_ENGINE, ViolationClasses, +}; use crate::bindings::types::CompilerDetach; -const NATIVE_ENGINE: &str = "native"; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; @@ -137,7 +138,7 @@ fn named_declaration_role_kind<'py>( /// Remove violations matched by suppressions and add the suppression findings, as lint models. #[pyfunction] -fn apply_lint_suppressions<'py>( +pub(crate) fn apply_lint_suppressions<'py>( py: Python<'py>, violations: Vec>, contents_by_path: Bound<'py, PyDict>, @@ -159,12 +160,15 @@ fn apply_lint_suppressions<'py>( let retained: Vec = compiler_guard(|| { apply_suppressions(&texts, &keys, allow_suppressions).map_err(value_error) })?; - let classes: FindingClasses<'py> = FindingClasses::new(py)?; + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; retained .into_iter() .map(|item| match item { RetainedViolation::Existing(index) => Ok(violations[index].clone()), - RetainedViolation::Added(finding) => classes.violation(&paths[finding.file], finding), + RetainedViolation::Added(finding) => { + let row: LintViolationRow = LintViolationRow::suppression(finding); + classes.violation(&paths[row.file], &row) + } }) .collect() } @@ -186,70 +190,6 @@ fn suppressible( }) } -/// The lint model classes suppression findings are built as. -struct FindingClasses<'py> { - violation: Bound<'py, PyAny>, - edit: Bound<'py, PyAny>, - severity: Bound<'py, PyAny>, -} - -impl<'py> FindingClasses<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - violation: registry::class(py, RecordId::LintViolation)?, - edit: registry::class(py, RecordId::LintEdit)?, - severity: registry::class(py, RecordId::LintSeverity)?, - }) - } - - fn violation( - &self, - file_path: &Bound<'py, PyAny>, - finding: SuppressionFinding, - ) -> PyResult> { - let py: Python<'py> = file_path.py(); - let code: Bound<'py, PyAny> = PyString::new(py, SUPPRESSION_CODE).into_any(); - let fix: Bound<'py, PyAny> = match finding.fix { - Some((start, end)) => call_with( - &self.edit, - &[ - ("file_path", file_path.clone()), - ("code", code.clone()), - ("start", start.into_pyobject(py)?.into_any()), - ("end", end.into_pyobject(py)?.into_any()), - ("replacement", PyString::new(py, "").into_any()), - ], - )?, - None => py.None().into_bound(py), - }; - let severity: &str = if finding.fault { "FAULT" } else { "WARNING" }; - call_with( - &self.violation, - &[ - ("file_path", file_path.clone()), - ("line", finding.line.into_pyobject(py)?.into_any()), - ("column", 1_usize.into_pyobject(py)?.into_any()), - ("code", code), - ("message", PyString::new(py, &finding.message).into_any()), - ("severity", self.severity.getattr(severity)?), - ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), - ( - "remediation", - PyString::new(py, finding.remediation).into_any(), - ), - ("fix", fix), - ( - "fix_unavailable_reason", - finding.fix_unavailable_reason.map_or_else( - || py.None().into_bound(py), - |reason| PyString::new(py, reason).into_any(), - ), - ), - ], - ) - } -} - /// The parts of `path` below `project_dir`, or `None` outside the project. fn relative_parts(path: &std::path::Path, project_dir: &std::path::Path) -> Option> { match path.strip_prefix(project_dir) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 5c3ad0aa26..9232d48281 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -19,10 +19,12 @@ pub(crate) mod guidance; pub(crate) mod lint; pub(crate) mod lint_expansion; pub(crate) mod lint_headers; +pub(crate) mod lint_inputs; +pub(crate) mod lint_pipeline; pub(crate) mod lint_preparation; pub(crate) mod lint_sql_bodies; -pub(crate) mod lint_stop; pub(crate) mod lint_text; +pub(crate) mod native_header_parser; pub(crate) mod project_tree_queries; pub(crate) mod registration; pub(crate) mod relation_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs new file mode 100644 index 0000000000..118e082fc6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs @@ -0,0 +1,275 @@ +//! DSL header values parsed natively for header rules, compared as Python compares their objects. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_rules::dsl_headers::main::inner_header_text::inner_header_text; +use sqlbuild_rules::dsl_headers::models::{HeaderSpan, ParseFailure, ParseOutcome}; +use sqlbuild_rules::dsl_headers::types::HeaderParser; +use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; +use sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch; +use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::records::ids::ErrorId; +use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sqltext::model_headers::header_failure; + +/// The path header rules report header syntax errors at. +const SENTINEL_PATH: &str = ""; +const DESCRIPTION_KEY: &str = "description"; +const FUNCTION_KIND: &str = "FUNCTION"; +const ARGUMENTS_KEY: &str = "arguments"; +const NEGATIVE_ZERO: &str = "-0"; +const ZERO: &str = "0"; + +/// Header values parsed natively, compared as their Python projections compare. +#[derive(Default)] +pub(crate) struct NativeHeaderParser; + +/// The `ModelSqlParseError` a rejected header parse raises. +pub(crate) fn rejection(py: Python<'_>, failure: ParseFailure) -> PyErr { + let raised: PyResult = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", failure.help)?; + let error: Bound<'_, PyAny> = registry::error(py, ErrorId::ModelSqlParse)? + .call((failure.message,), Some(&keywords))?; + Ok(PyErr::from_value(error)) + })(); + raised.unwrap_or_else(|error| error) +} + +impl HeaderParser for NativeHeaderParser { + type Values = AuthoredValue; + /// A native failure's message. + type Error = String; + + fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { + let headers: Vec = vec![inner.to_owned()]; + let parsed = match catch_compiler_panic(|| parse_batch(&headers)) { + Ok(parsed) => parsed, + Err(message) => { + return ParseOutcome::Raised( + message.clone(), + ParseFailure { + message, + help: None, + }, + ); + } + }; + let (value, _, error) = parsed.into_iter().next().unwrap_or((None, None, None)); + if let Some(value) = value { + return ParseOutcome::Parsed(value); + } + let error: String = error.unwrap_or_else(|| "None".to_owned()); + let (detail, help, _) = header_failure(&error, inner, None); + let failure: ParseFailure = ParseFailure { + message: format!( + "{kind}(...) in '{SENTINEL_PATH}' contains invalid SQLBuild header syntax: {detail}" + ), + help, + }; + ParseOutcome::Rejected(failure) + } + + fn description(&self, values: &AuthoredValue) -> Result, String> { + Ok(match map_value(values, DESCRIPTION_KEY) { + Some(AuthoredValue::String(text)) => Some(text.clone()), + Some(AuthoredValue::BareWord(word)) if header_word_value(word) == HeaderWord::Text => { + Some(word.clone()) + } + _ => None, + }) + } + + fn same(&self, left: &AuthoredValue, right: &AuthoredValue) -> Result { + Ok(python_equal(left, right)) + } +} + +/// Why a header could not be read: a header syntax error, or a native failure's message. +pub(crate) enum HeaderFailure { + Rejected(ParseFailure), + Native(String), +} + +impl HeaderFailure { + pub(crate) fn into_pyerr(self, py: Python<'_>) -> PyErr { + match self { + Self::Rejected(failure) => rejection(py, failure), + Self::Native(message) => value_error(message), + } + } +} + +/// The SQL function argument names the first FUNCTION header declares, lowercased and sorted. +pub(crate) fn external_identifiers( + text: &[char], + headers: &[HeaderSpan], + parser: &NativeHeaderParser, +) -> Result, HeaderFailure> { + for header in headers { + if header.kind != FUNCTION_KIND { + continue; + } + let inner: String = inner_header_text(&text[header.start..header.end]) + .iter() + .collect(); + let values: AuthoredValue = match parser.parse(&header.kind, &inner) { + ParseOutcome::Parsed(values) => values, + ParseOutcome::Rejected(failure) => return Err(HeaderFailure::Rejected(failure)), + ParseOutcome::Raised(message, _) => return Err(HeaderFailure::Native(message)), + }; + let Some(arguments) = map_value(&values, ARGUMENTS_KEY) else { + continue; + }; + if !matches!(arguments, AuthoredValue::Map(_)) { + continue; + } + let mut names: Vec = map_keys(arguments) + .into_iter() + .map(str::to_lowercase) + .collect(); + names.sort_unstable(); + return Ok(names); + } + Ok(Vec::new()) +} + +/// The value a header map projects for `key`: its last entry, as a Python dict keeps it. +pub(crate) fn map_value<'a>(values: &'a AuthoredValue, key: &str) -> Option<&'a AuthoredValue> { + match values { + AuthoredValue::Map(entries) => entries + .iter() + .rev() + .find(|(name, _)| name == key) + .map(|(_, value)| value), + _ => None, + } +} + +/// A header map's keys once each, as the projected Python dict holds them. +pub(crate) fn map_keys(values: &AuthoredValue) -> Vec<&str> { + let mut keys: Vec<&str> = Vec::new(); + if let AuthoredValue::Map(entries) = values { + for (name, _) in entries { + if !keys.contains(&name.as_str()) { + keys.push(name); + } + } + } + keys +} + +/// A scalar as its projected Python object compares. +#[derive(Debug, PartialEq)] +enum Scalar<'a> { + None, + Integer(i128), + BigInteger(String), + Float(f64), + Text(&'a str), +} + +fn scalar(value: &AuthoredValue) -> Option> { + Some(match value { + AuthoredValue::Null => Scalar::None, + AuthoredValue::Boolean(flag) => Scalar::Integer(i128::from(*flag)), + AuthoredValue::String(text) => Scalar::Text(text), + AuthoredValue::BareWord(word) => match header_word_value(word) { + HeaderWord::Bool(flag) => Scalar::Integer(i128::from(flag)), + HeaderWord::Null => Scalar::None, + HeaderWord::Integer(digits) => match digits.parse::() { + Ok(number) => Scalar::Integer(number), + Err(_) => Scalar::BigInteger(digits), + }, + HeaderWord::Float(number) => Scalar::Float(number), + HeaderWord::Text => Scalar::Text(word), + }, + _ => return None, + }) +} + +fn scalars_equal(left: &Scalar<'_>, right: &Scalar<'_>) -> bool { + match (left, right) { + (Scalar::Integer(integer), Scalar::Float(float)) + | (Scalar::Float(float), Scalar::Integer(integer)) => { + integral_float_text(*float).is_some_and(|text| text == integer.to_string()) + } + (Scalar::BigInteger(digits), Scalar::Float(float)) + | (Scalar::Float(float), Scalar::BigInteger(digits)) => { + integral_float_text(*float).is_some_and(|text| text == digits.trim_start_matches('+')) + } + (Scalar::Float(left), Scalar::Float(right)) => left == right, + (left, right) => left == right, + } +} + +/// The exact decimal digits of an integral finite float, as Python compares it with an int. +fn integral_float_text(float: f64) -> Option { + if !float.is_finite() || float.trunc() != float { + return None; + } + let text: String = format!("{float:.0}"); + Some(if text == NEGATIVE_ZERO { + ZERO.to_owned() + } else { + text + }) +} + +/// Whether two parsed header values project to equal Python objects. +pub(crate) fn python_equal(left: &AuthoredValue, right: &AuthoredValue) -> bool { + if let (Some(left), Some(right)) = (scalar(left), scalar(right)) { + return scalars_equal(&left, &right); + } + match (left, right) { + (AuthoredValue::List(left), AuthoredValue::List(right)) + | (AuthoredValue::Tuple(left), AuthoredValue::Tuple(right)) + | (AuthoredValue::Set(left), AuthoredValue::Set(right)) => sequences_equal(left, right), + (AuthoredValue::Map(_), AuthoredValue::Map(_)) => maps_equal(left, right), + (AuthoredValue::TypedConstant(left), AuthoredValue::TypedConstant(right)) => { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| left.0 == right.0 && python_equal(&left.1, &right.1)) + } + (AuthoredValue::InlineSqlHook(left), AuthoredValue::InlineSqlHook(right)) => left == right, + ( + AuthoredValue::NamedSqlHook(left_name, left_kwargs), + AuthoredValue::NamedSqlHook(right_name, right_kwargs), + ) + | ( + AuthoredValue::PythonHook(left_name, left_kwargs), + AuthoredValue::PythonHook(right_name, right_kwargs), + ) => { + left_name == right_name + && maps_equal( + &AuthoredValue::Map(left_kwargs.clone()), + &AuthoredValue::Map(right_kwargs.clone()), + ) + } + _ => false, + } +} + +fn sequences_equal(left: &[AuthoredValue], right: &[AuthoredValue]) -> bool { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| python_equal(left, right)) +} + +fn maps_equal(left: &AuthoredValue, right: &AuthoredValue) -> bool { + let (left_keys, right_keys) = (map_keys(left), map_keys(right)); + left_keys.len() == right_keys.len() + && left_keys + .iter() + .all(|key| match (map_value(left, key), map_value(right, key)) { + (Some(left), Some(right)) => python_equal(left, right), + _ => false, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 802295076a..52ec1a67af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -6,9 +6,9 @@ use pyo3::{types::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::rules::{ builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, - early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_stop, - lint_text, results, rule_api, rule_cache, rule_plan, rule_run, rules_config, run_lint, - selection, sql_rules, + early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_text, + results, rule_api, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, + sql_rules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -18,26 +18,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::rules::lint_headers::lint_file_role, module )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_headers::lint_body_ranges, - module - )?)?; module.add_function(wrap_pyfunction!( crate::bindings::_helpers::rules::lint_preparation::prepare_lint_body, module )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_preparation::expand_lint_body, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_preparation::expand_file_bodies, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_preparation::prepare_expanded_files, - module - )?)?; module.add_function(wrap_pyfunction!( crate::bindings::_helpers::rules::fix_session::plan_rule_fixes, module @@ -62,7 +46,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { catalogue::register(module)?; rules_config::register(module)?; run_lint::register(module)?; - lint_stop::register(module)?; selection::register(module)?; results::register(module)?; guidance::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index dce80ccc23..1bacb85af8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -8,6 +8,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation_keys as derive_keys; use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; +use crate::bindings::_helpers::rules::lint_inputs::model_key_row; + const MATERIALIZED_CONFIG: &str = "materialized"; const INCREMENTAL_MODE_CONFIG: &str = "incremental_mode"; const INCREMENTAL_STRATEGY_CONFIG: &str = "incremental_strategy"; @@ -145,19 +147,15 @@ fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult>)>; + /// Keys that prove one row per key, by `kind:name` identity, from model rows, seeds and sources. -#[pyfunction] -#[pyo3(signature = (*, model_rows, seeds, sources))] -fn declared_relation_keys<'py>( +pub(crate) fn relation_keys<'py>( py: Python<'py>, - model_rows: &Bound<'py, PyAny>, - seeds: &Bound<'py, PyAny>, - sources: &Bound<'py, PyAny>, -) -> PyResult> { - let mut rows: Vec = Vec::new(); - for row in model_rows.try_iter()? { - rows.push(row?.extract()?); - } + mut rows: Vec, + (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult { for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { for entry in entries.try_iter()? { let entry: Bound<'py, PyAny> = entry?; @@ -172,6 +170,11 @@ fn declared_relation_keys<'py>( } let relations: Vec = rows.into_iter().map(declared_relation).collect(); let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); + Ok(keys.into_iter().collect()) +} + +/// `keys` as a `{identity: ((column, ...), ...)}` dict. +fn keys_dict<'py>(py: Python<'py>, keys: RelationKeys) -> PyResult> { let found: Bound<'py, PyDict> = PyDict::new(py); for (identity, unique_keys) in keys { let mut tuples: Vec> = Vec::with_capacity(unique_keys.len()); @@ -183,30 +186,49 @@ fn declared_relation_keys<'py>( Ok(found) } -/// Declared keys of a compiled project's models, seeds and sources. +/// Keys that prove one row per key, by `kind:name` identity, from model rows, seeds and sources. #[pyfunction] -fn compiled_relation_keys<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - let mut model_rows: Vec> = Vec::new(); +#[pyo3(signature = (*, model_rows, seeds, sources))] +fn declared_relation_keys<'py>( + py: Python<'py>, + model_rows: Vec, + seeds: &Bound<'py, PyAny>, + sources: &Bound<'py, PyAny>, +) -> PyResult> { + keys_dict(py, relation_keys(py, model_rows, (seeds, sources))?) +} + +/// Declared keys of a compiled project's models, seeds and sources. +pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = project.py(); + let mut rows: Vec = Vec::new(); for model in project.getattr("models")?.try_iter()? { - model_rows.push(model?.getattr("relation_key_row")?); + rows.push(model_key_row(&model?)?); } - let mut seeds: Vec> = Vec::new(); + let mut seeds: Vec> = Vec::new(); for seed in project.getattr("seeds")?.try_iter()? { seeds.push(seed?.getattr("schema_entry")?); } - let mut sources: Vec> = Vec::new(); + let mut sources: Vec> = Vec::new(); for source in project.getattr("sources")?.try_iter()? { sources.push(source?.getattr("source_entry")?); } - declared_relation_keys( + relation_keys( py, - PyTuple::new(py, model_rows)?.as_any(), - PyTuple::new(py, seeds)?.as_any(), - PyTuple::new(py, sources)?.as_any(), + rows, + ( + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ), ) } +/// Declared keys of a compiled project's models, seeds and sources. +#[pyfunction] +fn compiled_relation_keys<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + keys_dict(project.py(), project_relation_keys(project)?) +} + fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits) = row; DeclaredRelation { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index 6600388df3..cfcc0e6bc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -1,7 +1,7 @@ //! SQL rule result cache keys, stored-entry lookup and the cache file. use std::collections::BTreeMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{ @@ -22,33 +22,33 @@ use sqlbuild_rules::rule_cache::main::parse_rule_cache::parse_rule_cache; use sqlbuild_rules::rule_cache::main::rule_cache_text::rule_cache_text; use sqlbuild_rules::rule_cache::models::{CacheLookup, CachedEntry, LintKeyInput, ModelKeyInput}; use sqlbuild_rules::rule_cache::types::CachedEntries; +use sqlbuild_rules::rule_results::models::RuleFinding; + +use crate::bindings::_helpers::rules::lint_pipeline::LintSettings; /// Stored entries as `{path: (key, findings JSON)}`. -type StoredEntries = BTreeMap; +pub(crate) type StoredEntries = BTreeMap; fn invalid(error: serde_json::Error) -> PyErr { PyValueError::new_err(error.to_string()) } +/// The lint settings part of every SQL rule cache key. +pub(crate) fn settings_lint_key(settings: &LintSettings) -> String { + lint_key(&LintKeyInput { + dialect: settings.sql.dialect.clone(), + max_literal_length: settings.sql.max_literal_length, + max_ranking_order_by: settings.sql.max_ranking_order_by, + relation_keys: settings.relation_keys.clone(), + allow_suppressions: settings.allow_suppressions, + }) +} + /// The lint settings part of every SQL rule cache key. #[pyfunction] #[pyo3(signature = (*, config))] fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { - let mut relation_keys: Vec<(String, Vec>)> = Vec::new(); - for item in config - .getattr("relation_keys")? - .call_method0("items")? - .try_iter()? - { - relation_keys.push(item?.extract()?); - } - Ok(lint_key(&LintKeyInput { - dialect: config.getattr("dialect")?.extract()?, - max_literal_length: config.getattr("max_literal_length")?.extract()?, - max_ranking_order_by: config.getattr("max_ranking_order_by")?.extract()?, - relation_keys, - allow_suppressions: config.getattr("allow_suppressions")?.extract()?, - })) + Ok(settings_lint_key(&LintSettings::from_record(config)?)) } /// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. @@ -58,6 +58,15 @@ fn sql_model_rule_keys( lint_key: &str, codes: Vec, models: Vec<(String, String, String, Option)>, +) -> BTreeMap { + model_rule_keys(lint_key, &codes, &models) +} + +/// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. +pub(crate) fn model_rule_keys( + lint_key: &str, + codes: &[String], + models: &[(String, String, String, Option)], ) -> BTreeMap { models .iter() @@ -67,7 +76,7 @@ fn sql_model_rule_keys( query_sql, contract: contract.as_deref(), }; - (path.clone(), model_rule_key(lint_key, &codes, &model)) + (path.clone(), model_rule_key(lint_key, codes, &model)) }) .collect() } @@ -94,7 +103,12 @@ fn sql_file_rule_key(request: Option<&Bound<'_, PyDict>>) -> PyResult PyResult { - let Ok(text) = std::fs::read_to_string(&path) else { + stored_entries(&path) +} + +/// The usable stored entries of the cache file at `path`; unreadable files read as empty. +pub(crate) fn stored_entries(path: &Path) -> PyResult { + let Ok(text) = std::fs::read_to_string(path) else { return Ok(StoredEntries::new()); }; parse_rule_cache(&text) @@ -120,6 +134,69 @@ fn cached_entries(entries: &StoredEntries) -> PyResult { .collect() } +/// The stored findings of each path whose key matches, as rows; undecodable stored findings miss. +pub(crate) fn cached_rule_findings( + entries: &StoredEntries, + keys: &[(String, String)], +) -> PyResult>> { + let lookup: CacheLookup = lookup_rule_cache(&cached_entries(entries)?, keys); + let mut hits: BTreeMap> = BTreeMap::new(); + for (path, findings) in lookup.hits { + let decoded: Option> = findings.iter().map(payload_finding).collect(); + if let Some(decoded) = decoded { + hits.insert(path, decoded); + } + } + Ok(hits) +} + +/// One finding of a stored payload, or `None` when the payload is malformed. +fn payload_finding(value: &Value) -> Option { + let payload = value.as_object()?; + let text = |key: &str| payload.get(key).and_then(Value::as_str).map(str::to_owned); + let number = |key: &str| { + payload + .get(key) + .filter(|value| value.is_u64()) + .and_then(Value::as_i64) + }; + let flag = |key: &str| payload.get(key).and_then(Value::as_bool) == Some(true); + Some(RuleFinding { + code: text("code")?, + path: text("path")?, + line: number("line")?, + column: number("column")?, + message: text("message")?, + remediation: text("remediation")?, + unevaluated: flag("unevaluated"), + affected_rules: Vec::new(), + fixable: flag("fixable"), + }) +} + +/// The stored payload of one finding row. +pub(crate) fn finding_json(finding: &RuleFinding) -> Value { + json!({ + "unevaluated": finding.unevaluated, + "code": finding.code, + "path": finding.path, + "line": finding.line, + "column": finding.column, + "message": finding.message, + "remediation": finding.remediation, + "fixable": finding.fixable, + }) +} + +/// The stored JSON of one path's finding rows. +pub(crate) fn encoded_findings(findings: &[&RuleFinding]) -> PyResult { + let payloads: Vec = findings + .iter() + .map(|finding| finding_json(finding)) + .collect(); + serde_json::to_string(&payloads).map_err(invalid) +} + /// The stored findings of each path whose key matches; undecodable stored findings miss. #[pyfunction] #[pyo3(signature = (*, entries, keys))] @@ -179,13 +256,18 @@ fn write_sql_rule_cache(path: PathBuf, entries: &Bound<'_, PyDict>) -> PyResult< for (key, value) in entries.iter() { stored.insert(key.extract()?, value.extract()?); } - let text: String = rule_cache_text(&cached_entries(&stored)?); + write_rule_cache(&path, &stored) +} + +/// Replace the cache file at `path` atomically with `entries`. +pub(crate) fn write_rule_cache(path: &Path, stored: &StoredEntries) -> PyResult<()> { + let text: String = rule_cache_text(&cached_entries(stored)?); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } let temporary: PathBuf = path.with_extension(format!("tmp-{}", std::process::id())); std::fs::write(&temporary, text)?; - std::fs::rename(&temporary, &path)?; + std::fs::rename(&temporary, path)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 5146f908a7..37f2cbce91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -11,13 +11,11 @@ use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::cache::native_store::current_code_identity; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, within, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -59,23 +57,7 @@ const EVALUATE_RULES_OPTIONS: (&[&str], &[&str]) = ( "project_dir", "dialect", ], - &[ - "selected_keys", - "prepared_sql", - "expansion_reuse", - "no_sql_analysis", - ], -); -const PREPARE_OPTIONS: (&[&str], &[&str]) = ( - &[ - "project_dir", - "config", - "discovered_inputs", - "static_declaration_scope", - "compiled_expansions", - "stop", - ], - &[], + &["selected_keys", "early_lint", "no_sql_analysis"], ); /// A running custom-rule evaluation, whose result is a `CustomRulesOutcome`. @@ -312,8 +294,7 @@ pub(crate) fn evaluate_rules<'py>( ("graph", graph), ("discovered_inputs", arguments.value("discovered_inputs")?), ("dialect", dialect), - ("prepared_sql", arguments.value("prepared_sql")?), - ("expansion_reuse", arguments.value("expansion_reuse")?), + ("early_lint", arguments.value("early_lint")?), ], ); cancel_on_interrupt(py, &sql_result, &run.cancel); @@ -330,97 +311,6 @@ pub(crate) fn evaluate_rules<'py>( ) } -/// Start early SQL lint on `executor` when the project selects SQL rules without a cache. -#[pyfunction] -#[pyo3(signature = (*, inputs, executor, dialect, stop))] -pub(crate) fn prepare_sql_rules<'py>( - inputs: &Bound<'py, PyAny>, - executor: &Bound<'py, PyAny>, - dialect: &Bound<'py, PyAny>, - stop: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = inputs.py(); - let preparation: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::SqlLintPreparation)?, - &[("inputs", inputs.clone()), ("dialect", dialect.clone())], - )?; - if preparation.is_none() { - return Ok(preparation); - } - let (project_dir, codes, config): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = - preparation.extract()?; - let expansions: Bound<'py, PyAny> = - native(py, NativeAttr::EarlyLintExpansions)?.call1((inputs.getattr("model_inputs")?,))?; - let work: Bound<'py, PyDict> = PyDict::new(py); - work.set_item("project_dir", &project_dir)?; - work.set_item("config", config)?; - work.set_item("discovered_inputs", inputs.getattr("discovered_inputs")?)?; - work.set_item( - "static_declaration_scope", - inputs.getattr("declaration_scope")?, - )?; - work.set_item("compiled_expansions", expansions)?; - work.set_item("stop", stop)?; - let future: Bound<'py, PyAny> = executor.call_method( - "submit", - (wrap_pyfunction!(prepare_early_sql_lint, py)?,), - Some(&work), - )?; - call_with( - ®istry::class(py, RecordId::PreparedSqlLint)?, - &[ - ("codes", codes), - ("project_dir", project_dir), - ("future", future), - ], - ) -} - -/// Expand and lint SQL early, stopping when `stop` is set; runs on the early-lint executor. -#[pyfunction] -#[pyo3(signature = (**options))] -fn prepare_early_sql_lint<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = - KeywordArguments::read(py, "prepare_early_sql_lint", options, PREPARE_OPTIONS)?; - let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; - let discovered_inputs: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - let scope: Bound<'py, PyAny> = - native(py, NativeAttr::LintStopScopeFunction)?.call1((arguments.value("stop")?,))?; - within(&scope, |_| { - let context: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::BuildLintExpansionContext)?, - &[ - ("project_dir", project_dir.clone()), - ("discovered_inputs", discovered_inputs.clone()), - ( - "static_declaration_scope", - arguments.value("static_declaration_scope")?, - ), - ], - )?; - let result: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::RunLint)?, - &[ - ("project_dir", project_dir.clone()), - ("config", arguments.value("config")?), - ("discovered_inputs", discovered_inputs.clone()), - ( - "compiled_expansions", - arguments.value("compiled_expansions")?, - ), - ("expansion_context", context.clone()), - ], - )?; - call_with( - ®istry::class(py, RecordId::PreparedSqlLintResult)?, - &[("result", result), ("context", context)], - ) - }) -} - /// Evaluate built-in rules natively while custom rules run beside them. fn evaluate_native<'py>( run: &NativeRun<'py>, @@ -690,7 +580,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(evaluate_project_rules, module)?)?; module.add_function(wrap_pyfunction!(evaluate_native_rules, module)?)?; module.add_function(wrap_pyfunction!(evaluate_custom_rules, module)?)?; - module.add_function(wrap_pyfunction!(evaluate_rules, module)?)?; - module.add_function(wrap_pyfunction!(prepare_sql_rules, module)?)?; - module.add_function(wrap_pyfunction!(prepare_early_sql_lint, module)?) + module.add_function(wrap_pyfunction!(evaluate_rules, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index 87c7f7e583..3b8c057520 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -1,6 +1,6 @@ //! Rules configuration loaded natively and returned as the Python configuration models. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -241,20 +241,50 @@ fn rules_error(py: Python<'_>, class: ErrorId, message: String, help: Option, project_dir: PathBuf) -> PyResult> { + let config: RulesConfig = match rules_config_of(&project_dir)? { + Ok(config) => config, + Err(invalid) => return Err(invalid.into_pyerr(py)), + }; + Ok(config_object(py, &config)?.unbind()) +} + +/// Why a project's rules configuration is invalid. +pub(crate) enum InvalidRulesConfig { + /// The configuration could not be read: a `RulesError`. + Unreadable(String), + /// It allows exceptions it may not: a `RulesConfigError` with help. + Forbidden { message: String, help: String }, +} + +impl InvalidRulesConfig { + fn into_pyerr(self, py: Python<'_>) -> PyErr { + match self { + Self::Unreadable(message) => rules_error(py, ErrorId::Rules, message, None), + Self::Forbidden { message, help } => { + rules_error(py, ErrorId::RulesConfig, message, Some(help)) + } + } + } +} + +/// The project's validated rules configuration, or why it is invalid. +pub(crate) fn rules_config_of( + project_dir: &Path, +) -> PyResult> { let project_dir: PathBuf = project_dir .canonicalize() - .or_else(|_| std::path::absolute(&project_dir))?; - let config: RulesConfig = compiler_guard(|| Ok(load_config(&project_dir)))? - .map_err(|message| rules_error(py, ErrorId::Rules, message, None))?; + .or_else(|_| std::path::absolute(project_dir))?; + let config: RulesConfig = match compiler_guard(|| Ok(load_config(&project_dir)))? { + Ok(config) => config, + Err(message) => return Ok(Err(InvalidRulesConfig::Unreadable(message))), + }; if let Some(error) = forbidden_exceptions(&config) { - return Err(rules_error( - py, - ErrorId::RulesConfig, - error.message, - Some(error.help), - )); + return Ok(Err(InvalidRulesConfig::Forbidden { + message: error.message, + help: error.help, + })); } - Ok(config_object(py, &config)?.unbind()) + Ok(Ok(config)) } /// The project's rules configuration, or `None` when it is invalid and the rules phase reports it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index a041f3769e..2a46abc916 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -1,258 +1,86 @@ //! Lint authored SQL files: header rules, then expanded bodies through the native SQL engine. -use std::collections::BTreeSet; -use std::fs; -use std::path::{Path, PathBuf}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::PathBuf; -use pyo3::exceptions::{PyException, PyOSError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; -use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ + SqlExpansionInputs, present, +}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::rules::lint_expansion::{ + LintExpansionSources, lint_expansion_inputs, +}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::rules::lint_expansion::build_lint_expansion_context; -use crate::bindings::_helpers::rules::lint_stop::check_stop; - -fn resolved(path: &Path) -> PathBuf { - fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) -} -fn selected_set(selected_paths: &Bound<'_, PyAny>) -> PyResult>> { - if selected_paths.is_none() { +/// The selected paths, or `None` for every file. +pub(crate) fn selected_paths(selected: &Bound<'_, PyAny>) -> PyResult>> { + if selected.is_none() { return Ok(None); } - selected_paths + selected .try_iter()? .map(|path| path?.extract()) .collect::>() .map(Some) } -/// Authored SQL files keyed by path: `source_files` filtered to the selection, or read from disk. -fn project_files<'py>( - project_dir: &Bound<'py, PyAny>, - selected_paths: &Bound<'py, PyAny>, - source_files: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let selected: Option> = selected_set(selected_paths)?; - let files: Bound<'py, PyDict> = PyDict::new(py); - if !source_files.is_none() { - for (path, contents) in source_files.downcast::()?.iter() { - let keep: bool = match &selected { - None => true, - Some(paths) => paths.contains(&resolved(&path.extract::()?)), - }; - if keep { - files.set_item(path, contents)?; - } - } - return Ok(files); +/// The files of a `{path: contents}` mapping, in its order. +pub(crate) fn given_files(files: &Bound<'_, PyAny>) -> PyResult>> { + if files.is_none() { + return Ok(None); + } + files + .downcast::()? + .iter() + .map(|(path, contents)| { + Ok(LintFile { + path: path.extract()?, + contents: contents.extract()?, + }) + }) + .collect::>() + .map(Some) +} + +/// The compile's expansions of a `{path: CompiledSqlExpansion}` mapping. +pub(crate) fn compiled_expansion_records( + expansions: &Bound<'_, PyAny>, +) -> PyResult> { + let mut compiled: HashMap = HashMap::new(); + if expansions.is_none() { + return Ok(compiled); } - let directory: PathBuf = project_dir.extract()?; - let collected: Vec<(PathBuf, String)> = collect_lint_files(&directory, selected.as_ref()) - .map_err(|error| PyOSError::new_err(error.to_string()))?; - for (path, contents) in collected { - files.set_item(path, contents)?; + for (path, expansion) in expansions.downcast::()?.iter() { + if !expansion.is_none() { + compiled.insert(path.extract()?, CompiledExpansion::from_record(&expansion)?); + } } - Ok(files) + Ok(compiled) } /// Collect supported SQL-analysis inputs of a project. #[pyfunction] #[pyo3(signature = (*, project_dir, selected_paths = None))] fn collect_lint_project_files<'py>( - project_dir: &Bound<'py, PyAny>, + py: Python<'py>, + project_dir: PathBuf, selected_paths: Option>, ) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let none: Bound<'py, PyAny> = py.None().into_bound(py); - project_files( - project_dir, - &selected_paths.unwrap_or_else(|| none.clone()), - &none, - ) -} - -/// The keyword arguments of one lint run. -struct LintRun<'py> { - project_dir: Bound<'py, PyAny>, - config: Bound<'py, PyAny>, - dynamic_output_paths: Bound<'py, PyAny>, - compiled_expansions: Bound<'py, PyAny>, - skip_unexpandable: bool, -} - -/// One file's header violations and, when expanding, its expanded bodies. -fn scan_file<'py>( - run: &LintRun<'py>, - (file_path, contents): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - context: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Option>)> { - let py: Python<'py> = file_path.py(); - let relative_path: Bound<'py, PyAny> = - file_path.call_method1("relative_to", (&run.project_dir,))?; - let relative_parts: Vec = relative_path.getattr("parts")?.extract()?; - let headers: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ScanHeaders)?, - &[ - ("contents", contents.clone()), - ( - "first_only", - PyBool::new(py, first_header_only(&relative_parts)) - .to_owned() - .into_any(), - ), - ], - )?; - let header_violations: Bound<'py, PyAny> = - if run.config.getattr("header_rules_enabled")?.is_truthy()? { - call_with( - ®istry::native(py, NativeAttr::LintNativeHeaders)?, - &[ - ("contents", contents.clone()), - ("file_path", file_path.clone()), - ("headers", headers.clone()), - ("config", run.config.clone()), - ], - )? - } else { - PyTuple::empty(py).into_any() - }; - if context.is_none() { - return Ok((header_violations, None)); - } - let expanded: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ExpandFileBodies)?, - &[ - ("file_path", file_path.clone()), - ("contents", contents.clone()), - ("headers", headers), - ("context", context.clone()), - ("dialect", run.config.getattr("dialect")?), - ("project_dir", run.project_dir.clone()), - ("relative_path", relative_path), - ("dynamic_output_paths", run.dynamic_output_paths.clone()), - ("compiled_expansions", run.compiled_expansions.clone()), - ], - )?; - Ok((header_violations, Some(expanded))) -} - -/// The expansion context: the caller's, or one built when native lint is enabled. -fn expansion_context<'py>( - run: &LintRun<'py>, - arguments: &KeywordArguments<'py>, -) -> PyResult> { - let py: Python<'py> = run.config.py(); - let given: Bound<'py, PyAny> = arguments.value("expansion_context")?; - if given.is_truthy()? { - return Ok(given); - } - if !run.config.getattr("native_enabled")?.is_truthy()? { - return Ok(py.None().into_bound(py)); - } - build_lint_expansion_context( - py, - Some(keywords( - py, - &[ - ("project_dir", run.project_dir.clone()), - ("value_renderer", arguments.value("value_renderer")?), - ("discovered_inputs", arguments.value("discovered_inputs")?), - ], - )?), - ) -} - -/// Violations sorted by file path text, then line and column. -fn sorted_violations<'py>(violations: Vec>) -> PyResult>> { - let mut keyed: Vec<((String, i64, i64), Bound<'py, PyAny>)> = violations - .into_iter() - .map(|violation| { - let key: (String, i64, i64) = ( - violation.getattr("file_path")?.str()?.extract()?, - violation.getattr("line")?.extract()?, - violation.getattr("column")?.extract()?, - ); - Ok((key, violation)) - }) - .collect::>()?; - keyed.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(keyed.into_iter().map(|(_, violation)| violation).collect()) -} - -/// Header and expanded-body violations of every file, and the files expansion skipped. -fn lint_files<'py>( - run: &LintRun<'py>, - files: &Bound<'py, PyDict>, - context: &Bound<'py, PyAny>, -) -> PyResult<(Vec>, Bound<'py, PyAny>)> { - let py: Python<'py> = files.py(); - let mut items: Vec<(PathBuf, Bound<'py, PyAny>, Bound<'py, PyAny>)> = files - .iter() - .map(|(path, contents)| Ok((path.extract::()?, path, contents))) - .collect::>()?; - items.sort_by(|left, right| left.0.cmp(&right.0)); - let mut violations: Vec> = Vec::new(); - let mut expanded_files: Vec> = Vec::new(); - let mut failure: Option = None; - for (_, file_path, contents) in &items { - let (header_violations, expanded) = match scan_file(run, (file_path, contents), context) { - Ok(scanned) => scanned, - Err(error) if error.is_instance_of::(py) => { - failure = Some(error); - break; - } - Err(error) => return Err(error), - }; - for violation in header_violations.try_iter()? { - violations.push(violation?); - } - let Some(expanded) = expanded else { - continue; - }; - let failed: bool = !expanded.getattr("failure")?.is_none(); - expanded_files.push(expanded); - if failed && !run.skip_unexpandable { - break; - } - } - let skip: Bound<'py, PyAny> = PyBool::new(py, run.skip_unexpandable).to_owned().into_any(); - let prepared: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::PrepareExpandedFiles)?, - &[ - ( - "expanded_files", - PyTuple::new(py, expanded_files)?.into_any(), - ), - ("dialect", run.config.getattr("dialect")?), - ("skip_unexpandable", skip), - ], - )?; - if let Some(error) = failure { - return Err(error); - } - let bodies: Bound<'py, PyAny> = prepared.getattr("bodies")?; - if bodies.is_truthy()? && run.config.getattr("native_enabled")?.is_truthy()? { - check_stop(py)?; - let body_list: Vec> = bodies.extract()?; - let found: Bound<'py, PyAny> = registry::native(py, NativeAttr::LintNativeSqlBodies)? - .call1((body_list, files, &run.config))?; - for violation in found.try_iter()? { - violations.push(violation?); - } - check_stop(py)?; + let selected: Option> = match selected_paths { + Some(paths) => self::selected_paths(&paths)?, + None => None, + }; + let files: Bound<'py, PyDict> = PyDict::new(py); + for file in lint_sources(&project_dir, selected.as_ref(), None)? { + files.set_item(file.path, file.contents)?; } - Ok((violations, prepared.getattr("unexpandable")?)) + Ok(files) } /// Lint all DSL files without modifying anything; optionally skip unexpandable SQL. @@ -281,42 +109,51 @@ fn run_lint<'py>( ), )?; let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; - let dynamic_output_paths: Bound<'py, PyAny> = if arguments.passed("dynamic_output_paths")? { - arguments.value("dynamic_output_paths")? + let settings: LintSettings = LintSettings::from_record(&arguments.value("config")?)?; + let project_path: PathBuf = project_dir.extract()?; + let files: Vec = lint_sources( + &project_path, + selected_paths(&arguments.value("selected_paths")?)?.as_ref(), + given_files(&arguments.value("source_files")?)?, + )?; + let given: Bound<'py, PyAny> = arguments.value("expansion_context")?; + let context: Option = if given.is_truthy()? { + Some(SqlExpansionInputs::from_record(&given)?) + } else if settings.native_enabled { + let renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; + Some(lint_expansion_inputs(LintExpansionSources { + project_dir: project_dir.clone(), + value_renderer: renderer.is_truthy()?.then_some(renderer), + discovered_inputs: present(arguments.value("discovered_inputs")?), + declaration_scope: None, + static_declaration_scope: None, + })?) } else { - PyFrozenSet::empty(py)?.into_any() - }; - let run: LintRun<'py> = LintRun { - project_dir, - config: arguments.value("config")?, - dynamic_output_paths, - compiled_expansions: arguments.value("compiled_expansions")?, - skip_unexpandable: arguments.flag("skip_unexpandable")?, + None }; - let files: Bound<'py, PyDict> = project_files( - &run.project_dir, - &arguments.value("selected_paths")?, - &arguments.value("source_files")?, + let mut dynamic_output_paths: HashSet = HashSet::new(); + let dynamic: Bound<'py, PyAny> = arguments.value("dynamic_output_paths")?; + if !dynamic.is_none() { + for path in dynamic.try_iter()? { + dynamic_output_paths.insert(path?.extract()?); + } + } + let outcome: LintOutcome = lint( + py, + &LintRequest { + project_dir: &project_path, + settings: &settings, + dynamic_output_paths: &dynamic_output_paths, + compiled_expansions: &compiled_expansion_records( + &arguments.value("compiled_expansions")?, + )?, + context: context.as_ref(), + skip_unexpandable: arguments.flag("skip_unexpandable")?, + stop: None, + }, + files, )?; - let context: Bound<'py, PyAny> = expansion_context(&run, &arguments)?; - let (violations, unexpandable) = lint_files(&run, &files, &context)?; - let allow_suppressions: Bound<'py, PyAny> = run.config.getattr("allow_suppressions")?; - let retained: Vec> = registry::native(py, NativeAttr::ApplyLintSuppressions)? - .call1((violations, &files, allow_suppressions))? - .extract()?; - call_with( - ®istry::class(py, RecordId::LintRunResult)?, - &[ - ("files_checked", files.len().into_pyobject(py)?.into_any()), - ( - "violations", - PyTuple::new(py, sorted_violations(retained)?)?.into_any(), - ), - ("formatted_files", PyTuple::empty(py).into_any()), - ("source_texts", files.into_any()), - ("unexpandable", unexpandable), - ], - ) + outcome.record(py) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index c7de972fb2..a98ccc8576 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -1,34 +1,52 @@ //! Run the planned SQL rules through native lint, reusing cached and early-lint results. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::path::{Path, PathBuf}; use std::time::Instant; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, - PyTuple, +use pyo3::exceptions::PyException; +use pyo3::prelude::{ + Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, + Python, }; +use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_rules::rule_cache::main::file_rule_key::file_rule_key; +use sqlbuild_rules::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; +use sqlbuild_rules::rule_results::models::RuleFinding; use crate::bindings::_helpers::analysis::model_reads::output_proven; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; -use crate::bindings::_helpers::rules::early_sql_lint::{sql_lint_config, sql_rule_cache_path}; +use crate::bindings::_helpers::rules::early_sql_lint::{ + EarlyLintResult, EarlySqlLint, PreparedSqlLint, sql_lint_settings, sql_rule_cache_path, +}; +use crate::bindings::_helpers::rules::lint_expansion::{ + LintExpansionSources, lint_expansion_inputs, +}; +use crate::bindings::_helpers::rules::lint_inputs::project_sql_expansions; +use crate::bindings::_helpers::rules::lint_pipeline::{ + CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, LintViolationRow, lint, + lint_sources, outside_project, resolved, +}; +use crate::bindings::_helpers::rules::relation_keys::project_relation_keys; +use crate::bindings::_helpers::rules::rule_cache::{ + StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, settings_lint_key, + stored_entries, write_rule_cache, +}; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; const BUILTINS_MODULE: &str = "builtins"; +const PATHLIB_MODULE: &str = "pathlib"; const SQL_RULE_PREFIX: &str = "SQBRSQL"; const SQL_RULE_SUPPRESSION_CODE: &str = "SQBRSQL000"; const DEFAULT_REMEDIATION: &str = "Update the authored SQL to satisfy this rule."; - -/// `native.name`, looked up at call time. -fn native<'py>(py: Python<'py>, name: NativeAttr) -> PyResult> { - registry::native(py, name) -} +const UTF8: &str = "utf-8"; /// The SQL rule codes among `rules`, in order. fn sql_rule_codes(rules: &[Bound<'_, PyAny>]) -> PyResult> { @@ -45,14 +63,30 @@ fn sql_rule_codes(rules: &[Bound<'_, PyAny>]) -> PyResult> { /// The settings, project and inputs one SQL rules evaluation reads. struct SqlRun<'a, 'py> { codes: &'a [String], - project_dir: Bound<'py, PyAny>, + project_dir: PathBuf, discovered_inputs: Bound<'py, PyAny>, project: Bound<'py, PyAny>, - lint_config: Bound<'py, PyAny>, - selected_model_paths: Bound<'py, PyAny>, + settings: LintSettings, + /// Whether the plan selected model paths, rather than the whole project. + model_selection: bool, cache_enabled: bool, - prepared_sql: Bound<'py, PyAny>, - expansion_reuse: Bound<'py, PyAny>, + early_lint: Option>, +} + +impl<'py> SqlRun<'_, 'py> { + /// The early SQL lint that started for this compile, if any. + fn prepared(&self) -> Option<&PreparedSqlLint> { + self.early_lint + .as_ref() + .and_then(|early| early.preparation()) + .map(|preparation| preparation.get()) + } +} + +/// `path` with forward slashes, as `PurePath.as_posix` writes it. +fn posix(path: &Path) -> String { + path.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") } /// Run the planned SQL rules on the caller while built-in rules evaluate natively. @@ -72,39 +106,43 @@ fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult< } } let codes: Vec = sql_rule_codes(&native_rules)?; - let lint_config: Bound<'py, PyAny> = sql_lint_config( - &options.get("dialect")?, + let thresholds: Bound<'py, PyAny> = effective_config.getattr("thresholds")?; + let dialect: String = options.get("dialect")?.extract()?; + let settings: LintSettings = sql_lint_settings( + &dialect, &codes, - &effective_config.getattr("thresholds")?, + |key| thresholds.call_method1("get", (key,))?.extract(), ( - native(py, NativeAttr::CompiledRelationKeys)?.call1((graph.getattr("project")?,))?, - effective_config.getattr("allow_exceptions")?, + project_relation_keys(&graph.getattr("project")?)?, + effective_config.getattr("allow_exceptions")?.is_truthy()?, ), )?; + let early_lint: Bound<'py, PyAny> = options.get("early_lint")?; let run = SqlRun { codes: &codes, - project_dir: plan.getattr("project_dir")?, + project_dir: plan.getattr("project_dir")?.extract()?, discovered_inputs: options.get("discovered_inputs")?, project: plan.getattr("project")?, - lint_config, - selected_model_paths: plan.getattr("model_paths")?, + settings, + model_selection: !plan.getattr("model_paths")?.is_none(), cache_enabled: effective_config .getattr("cache")? .getattr("enabled")? .is_truthy()?, - prepared_sql: options.get("prepared_sql")?, - expansion_reuse: options.get("expansion_reuse")?, + early_lint: if early_lint.is_none() { + None + } else { + Some(early_lint.downcast::()?.borrow()) + }, }; - let lint_key: String = lint_key(&run.lint_config)?; + let lint_key: String = settings_lint_key(&run.settings); let identities: Vec<(String, String)> = model_rule_identities(&run, &lint_key)?; - let project_files: Option> = if !identities.is_empty() - && run.prepared_sql.is_none() - && run.selected_model_paths.is_none() - { - Some(collected_project_files(&run.project_dir)?) - } else { - None - }; + let project_files: Option> = + if !identities.is_empty() && run.prepared().is_none() && !run.model_selection { + Some(lint_sources(&run.project_dir, None, None)?) + } else { + None + }; let started: Instant = Instant::now(); let (findings, hits, misses) = evaluated_sql_rules(&run, (&lint_key, identities), project_files)?; @@ -112,7 +150,7 @@ fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult< call_with( ®istry::class(py, RecordId::SqlRulesEvaluation)?, &[ - ("findings", findings), + ("findings", finding_objects(py, findings)?.into_any()), ("cache_hits", hits.into_bound_py_any(py)?), ("cache_misses", misses.into_bound_py_any(py)?), ("sql_ms", sql_ms.into_bound_py_any(py)?), @@ -120,23 +158,32 @@ fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult< ) } -fn lint_key(lint_config: &Bound<'_, PyAny>) -> PyResult { - call_with( - &native(lint_config.py(), NativeAttr::SqlRuleLintKey)?, - &[("config", lint_config.clone())], - )? - .extract() -} - -fn collected_project_files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project_dir.py(); - call_with( - &native(py, NativeAttr::CollectLintProjectFiles)?, - &[ - ("project_dir", project_dir.clone()), - ("selected_paths", py.None().into_bound(py)), - ], - ) +/// Finding rows as `Finding` records. +fn finding_objects(py: Python<'_>, findings: Vec) -> PyResult> { + let class: Bound<'_, PyAny> = registry::class(py, RecordId::Finding)?; + let path_class: Bound<'_, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let objects: Vec> = findings + .into_iter() + .map(|finding| { + call_with( + &class, + &[ + ("code", PyString::new(py, &finding.code).into_any()), + ("path", path_class.call1((finding.path,))?), + ("line", finding.line.into_pyobject(py)?.into_any()), + ("column", finding.column.into_pyobject(py)?.into_any()), + ("message", PyString::new(py, &finding.message).into_any()), + ( + "remediation", + PyString::new(py, &finding.remediation).into_any(), + ), + ("unevaluated", finding.unevaluated.into_bound_py_any(py)?), + ("fixable", finding.fixable.into_bound_py_any(py)?), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, objects) } /// The project's models SQL rules analyse, keyed by relative path in project order. @@ -174,192 +221,130 @@ fn sql_rule_models<'py>(project: &Bound<'py, PyAny>) -> PyResult, lint_key: &str) -> PyResult> { - let py: Python<'_> = run.project.py(); if run.codes.is_empty() { return Ok(Vec::new()); } - let mut models: Vec> = Vec::new(); - let mut paths: Vec = Vec::new(); + let mut models: Vec<(String, String, String, Option)> = Vec::new(); for (path, model) in sql_rule_models(&run.project)? { - models.push(PyTuple::new( - py, - [ - PyString::new(py, &path).into_any(), - model.getattr("authored_sql")?, - model.getattr("query_sql")?, - contract_text(&model)?, - ], - )?); - paths.push(path); + models.push(( + path, + model.getattr("authored_sql")?.extract()?, + model.getattr("query_sql")?.extract()?, + contract_text(&model)?, + )); } - let keys: Bound<'_, PyAny> = call_with( - &native(py, NativeAttr::SqlModelRuleKeys)?, - &[ - ("lint_key", PyString::new(py, lint_key).into_any()), - ("codes", PyList::new(py, run.codes)?.into_any()), - ("models", PyList::new(py, models)?.into_any()), - ], - )?; - paths + let keys: BTreeMap = model_rule_keys(lint_key, run.codes, &models); + Ok(models .into_iter() - .map(|path| { - let key: String = keys.get_item(&path)?.extract()?; - Ok((path, key)) + .map(|(path, ..)| { + let key: String = keys[&path].clone(); + (path, key) }) - .collect() + .collect()) } /// Canonical text of the dynamic column contract that SQL rule findings depend on. -fn contract_text<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - let contract: Bound<'py, PyAny> = model.getattr("dynamic_column_contract")?; +fn contract_text(model: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model.py(); + let contract: Bound<'_, PyAny> = model.getattr("dynamic_column_contract")?; if contract.is_none() { - return Ok(contract); - } - imported(py, JSON_MODULE, "dumps")?.call( - (imported(py, DATACLASSES_MODULE, "asdict")?.call1((contract,))?,), - Some(&keywords( - py, - &[ - ("sort_keys", true.into_bound_py_any(py)?), - ("separators", (",", ":").into_bound_py_any(py)?), - ("default", imported(py, BUILTINS_MODULE, "str")?), - ], - )?), - ) -} - -/// The stored findings of every path whose key still matches; undecodable ones miss. -fn cached_findings<'py>( - bucket: &Bound<'py, PyAny>, - keys: Vec<(String, String)>, -) -> PyResult> { - let py: Python<'py> = bucket.py(); - Ok(call_with( - &native(py, NativeAttr::LookupSqlRuleCache)?, - &[ - ("entries", bucket.clone()), - ("keys", keys.into_bound_py_any(py)?), - ], - )? - .downcast_into()?) -} - -/// The early SQL lint's full-project file read, when it ran for this project and these codes. -fn prepared_project_files<'py>(run: &SqlRun<'_, 'py>) -> PyResult>> { - let py: Python<'py> = run.project_dir.py(); - let prepared: &Bound<'py, PyAny> = &run.prepared_sql; - if prepared.is_none() - || prepared.getattr("project_dir")?.ne(&run.project_dir)? - || !same_codes(&prepared.getattr("codes")?, run.codes)? - || !prepared - .getattr("future")? - .call_method0("exception")? - .is_none() - { return Ok(None); } - let texts: Bound<'py, PyAny> = prepared - .getattr("future")? - .call_method0("result")? - .getattr("result")? - .getattr("source_texts")?; - Ok(Some( - imported(py, BUILTINS_MODULE, "dict")?.call1((texts,))?, - )) + imported(py, JSON_MODULE, "dumps")? + .call( + (imported(py, DATACLASSES_MODULE, "asdict")?.call1((contract,))?,), + Some(&keywords( + py, + &[ + ("sort_keys", true.into_bound_py_any(py)?), + ("separators", (",", ":").into_bound_py_any(py)?), + ("default", imported(py, BUILTINS_MODULE, "str")?), + ], + )?), + )? + .extract() + .map(Some) } -fn same_codes(codes: &Bound<'_, PyAny>, expected: &[String]) -> PyResult { - let mut known: BTreeSet = BTreeSet::new(); - if !codes.is_none() { - for code in codes.try_iter()? { - known.insert(code?.extract()?); - } +/// The early SQL lint's result, when it ran for this project and these codes. +fn prepared_result<'a>( + run: &'a SqlRun<'_, '_>, + py: Python<'_>, +) -> Option<&'a Result> { + let prepared: &PreparedSqlLint = run.prepared()?; + if !prepared.matches(&run.project_dir, run.codes) { + return None; } - Ok(known == expected.iter().cloned().collect()) + Some(prepared.result(py)) } /// Each analysed model's findings, cached or linted, then cache accounting. -fn evaluated_sql_rules<'py>( - run: &SqlRun<'_, 'py>, +fn evaluated_sql_rules( + run: &SqlRun<'_, '_>, (lint_key, identities): (&str, Vec<(String, String)>), - prepared_files: Option>, -) -> PyResult<(Bound<'py, PyAny>, usize, usize)> { - let py: Python<'py> = run.project_dir.py(); + prepared_files: Option>, +) -> PyResult<(Vec, usize, usize)> { + let py: Python<'_> = run.project.py(); if run.codes.is_empty() { - return Ok((PyTuple::empty(py).into_any(), 0, 0)); + return Ok((Vec::new(), 0, 0)); } - let project_dir: &Bound<'py, PyAny> = &run.project_dir; - let models: Vec<(String, Bound<'py, PyAny>)> = sql_rule_models(&run.project)?; - let cache_path: Bound<'py, PyAny> = sql_rule_cache_path(project_dir)?; - let bucket: Bound<'py, PyAny> = if run.cache_enabled { - call_with( - &native(py, NativeAttr::LoadSqlRuleCache)?, - &[("path", cache_path.clone())], - )? + let project_dir: &Path = &run.project_dir; + let models: Vec<(String, Bound<'_, PyAny>)> = sql_rule_models(&run.project)?; + let cache_path: PathBuf = sql_rule_cache_path(project_dir); + let mut bucket: StoredEntries = if run.cache_enabled { + stored_entries(&cache_path)? } else { - PyDict::new(py).into_any() + StoredEntries::new() }; - let findings: Bound<'py, PyList> = PyList::empty(py); - let model_hits: Bound<'py, PyDict> = cached_findings(&bucket, identities.clone())?; + let mut findings: Vec = Vec::new(); + let mut model_hits: BTreeMap> = + cached_rule_findings(&bucket, &identities)?; let mut misses: Vec<(String, String)> = Vec::new(); for (path, identity) in &identities { - match model_hits.get_item(path)? { - Some(hit) => { - for finding in hit.try_iter()? { - findings.append(finding?)?; - } - } + match model_hits.remove(path) { + Some(hit) => findings.extend(hit), None => misses.push((path.clone(), identity.clone())), } } - let selected_paths: Bound<'py, PySet> = PySet::empty(py)?; - for (path, _) in &misses { - selected_paths.add(project_dir.div(path)?)?; - } + let mut selected_paths: BTreeSet = misses + .iter() + .map(|(path, _)| project_dir.join(path)) + .collect(); let mut file_identities: Vec<(String, String)> = Vec::new(); - let mut project_files: Bound<'py, PyAny> = py.None().into_bound(py); - if run.selected_model_paths.is_none() { - let model_paths: Bound<'py, PySet> = PySet::empty(py)?; + let mut project_files: Option> = None; + let compiled: HashMap = project_sql_expansions(&run.project)?; + if !run.model_selection { + let mut model_paths: HashSet = HashSet::new(); for model in run.project.getattr("models")?.try_iter()? { - model_paths.add(project_dir.div(model?.getattr("relative_path")?)?)?; + model_paths + .insert(project_dir.join(model?.getattr("relative_path")?.extract::()?)); } - project_files = match prepared_project_files(run)? { - Some(files) => files, - None => match prepared_files { + let files: Vec = match prepared_result(run, py) { + Some(Ok(result)) => result.outcome.files.clone(), + Some(Err(error)) if !error.is_instance_of::(py) => { + return Err(error.clone_ref(py)); + } + Some(Err(_)) | None => match prepared_files { Some(files) => files, - None => collected_project_files(project_dir)?, + None => lint_sources(project_dir, None, None)?, }, }; - let expansions: Bound<'py, PyAny> = run.project.getattr("sql_expansions")?; - let mut file_keys: Vec<(Bound<'py, PyAny>, String, Option)> = Vec::new(); - for item in project_files.downcast::()?.iter() { - let (file_path, contents) = item; - if model_paths.contains(&file_path)? { + let mut file_keys: Vec<(PathBuf, String, Option)> = Vec::new(); + for file in &files { + if model_paths.contains(&file.path) { continue; } - let relative_path: String = file_path - .call_method1("relative_to", (project_dir,))? - .call_method0("as_posix")? - .extract()?; - let key: Option = call_with( - &native(py, NativeAttr::SqlFileRuleKey)?, - &[ - ("lint_key", PyString::new(py, lint_key).into_any()), - ("codes", PyList::new(py, run.codes)?.into_any()), - ( - "relative_path", - PyString::new(py, &relative_path).into_any(), - ), - ("contents", contents), - ( - "expanded", - expansions.contains(&file_path)?.into_bound_py_any(py)?, - ), - ], - )? - .extract()?; - file_keys.push((file_path, relative_path, key)); + let Ok(relative) = file.path.strip_prefix(project_dir) else { + return Err(outside_project(&file.path, project_dir)); + }; + let relative_path: String = posix(relative); + let key: Option = file_rule_key( + lint_key, + run.codes, + (&relative_path, &file.contents), + compiled.contains_key(&file.path), + ); + file_keys.push((file.path.clone(), relative_path, key)); } let mut keyed: Vec<(String, String)> = Vec::new(); for (_, path, key) in &file_keys { @@ -367,246 +352,230 @@ fn evaluated_sql_rules<'py>( keyed.push((path.clone(), key.clone())); } } - let file_hits: Bound<'py, PyDict> = cached_findings(&bucket, keyed)?; + let mut file_hits: BTreeMap> = + cached_rule_findings(&bucket, &keyed)?; for (file_path, relative_path, identity) in file_keys { - match file_hits.get_item(&relative_path)? { - Some(hit) => { - for finding in hit.try_iter()? { - findings.append(finding?)?; - } - } + match file_hits.remove(&relative_path) { + Some(hit) => findings.extend(hit), None => { - selected_paths.add(file_path)?; + selected_paths.insert(file_path); if let Some(identity) = identity { file_identities.push((relative_path, identity)); } } } } + project_files = Some(files); } - let violations: Vec> = if selected_paths.is_empty() { + let violations: Vec<(PathBuf, LintViolationRow)> = if selected_paths.is_empty() { Vec::new() } else { - let mut dynamic: Vec> = Vec::new(); + let mut dynamic: HashSet = HashSet::new(); for (_, model) in &models { if output_proven(model)? { - dynamic.push( - project_dir - .div(model.getattr("relative_path")?)? - .call_method0("resolve")?, - ); + dynamic.insert(resolved( + &project_dir.join(model.getattr("relative_path")?.extract::()?), + )); } } - prepared_lint_violations( - run, - &PyFrozenSet::new(py, selected_paths.iter())?, - &PyFrozenSet::new(py, dynamic)?, - &project_files, - )? + prepared_lint_violations(run, (&selected_paths, &dynamic), &compiled, project_files)? }; let model_paths: BTreeSet = models.into_iter().map(|(path, _)| path).collect(); - let evaluated: Vec> = violation_findings(run, &violations, &model_paths)?; - for finding in &evaluated { - findings.append(finding)?; - } + let evaluated: Vec = violation_findings(run, &violations, &model_paths)?; + findings.extend(evaluated.iter().cloned()); let mut written: Vec<(String, String)> = misses.clone(); written.extend(file_identities); if run.cache_enabled && !written.is_empty() { - write_cache(&bucket, (&cache_path, written), &evaluated)?; + let mut by_path: HashMap<&str, Vec<&RuleFinding>> = HashMap::new(); + for finding in &evaluated { + by_path + .entry(finding.path.as_str()) + .or_default() + .push(finding); + } + for (path, identity) in written { + let own: &[&RuleFinding] = by_path.get(path.as_str()).map_or(&[], Vec::as_slice); + let encoded: String = encoded_findings(own)?; + bucket.insert(path, (identity, encoded)); + } + write_rule_cache(&cache_path, &bucket)?; } - let hinted: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::WithDuplicateLiteralHints)?, - &[ - ("findings", PyTuple::new(py, findings.iter())?.into_any()), - ("project_dir", project_dir.clone()), - ], - )?; - Ok((hinted, identities.len() - misses.len(), misses.len())) + let hints: Vec> = + duplicate_literal_hints::(&findings, |path| file_lines(py, project_dir, path))?; + for (finding, hint) in findings.iter_mut().zip(hints) { + if let Some(remediation) = hint { + finding.remediation = remediation; + } + } + Ok((findings, identities.len() - misses.len(), misses.len())) } -/// Store each rewritten path's evaluated findings in `bucket` and replace the cache file. -fn write_cache<'py>( - bucket: &Bound<'py, PyAny>, - (cache_path, written): (&Bound<'py, PyAny>, Vec<(String, String)>), - evaluated: &[Bound<'py, PyAny>], -) -> PyResult<()> { - let py: Python<'py> = bucket.py(); - let encode: Bound<'py, PyAny> = native(py, NativeAttr::EncodeSqlRuleFindings)?; - let mut by_path: HashMap>> = HashMap::new(); - for finding in evaluated { - let path: String = finding - .getattr("path")? - .call_method0("as_posix")? - .extract()?; - by_path.entry(path).or_default().push(finding.clone()); - } - for (path, identity) in written { - let own: &[Bound<'py, PyAny>] = by_path.get(&path).map(Vec::as_slice).unwrap_or_default(); - let encoded: Bound<'py, PyAny> = - call_with(&encode, &[("findings", PyList::new(py, own)?.into_any())])?; - bucket.set_item(path, (identity, encoded))?; - } - let _ = call_with( - &native(py, NativeAttr::WriteSqlRuleCache)?, - &[("path", cache_path.clone()), ("entries", bucket.clone())], - )?; - Ok(()) +/// The authored lines of one finding's file; an unreadable file has none. +fn file_lines(py: Python<'_>, project_dir: &Path, path: &str) -> PyResult> { + let bytes: Vec = match std::fs::read(project_dir.join(path)) { + Ok(bytes) => bytes, + Err(_) => return Ok(Vec::new()), + }; + let text: String = match String::from_utf8(bytes) { + Ok(text) => text, + Err(error) => { + return PyBytes::new(py, error.as_bytes()) + .call_method1("decode", (UTF8,))? + .extract(); + } + }; + Ok(python_splitlines(&text) + .into_iter() + .map(str::to_owned) + .collect()) } -/// Lint violations of `selected_paths`, reusing a matching early SQL lint. -fn prepared_lint_violations<'py>( - run: &SqlRun<'_, 'py>, - selected_paths: &Bound<'py, PyFrozenSet>, - dynamic_output_paths: &Bound<'py, PyFrozenSet>, - source_files: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = run.project_dir.py(); - let run_lint: Bound<'py, PyAny> = native(py, NativeAttr::RunLint)?; - let compiled_expansions: Bound<'py, PyAny> = run.project.getattr("sql_expansions")?; - let enabled_codes: Bound<'py, PyAny> = run.lint_config.getattr("enabled_native_rules")?; - let enabled: Vec = if enabled_codes.is_none() { - Vec::new() - } else { - enabled_codes.extract()? +/// Lint violations of `selected_paths` by file path, reusing a matching early SQL lint. +fn prepared_lint_violations( + run: &SqlRun<'_, '_>, + (selected_paths, dynamic_output_paths): (&BTreeSet, &HashSet), + compiled: &HashMap, + source_files: Option>, +) -> PyResult> { + let py: Python<'_> = run.project.py(); + let enabled: &[String] = run + .settings + .sql + .enabled_rules + .as_deref() + .unwrap_or_default(); + let prepared: Option<&PreparedSqlLint> = run + .prepared() + .filter(|prepared| prepared.has_codes(enabled)); + let Some(prepared) = prepared else { + let context = if run.settings.native_enabled { + Some(lint_expansion_context(run)?) + } else { + None + }; + let files: Vec = + lint_sources(&run.project_dir, Some(selected_paths), source_files)?; + let outcome: LintOutcome = lint( + py, + &LintRequest { + project_dir: &run.project_dir, + settings: &run.settings, + dynamic_output_paths, + compiled_expansions: compiled, + context: context.as_ref(), + skip_unexpandable: false, + stop: None, + }, + files, + )?; + return Ok(path_rows(outcome)); }; - if run.prepared_sql.is_none() || !same_codes(&run.prepared_sql.getattr("codes")?, &enabled)? { - let expansion_context: Bound<'py, PyAny> = - if run.lint_config.getattr("native_enabled")?.is_truthy()? { - lint_expansion_context(run)? - } else { - py.None().into_bound(py) - }; - return call_with( - &run_lint, - &[ - ("project_dir", run.project_dir.clone()), - ("config", run.lint_config.clone()), - ("selected_paths", selected_paths.clone().into_any()), - ("discovered_inputs", run.discovered_inputs.clone()), - ("compiled_expansions", compiled_expansions), - ( - "dynamic_output_paths", - dynamic_output_paths.clone().into_any(), - ), - ("expansion_context", expansion_context), - ("source_files", source_files.clone()), - ], - )? - .getattr("violations")? - .try_iter()? + let result: &EarlyLintResult = match prepared.result(py) { + Ok(result) => result, + Err(error) => return Err(error.clone_ref(py)), + }; + let proof_paths: BTreeSet = selected_paths + .iter() + .filter(|path| dynamic_output_paths.contains(*path)) + .cloned() .collect(); - } - let completed: Bound<'py, PyAny> = - run.prepared_sql.getattr("future")?.call_method0("result")?; - let prepared: Bound<'py, PyAny> = completed.getattr("result")?; - let proof_paths: Bound<'py, PyAny> = - selected_paths.call_method1("__and__", (dynamic_output_paths,))?; - let mut violations: Vec> = Vec::new(); - for violation in prepared.getattr("violations")?.try_iter()? { - let violation: Bound<'py, PyAny> = violation?; - let path: Bound<'py, PyAny> = violation.getattr("file_path")?; - if selected_paths.contains(&path)? && !proof_paths.contains(&path)? { - violations.push(violation); - } - } - if proof_paths.is_truthy()? { - let proven: Bound<'py, PyAny> = call_with( - &run_lint, - &[ - ("project_dir", run.project_dir.clone()), - ("config", run.lint_config.clone()), - ("selected_paths", proof_paths.clone()), - ("discovered_inputs", run.discovered_inputs.clone()), - ("compiled_expansions", compiled_expansions), - ("dynamic_output_paths", proof_paths), - ("expansion_context", completed.getattr("context")?), - ( - "source_files", - imported(py, BUILTINS_MODULE, "dict")? - .call1((prepared.getattr("source_texts")?,))?, - ), - ], + let early: &LintOutcome = &result.outcome; + let mut violations: Vec<(PathBuf, LintViolationRow)> = early + .violations + .iter() + .filter_map(|row| { + let path: &PathBuf = &early.files[row.file].path; + (selected_paths.contains(path) && !proof_paths.contains(path)) + .then(|| (path.clone(), row.clone())) + }) + .collect(); + if !proof_paths.is_empty() { + let proof_set: HashSet = proof_paths.iter().cloned().collect(); + let files: Vec = lint_sources( + &run.project_dir, + Some(&proof_paths), + Some(early.files.clone()), )?; - for violation in proven.getattr("violations")?.try_iter()? { - violations.push(violation?); - } + let proven: LintOutcome = lint( + py, + &LintRequest { + project_dir: &run.project_dir, + settings: &run.settings, + dynamic_output_paths: &proof_set, + compiled_expansions: compiled, + context: Some(&result.context), + skip_unexpandable: false, + stop: None, + }, + files, + )?; + violations.extend(path_rows(proven)); } Ok(violations) } +/// An outcome's violations with their file paths. +fn path_rows(outcome: LintOutcome) -> Vec<(PathBuf, LintViolationRow)> { + let files: Vec = outcome.files; + outcome + .violations + .into_iter() + .map(|row| (files[row.file].path.clone(), row)) + .collect() +} + /// Build lint expansion, reusing the compiler's scope only for the same discovery. -fn lint_expansion_context<'py>(run: &SqlRun<'_, 'py>) -> PyResult> { - let py: Python<'py> = run.project_dir.py(); - let reuse: &Bound<'py, PyAny> = &run.expansion_reuse; - let declaration_scope: Bound<'py, PyAny> = if !reuse.is_none() - && reuse - .getattr("discovered_inputs")? - .is(&run.discovered_inputs) - { - reuse.getattr("declaration_scope")? - } else { - py.None().into_bound(py) - }; - call_with( - &native(py, NativeAttr::BuildLintExpansionContext)?, - &[ - ("project_dir", run.project_dir.clone()), - ("discovered_inputs", run.discovered_inputs.clone()), - ("declaration_scope", declaration_scope), - ], - ) +fn lint_expansion_context( + run: &SqlRun<'_, '_>, +) -> PyResult< + crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs, +> { + let py: Python<'_> = run.project.py(); + let declaration_scope: Option> = run + .early_lint + .as_ref() + .and_then(|early| early.expansion_reuse()) + .filter(|reuse| reuse.discovered_inputs.bind(py).is(&run.discovered_inputs)) + .map(|reuse| reuse.declaration_scope.bind(py).clone()); + lint_expansion_inputs(LintExpansionSources { + project_dir: run.project_dir.clone().into_pyobject(py)?.into_any(), + value_renderer: None, + discovered_inputs: (!run.discovered_inputs.is_none()) + .then(|| run.discovered_inputs.clone()), + declaration_scope, + static_declaration_scope: None, + }) } /// The findings of the selected SQL rules' violations; an engine failure leaves each unevaluated. -fn violation_findings<'py>( - run: &SqlRun<'_, 'py>, - violations: &[Bound<'py, PyAny>], +fn violation_findings( + run: &SqlRun<'_, '_>, + violations: &[(PathBuf, LintViolationRow)], model_paths: &BTreeSet, -) -> PyResult>> { - let py: Python<'py> = run.project_dir.py(); +) -> PyResult> { + let py: Python<'_> = run.project.py(); let engine_error: String = registry::error(py, ErrorId::NativeLint)? .getattr("code")? .extract()?; - let safe_fixes: Bound<'py, PyAny> = PyFrozenSet::new(py, SAFE_RULE_FIX_CODES)?.into_any(); - let resolved: Bound<'py, PyAny> = run.project_dir.call_method0("resolve")?; - let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let resolved_dir: PathBuf = resolved(&run.project_dir); let no_paths: BTreeSet = BTreeSet::new(); - let mut evaluated: Vec> = Vec::new(); - for violation in violations { - let code: String = violation.getattr("code")?.extract()?; - if code == SQL_RULE_SUPPRESSION_CODE || run.codes.contains(&code) { - evaluated.push(lint_finding( - violation, - (&resolved, &safe_fixes), - model_paths, - )?); + let mut evaluated: Vec = Vec::new(); + for (path, violation) in violations { + let code: &str = &violation.code; + if code == SQL_RULE_SUPPRESSION_CODE || run.codes.iter().any(|known| known == code) { + evaluated.push(lint_finding((path, violation), &resolved_dir, model_paths)); } else if code == engine_error { - let finding: Bound<'py, PyAny> = - lint_finding(violation, (&resolved, &safe_fixes), &no_paths)?; - let message: String = violation.getattr("message")?.extract()?; + let finding: RuleFinding = lint_finding((path, violation), &resolved_dir, &no_paths); for rule_code in run.codes { - evaluated.push( - replace.call( - (&finding,), - Some(&keywords( - py, - &[ - ("code", PyString::new(py, rule_code).into_any()), - ( - "message", - PyString::new( - py, - &format!( - "Rule {rule_code} could not be evaluated: {message}" - ), - ) - .into_any(), - ), - ("unevaluated", true.into_bound_py_any(py)?), - ], - )?), - )?, - ); + evaluated.push(RuleFinding { + code: rule_code.clone(), + message: format!( + "Rule {rule_code} could not be evaluated: {}", + violation.message + ), + unevaluated: true, + ..finding.clone() + }); } } } @@ -614,44 +583,35 @@ fn violation_findings<'py>( } /// One lint violation as a finding; fixable when `format --fix` would attempt its edit. -fn lint_finding<'py>( - violation: &Bound<'py, PyAny>, - (resolved_dir, safe_fixes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +fn lint_finding( + (path, violation): (&Path, &LintViolationRow), + resolved_dir: &Path, fixable_paths: &BTreeSet, -) -> PyResult> { - let py: Python<'py> = violation.py(); - let mut path: Bound<'py, PyAny> = violation.getattr("file_path")?; - if path.call_method0("is_absolute")?.is_truthy()? - && path - .call_method1("is_relative_to", (resolved_dir,))? - .is_truthy()? - { - path = path.call_method1("relative_to", (resolved_dir,))?; +) -> RuleFinding { + let path: &Path = if path.is_absolute() { + path.strip_prefix(resolved_dir).unwrap_or(path) + } else { + path + }; + let path: String = posix(path); + let fixable: bool = violation.fix.is_some() + && SAFE_RULE_FIX_CODES.contains(&violation.code.as_str()) + && fixable_paths.contains(&path); + RuleFinding { + code: violation.code.clone(), + path, + line: violation.line, + column: violation.column, + message: violation.message.clone(), + remediation: violation + .remediation + .clone() + .filter(|remediation| !remediation.is_empty()) + .unwrap_or_else(|| DEFAULT_REMEDIATION.to_owned()), + unevaluated: false, + affected_rules: Vec::new(), + fixable, } - let code: Bound<'py, PyAny> = violation.getattr("code")?; - let remediation: Bound<'py, PyAny> = violation.getattr("remediation")?; - let fixable: bool = !violation.getattr("fix")?.is_none() - && safe_fixes.contains(&code)? - && fixable_paths.contains(&path.call_method0("as_posix")?.extract::()?); - call_with( - ®istry::class(py, RecordId::Finding)?, - &[ - ("code", code), - ("path", path), - ("line", violation.getattr("line")?), - ("column", violation.getattr("column")?), - ("message", violation.getattr("message")?), - ( - "remediation", - if remediation.is_truthy()? { - remediation - } else { - PyString::new(py, DEFAULT_REMEDIATION).into_any() - }, - ), - ("fixable", fixable.into_bound_py_any(py)?), - ], - ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 05a24599f4..4464779cb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -6,12 +6,16 @@ use pyo3::exceptions::{PyImportError, PyOSError, PyUnicodeError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyBool, PyCFunction, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; use crate::bindings::_helpers::discovery::scope_snapshot::discover_scope_snapshot; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; +use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; @@ -24,6 +28,7 @@ use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; +use pyo3::IntoPyObjectExt; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; @@ -53,13 +58,11 @@ fn project_adapter<'py>( discover_keywords.set_item("extract_output_column_locations", false)?; let discovered: Bound<'py, PyAny> = registry::native(py, NativeAttr::DiscoverProjectInputs)? .call((), Some(&discover_keywords))?; - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ], - )?; + let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + ) + .and_then(|value| value.into_bound_py_any(py))?; let adapter: Bound<'py, PyAny> = call_with( &host::entry(py, AdapterHost::ResolveAdapter)?, &[("adapter_name", adapter_name)], @@ -75,13 +78,9 @@ fn compiled_index<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let collection_rendering: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveCollectionRendering)?, - &[ - ("project_config", project_config), - ("declaration_override", py.None().into_bound(py)), - ], - )?; + let collection_rendering: Bound<'py, PyAny> = + resolve_effective_collection_rendering(&project_config, &py.None().into_bound(py)) + .and_then(|value| value.into_bound_py_any(py))?; let adapter_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompileAdapterContext)?, &[ @@ -94,26 +93,29 @@ fn compiled_index<'py>( ("sql_lexical_syntax", syntax.clone()), ], )?; - let stage_keywords: Bound<'py, PyDict> = PyDict::new(py); - stage_keywords.set_item("discovered_inputs", discovered)?; - stage_keywords.set_item("adapter_context", adapter_context)?; - stage_keywords.set_item("resolved_connection", PyDict::new(py))?; - stage_keywords.set_item("no_sql_validation", true)?; - stage_keywords.set_item("defer_model_sql_validation", true)?; - stage_keywords.set_item("no_cache", true)?; - let python_stage: Bound<'py, PyAny> = py.import("functools")?.getattr("partial")?.call( - (registry::native(py, NativeAttr::BuildCompileInputs)?,), - Some(&stage_keywords), + let inputs: InputsRequest = InputsRequest { + discovered_inputs: discovered.unbind(), + adapter_context: adapter_context.unbind(), + selected_target: py.None(), + cli_vars: py.None(), + run_id: py.None(), + no_sql_validation: PyBool::new(py, true).to_owned().into_any().unbind(), + defer_model_sql_validation: PyBool::new(py, true).to_owned().into_any().unbind(), + external_sql_reference_resolver: py.None(), + resolved_connection: PyDict::new(py).into_any().unbind(), + no_cache: PyBool::new(py, true).to_owned().into_any().unbind(), + }; + let stage: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + attached_inputs(arguments.py(), &inputs).map(Bound::unbind) + }, )?; - let compile_inputs: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CompileFrontier)?, - &[ - ( - "until", - registry::class(py, RecordId::CompilerStage)?.getattr("COMPILE_PROJECT_INPUTS")?, - ), - ("python_stage", python_stage), - ], + let compile_inputs: Bound<'py, PyAny> = produced( + FrontierStage::CompileProjectInputs, + bridged(stage.as_any())?, )?; scope_index_with_compile_usages(&compile_inputs) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 2b743d310b..00a9cddbbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -14,6 +14,8 @@ use sqlbuild_analysis::semantic_checks::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compiled_project::audit_views::audit_reference_rows; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::models::ProjectCatalog; @@ -162,7 +164,10 @@ fn audit_input( contents: Arc::from(file.getattr("contents")?.extract::()?), bodies, opts_out: opts_out(&block.getattr("header_values")?)?, - relations: relations(&audit.getattr("references")?, reference_class)?, + relations: match audit_reference_rows(audit)? { + Some(rows) => row_relations(rows), + None => relations(&audit.getattr("references")?, reference_class)?, + }, }) } @@ -253,6 +258,29 @@ fn opts_out(values: &Bound<'_, PyAny>) -> PyResult { .is(PyBool::new(values.py(), false))) } +/// `sorted(binding_relation_names(...))` of a native view's reference rows. +fn row_relations(rows: Vec) -> Vec { + let mut names: Vec = rows + .into_iter() + .filter_map(|(kind, name, _, _)| relation_name(&kind, name)) + .collect(); + names.sort_unstable(); + names.dedup(); + names +} + +/// The binding relation a reference of `kind` to `name` reads, or `None` for a UDF call. +fn relation_name(kind: &str, name: String) -> Option { + if kind == UDF_KIND { + return None; + } + Some(if kind == TABLE_FUNCTION_KIND { + format!("{TABLE_FUNCTION_STUB_PREFIX}{name}") + } else { + name + }) +} + /// `sorted(binding_relation_names(...))` of the `CompileSqlReference`s among `references`. fn relations( references: &Bound<'_, PyAny>, @@ -265,15 +293,10 @@ fn relations( continue; } let kind: String = reference.getattr("ref_kind")?.str()?.extract()?; - let name: String = reference.getattr("ref_name")?.extract()?; - if kind == UDF_KIND { - continue; - } - names.push(if kind == TABLE_FUNCTION_KIND { - format!("{TABLE_FUNCTION_STUB_PREFIX}{name}") - } else { - name - }); + names.extend(relation_name( + &kind, + reference.getattr("ref_name")?.extract()?, + )); } names.sort_unstable(); names.dedup(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index b067230958..f36ea6e975 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -4,6 +4,7 @@ mod compiled_tests; pub(crate) mod constants; mod cursor_window; mod fixture_validation; +pub(crate) mod models; pub(crate) mod plan_assembly; pub(crate) mod plan_driver; mod plan_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs new file mode 100644 index 0000000000..75520b0694 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs @@ -0,0 +1,20 @@ +//! Typed requests of SQL-test planning. + +use pyo3::prelude::{Bound, PyAny}; + +/// How one batch of SQL tests is planned. +#[derive(Debug)] +pub(crate) struct SqlTestPlanOptions<'py> { + pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, + pub(crate) render_sql: bool, + pub(crate) include_plan: bool, + pub(crate) artifact: bool, +} + +/// How assembling SQL-test plan results validates fixtures. +#[derive(Debug)] +pub(crate) struct SqlTestAssembly<'py> { + pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, + pub(crate) validate_fixtures: bool, + pub(crate) fixture_planning_context: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs index dca5cd13fe..25b43046fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -4,18 +4,17 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::sql_test_glue::plan_driver::plan_sql_tests_natively; +use crate::bindings::_helpers::sql_test_glue::models::{SqlTestAssembly, SqlTestPlanOptions}; +use crate::bindings::_helpers::sql_test_glue::plan_driver::planned as planned_tests; -#[pyfunction(name = "plan_sql_tests")] +#[pyfunction] #[pyo3(signature = (**kwargs))] -pub(crate) fn assemble_sql_test_plans<'py>( +pub(super) fn plan_sql_tests<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -33,47 +32,54 @@ pub(crate) fn assemble_sql_test_plans<'py>( ), )?; let tests: Vec> = args.value("tests")?.extract()?; - let project = args.value("project")?; - let adapter = args.value("adapter")?; + assemble_sql_test_plans( + &tests, + (&args.value("project")?, &args.value("adapter")?), + &SqlTestAssembly { + sql_analysis_enabled: args.value_or_false("sql_analysis_enabled")?, + validate_fixtures: args.flag("validate_fixtures")?, + fixture_planning_context: args.value("fixture_planning_context")?, + }, + ) +} + +/// Plan results of SQL tests, with fixture and expected-output diagnostics in place of failing plans. +pub(crate) fn assemble_sql_test_plans<'py>( + tests: &[Bound<'py, PyAny>], + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + assembly: &SqlTestAssembly<'py>, +) -> PyResult> { + let py = project.py(); let mut context: Option> = None; - let (planned, diagnostics) = if args.flag("sql_analysis_enabled")? - && args.flag("validate_fixtures")? + let (planned, diagnostics) = if assembly.sql_analysis_enabled.is_truthy()? + && assembly.validate_fixtures { - let given = args.value("fixture_planning_context")?; + let given = assembly.fixture_planning_context.clone(); let shared = if given.is_none() { - Bound::new(py, planning_context(&project, None)?)? + Bound::new(py, planning_context(project, None)?)? } else { given.cast_into::()? }; - let validated = super::plan_fixtures::validate(&tests, (&project, &adapter), shared.get())?; + let validated = super::plan_fixtures::validate(tests, (project, adapter), shared.get())?; context = Some(shared); validated } else { - (tests.clone(), Default::default()) + (tests.to_vec(), Default::default()) }; let plannable: Vec<_> = planned .into_iter() .enumerate() .filter_map(|(index, test)| (!diagnostics.contains_key(&index)).then_some(test)) .collect(); - let plans = plan_sql_tests_natively( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("tests", PyTuple::new(py, plannable)?.into_any()), - ("adapter", adapter), - ( - "sql_analysis_enabled", - args.value_or_false("sql_analysis_enabled")?, - ), - ( - "render_sql", - pyo3::types::PyBool::new(py, false).to_owned().into_any(), - ), - ], - )?), + let plans = planned_tests( + (project, adapter), + &PyTuple::new(py, plannable)?.into_any(), + &SqlTestPlanOptions { + sql_analysis_enabled: assembly.sql_analysis_enabled.clone(), + render_sql: false, + include_plan: true, + artifact: false, + }, )?; let plans: Vec> = plans.extract()?; if plans.len() != tests.len() - diagnostics.len() { @@ -129,8 +135,8 @@ pub(crate) fn assemble_sql_test_plans<'py>( super::plan_entries::entry( test, &plan, - (&project, &models), - &args.value_or_false("sql_analysis_enabled")?, + (project, &models), + &assembly.sql_analysis_enabled, )?, ), ("warnings", plan.getattr("warnings")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index 4e5d1a0190..d10b065dd6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyTuple}; @@ -47,36 +48,32 @@ pub(super) fn chains<'py>( Ok(PyTuple::new(py, result)?.into_any()) } -#[pyfunction] -#[pyo3(signature = (**kwargs))] +/// The model chain of each SQL test, resolved in one project batch and counted. pub(crate) fn resolve_sql_test_model_chains<'py>( - py: Python<'py>, - kwargs: Option>, + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, ) -> PyResult> { - let args = KeywordArguments::read( - py, - "resolve_sql_test_model_chains", - kwargs, - (&["project", "tests"], &[]), - )?; - count_chain_resolution(py)?; - chains(&args.value("project")?, &args.value("tests")?) + count_chain_resolution(project.py())?; + chains(project, tests) } -fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult> { - let project = args.value("project")?; +/// Plans of one SQL-test batch, or their compiled artifacts. +pub(crate) fn planned<'py>( + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + tests: &Bound<'py, PyAny>, + options: &SqlTestPlanOptions<'py>, +) -> PyResult> { let py = project.py(); - let tests = args.value("tests")?; + let artifact = options.artifact; if !tests.is_truthy()? { return Ok(PyTuple::empty(py)); } let start: u64 = imported(py, "time", "perf_counter_ns")? .call0()? .extract()?; - let adapter = args.value("adapter")?; - let model_sql = super::plan_rendering::models(&project, &adapter)?; - let functions = super::plan_rendering::functions(&project, &adapter)?; - let overrides = super::plan_rendering::overrides(&project, &tests, &adapter)?; + let model_sql = super::plan_rendering::models(project, adapter)?; + let functions = super::plan_rendering::functions(project, adapter)?; + let overrides = super::plan_rendering::overrides(project, tests, adapter)?; let request = call_with( ®istry::class(py, RecordId::NativeSqlTestPlanningRequest)?, &[ @@ -85,7 +82,7 @@ fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult(args: &KeywordArguments<'py>, artifact: bool) -> PyResult(args: &KeywordArguments<'py>, artifact: bool) -> PyResult value, Err(error) if error.is_instance_of::(py) => { - return Err(super::plan_errors::planning_error(py, error, &tests)?); + return Err(super::plan_errors::planning_error(py, error, tests)?); } Err(error) => return Err(error), }; @@ -175,7 +169,7 @@ fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult( +pub(super) fn plan_sql_tests_natively<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -194,7 +188,16 @@ pub(crate) fn plan_sql_tests_natively<'py>( &["include_plan"], ), )?; - planned(&args, false) + planned( + (&args.value("project")?, &args.value("adapter")?), + &args.value("tests")?, + &SqlTestPlanOptions { + sql_analysis_enabled: args.value("sql_analysis_enabled")?, + render_sql: args.flag("render_sql")?, + include_plan: !args.passed("include_plan")? || args.flag("include_plan")?, + artifact: false, + }, + ) } #[pyfunction] @@ -212,7 +215,16 @@ pub(crate) fn plan_compiled_sql_test_artifacts<'py>( &[], ), )?; - planned(&args, true) + planned( + (&args.value("project")?, &args.value("adapter")?), + &args.value("tests")?, + &SqlTestPlanOptions { + sql_analysis_enabled: args.value("sql_analysis_enabled")?, + render_sql: true, + include_plan: false, + artifact: true, + }, + ) } #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs index 4e8d3f69c5..4d68666e41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; @@ -92,16 +92,8 @@ pub(super) fn validate<'py>( if indexes.is_empty() { return Ok((tests.to_vec(), Default::default())); } - let chains = resolve_sql_test_model_chains( - py, - Some(keywords( - py, - &[ - ("project", inputs.0.clone()), - ("tests", PyTuple::new(py, model_tests)?.into_any()), - ], - )?), - )?; + let chains = + resolve_sql_test_model_chains(inputs.0, &PyTuple::new(py, model_tests)?.into_any())?; let chains: Vec> = chains.extract()?; if chains.len() != indexes.len() { return Err(pyo3::exceptions::PyValueError::new_err( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index c6df114194..2e414fa4ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -7,17 +7,13 @@ use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests, pl pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( - super::plan_assembly::assemble_sql_test_plans, + super::plan_assembly::plan_sql_tests, module )?)?; module.add_function(wrap_pyfunction!( super::plan_driver::plan_sql_tests_natively, module )?)?; - module.add_function(wrap_pyfunction!( - super::plan_driver::resolve_sql_test_model_chains, - module - )?)?; module.add_function(wrap_pyfunction!( super::plan_driver::plan_compiled_sql_test_artifacts, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index 9840061ae3..2f7cec3bcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -105,9 +105,7 @@ fn model_header_column_locations( } /// A header syntax error's detail, help and file line (when `header_line` is known). -#[pyfunction] -#[pyo3(signature = (error, header, header_line))] -fn model_header_failure( +pub(crate) fn header_failure( error: &str, header: &str, header_line: Option, @@ -124,6 +122,16 @@ fn model_header_failure( } } +/// A header syntax error's detail, help and file line (when `header_line` is known). +#[pyfunction] +fn model_header_failure( + error: &str, + header: &str, + header_line: Option, +) -> (String, Option, Option) { + header_failure(error, header, header_line) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; module.add_function(wrap_pyfunction!(match_model_headers, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index e800067b7b..4650dca5ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; /// `(kind, name, package, call_argument_count)` for one reference. @@ -179,18 +178,17 @@ pub(crate) fn extract_references<'py>( let (references, invalid_calls) = scanned(py, sql, syntax)?; let report: Bound<'py, PyAny> = registry::native(py, NativeAttr::ReportCompileDiagnostic)?; let diagnostic_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompilerDiagnostic)?; - let types = py.import(COMPILE_TYPES_MODULE)?; for call in invalid_calls { let diagnostic: Bound<'py, PyAny> = call_with( &diagnostic_type, &[ ( "phase", - registry::class(types.py(), RecordId::DiagnosticPhase)?.getattr("COMPILE")?, + registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?, ), ( "severity", - registry::class(types.py(), RecordId::DiagnosticSeverity)?.getattr("ERROR")?, + registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?, ), ("code", text(py, REFERENCE_CALL_SYNTAX_CODE)?), ("message", call.getattr("message")?), diff --git a/crates/sqlbuild-rules/src/project_tree/main/mod.rs b/crates/sqlbuild-rules/src/project_tree/main/mod.rs index 4ef4911b9a..db27d63ca8 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/mod.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/mod.rs @@ -1,4 +1,5 @@ pub mod project_path_escapes; +pub mod project_real_path; pub mod project_relative_parts; pub mod project_tree_children; pub mod project_tree_descendants; diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_real_path.rs b/crates/sqlbuild-rules/src/project_tree/main/project_real_path.rs new file mode 100644 index 0000000000..46e2db8ea7 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_real_path.rs @@ -0,0 +1,8 @@ +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::files::real_path; + +/// `path.resolve()` of an existing path without Windows' verbatim prefix, or `path` when it cannot be resolved. +pub fn project_real_path(path: &Path) -> PathBuf { + real_path(path).unwrap_or_else(|_| path.to_path_buf()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bbe387de6a..4504f51c5a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4,7 +4,6 @@ import logging import threading from collections import Counter from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set -from concurrent.futures import Executor, Future from contextlib import AbstractContextManager from contextvars import Token from dataclasses import Field @@ -12,7 +11,6 @@ from datetime import date, datetime from decimal import Decimal from enum import StrEnum from pathlib import Path, PurePath -from threading import Event from types import TracebackType from typing import Any, ClassVar, Literal, Protocol, Self, TextIO, TypedDict, overload @@ -254,11 +252,6 @@ def plan_compiled_sql_test_artifacts( adapter: BaseAdapter, sql_analysis_enabled: bool, ) -> tuple[NativeSqlTestArtifact, ...]: ... -def resolve_sql_test_model_chains( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], -) -> tuple[tuple[str, ...], ...]: ... def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[str, ...]: ... STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] @@ -1110,16 +1103,6 @@ class DirectModelVersionIdentities(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> DirectModelVersionIdentities: ... -class PlannerModelEntryResults(NativeRecord): - """Model plan-entry phase output.""" - - entries: tuple[ModelPlanEntry, ...] - warnings: tuple[PlanWarning, ...] - def __init__( - self, entries: tuple[ModelPlanEntry, ...], warnings: tuple[PlanWarning, ...] = ... - ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerModelEntryResults: ... - class SchemaAction(NativeRecord): """One concrete schema change action to apply to a target table.""" @@ -1335,7 +1318,7 @@ class PlanProviderUsage(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> PlanProviderUsage: ... -class PlannerRuntime(NativeRecord): +class PlannerRuntime: """Resolved execution environment shared by planner phases.""" project: CompiledProjectView @@ -1347,6 +1330,7 @@ class PlannerRuntime(NativeRecord): invocation_time: datetime def __init__( self, + *, project: CompiledProjectView, adapter: BaseAdapter, connection: Any, @@ -1355,7 +1339,6 @@ class PlannerRuntime(NativeRecord): on_progress: Callable[[str], None] | None = ..., invocation_time: datetime = ..., ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerRuntime: ... class PlannerScopeResolution(NativeRecord): """Resolved planner scopes for one plan build.""" @@ -1371,7 +1354,7 @@ class PlannerScopeResolution(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> PlannerScopeResolution: ... -class PlannerWarehouseState(NativeRecord): +class PlannerWarehouseState: """Warehouse snapshot and inspection relations gathered once per plan.""" snapshot: WarehouseSnapshot @@ -1385,6 +1368,7 @@ class PlannerWarehouseState(NativeRecord): migration_fingerprints: MigrationFingerprintCache def __init__( self, + *, snapshot: WarehouseSnapshot, inspection_relations: PlannerRelationsContext, migration_entries: tuple[ModelMigrationPlanEntry, ...] = ..., @@ -1395,7 +1379,6 @@ class PlannerWarehouseState(NativeRecord): old_name_views: tuple[OldNameView, ...] = ..., migration_fingerprints: MigrationFingerprintCache = ..., ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerWarehouseState: ... class ModelMigrationRequest(NativeRecord): """One destination model whose data should come from an earlier relation.""" @@ -1452,50 +1435,6 @@ class MigrationCompatibilityResult(NativeRecord): def __init__(self, status: MigrationCompatibility, findings: tuple[str, ...] = ...) -> None: ... def __replace__(self, **changes: Any) -> MigrationCompatibilityResult: ... -class ModelMigrationPlanning(NativeRecord): - """Migration decisions plus the warehouse snapshot as it will look after moves.""" - - snapshot: WarehouseSnapshot - entries: tuple[ModelMigrationPlanEntry, ...] - warnings: tuple[PlanWarning, ...] - def __init__( - self, - snapshot: WarehouseSnapshot, - entries: tuple[ModelMigrationPlanEntry, ...] = ..., - warnings: tuple[PlanWarning, ...] = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> ModelMigrationPlanning: ... - -class OldNameViewPlanning(NativeRecord): - """Old-name steps for this build plus every compatibility view still recorded.""" - - entries: tuple[OldNameViewPlanEntry, ...] - views: tuple[OldNameView, ...] - warnings: tuple[PlanWarning, ...] - def __init__( - self, - entries: tuple[OldNameViewPlanEntry, ...] = ..., - views: tuple[OldNameView, ...] = ..., - warnings: tuple[PlanWarning, ...] = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> OldNameViewPlanning: ... - -class ColumnMigrationPlanning(NativeRecord): - """Column rename decisions plus the warehouse snapshot as it will look after renames.""" - - snapshot: WarehouseSnapshot - entries: tuple[ColumnMigrationPlanEntry, ...] - hints: tuple[ColumnRenameHint, ...] - warnings: tuple[PlanWarning, ...] - def __init__( - self, - snapshot: WarehouseSnapshot, - entries: tuple[ColumnMigrationPlanEntry, ...] = ..., - hints: tuple[ColumnRenameHint, ...] = ..., - warnings: tuple[PlanWarning, ...] = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> ColumnMigrationPlanning: ... - class PlannerIdentityContext(NativeRecord): """Expected version identities for inspection and stale-warning scopes.""" @@ -1508,41 +1447,6 @@ class PlannerIdentityContext(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> PlannerIdentityContext: ... -class PlannerScopePruningResult(NativeRecord): - """Scope pruning outcome with pruned scopes, actions, and staleness state.""" - - inspection_scope: PlannerScope - execution_scope: PlannerScope - resolved_actions: PlannerResolvedActions - pruned_direct_model_names: tuple[str, ...] - direct_identity_stale_model_names: frozenset[str] - def __init__( - self, - inspection_scope: PlannerScope, - execution_scope: PlannerScope, - resolved_actions: PlannerResolvedActions, - pruned_direct_model_names: tuple[str, ...], - direct_identity_stale_model_names: frozenset[str], - ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerScopePruningResult: ... - -class PlannerChangeReconciliation(NativeRecord): - """Write-hash-honest changes merged back into resolved actions.""" - - changes: PlannerChangeResults - resolved_actions: PlannerResolvedActions - def __init__( - self, changes: PlannerChangeResults, resolved_actions: PlannerResolvedActions - ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerChangeReconciliation: ... - -class PlannerEntryResults(NativeRecord): - """Model plan entries for one plan build.""" - - model_entry_results: PlannerModelEntryResults - def __init__(self, model_entry_results: PlannerModelEntryResults) -> None: ... - def __replace__(self, **changes: Any) -> PlannerEntryResults: ... - class NativeModelInput: @property def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... @@ -2129,18 +2033,6 @@ class NativeCompiledProject: def compiled_model_record(model: CompiledModelView) -> CompiledModel: ... def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: ... -def compute_model_cursor_bounds( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - backfill: BackfillResult, - full_refresh: bool, - start_cursor_override: str | None, - end_cursor_override: str | None, - runtime_cursor_producer_names: frozenset[str], - context: ModelPlanContext, - replaces_relation: bool = False, -) -> CursorBounds | None: ... def compute_cursor_bounds( *, cursor_snapshot: ModelCursorSnapshot, @@ -2396,14 +2288,6 @@ def compile_facts(*, project: CompiledProjectView, project_dir: Path) -> Compile def lint_file_role( *, file_path: Path, project_dir: Path, relative_path: PurePath | None = None ) -> LintFileRole: ... -def lint_body_ranges( - *, - contents: str, - headers: tuple[HeaderSpan, ...], - file_path: Path, - project_dir: Path, - role: LintFileRole | None = None, -) -> tuple[tuple[int, int], ...]: ... def prepare_lint_body( *, role: LintFileRole, @@ -2412,39 +2296,7 @@ def prepare_lint_body( body_range: tuple[int, int], context: SqlExpansionContext, dialect: str, - external_identifiers: tuple[str, ...] = (), - allows_ceremonial_select: bool = False, - allows_dynamic_output_star: bool = False, - compiled_expansion: CompiledSqlExpansion | None = None, ) -> LintBody: ... -def expand_lint_body( - *, - role: LintFileRole, - file_path: Path, - contents: str, - body_range: tuple[int, int], - context: SqlExpansionContext, - dialect: str, - external_identifiers: tuple[str, ...] = (), - allows_ceremonial_select: bool = False, - allows_dynamic_output_star: bool = False, - compiled_expansion: CompiledSqlExpansion | None = None, -) -> PendingLintBody: ... -def expand_file_bodies( - *, - file_path: Path, - contents: str, - headers: tuple[HeaderSpan, ...], - context: SqlExpansionContext, - dialect: str, - project_dir: Path, - relative_path: Path, - dynamic_output_paths: frozenset[Path], - compiled_expansions: dict[Path, CompiledSqlExpansion] | None, -) -> ExpandedLintFile: ... -def prepare_expanded_files( - *, expanded_files: tuple[ExpandedLintFile, ...], dialect: str, skip_unexpandable: bool -) -> PreparedLintFiles: ... def plan_rule_fixes( *, project_dir: Path, @@ -4140,9 +3992,6 @@ def lint_sql_batch_json(request_json: str) -> str: ... def prepare_lint_sql( request: LintPreparationRequest, ) -> tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None: ... -def prepare_lint_sql_batch( - requests: list[LintPreparationRequest], -) -> list[tuple[bool, tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None]]: ... def lint_backtick_identifiers(dialect: str) -> bool: ... def neutralize_lint_interpolation( body: str, dialect: str, for_formatting: bool = False @@ -4863,12 +4712,6 @@ class ProcessResourceTracker: ) -> None: ... def finish(self) -> ProcessResourceUsage: ... -class LintStopScope: - def __enter__(self) -> None: ... - def __exit__(self, *exception: object) -> bool: ... - -def lint_stop_scope(stop: Event, /) -> LintStopScope: ... -def active_lint_stop() -> Event | None: ... def error_code(*, error: BaseException, fallback_code: str) -> str: ... def error_help(error: BaseException, /) -> str | None: ... def error_message(error: BaseException, /) -> str: ... @@ -6683,18 +6526,6 @@ def apply_maximum_start_policy( policy: MaximumStartPolicyInputs, has_start_override: bool, ) -> CursorBounds: ... -def build_plan_output( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - changes: PlannerChangeResults, - model_entry_results: PlannerModelEntryResults, - reload_sources: bool, - extras: PlanOutputExtras | None = None, -) -> PlanOutput: ... def plan_function( *, function: CompiledFunctionView, @@ -6903,18 +6734,6 @@ def build_planner_relations_context( known_source_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, relation_keys: frozenset[CompiledObjectKey] | None = None, ) -> PlannerRelationsContext: ... -def build_plan_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - resolved_actions: PlannerResolvedActions, - cursor_overrides: CursorOverrides | None, - full_refresh: bool, - build_inputs: PlanEntryBuildInputs | None = None, -) -> PlannerModelEntryResults: ... def build_model_materializations( model_entries: tuple[ModelPlanEntry, ...], / ) -> dict[str, str]: ... @@ -7064,22 +6883,9 @@ def evaluate_rules( project_dir: Path, dialect: str, selected_keys: frozenset[CompiledObjectKey] | None = None, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, + early_lint: EarlySqlLint | None = None, no_sql_analysis: bool = False, ) -> RulesRunResult: ... -def prepare_sql_rules( - *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event -) -> PreparedSqlLint | None: ... -def prepare_early_sql_lint( - *, - project_dir: Path, - config: LintConfig, - discovered_inputs: DiscoveredProjectInputs, - static_declaration_scope: DeclarationScopeBuild | None, - compiled_expansions: dict[Path, CompiledSqlExpansion], - stop: Event, -) -> PreparedSqlLintResult: ... def custom_rule_test_evidence( *, rule: Rule, project_dir: Path ) -> list[tuple[str, int, int, str]]: ... @@ -7096,8 +6902,7 @@ def run_rules( project_dir: Path, dialect: str, selected_keys: frozenset[CompiledObjectKey] | None = None, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, + early_lint: EarlySqlLint | None = None, no_sql_analysis: bool = False, ) -> RulesRunResult: ... def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: ... @@ -7115,15 +6920,8 @@ def run_planned_sql_rules( graph: ProjectGraph, discovered_inputs: DiscoveredProjectInputs, dialect: str, - prepared_sql: PreparedSqlLint | None, - expansion_reuse: SqlExpansionReuse | None, + early_lint: EarlySqlLint | None, ) -> SqlRulesEvaluation: ... -def sql_lint_preparation( - *, inputs: CompileInputsView, dialect: str -) -> tuple[Path, tuple[str, ...], LintConfig] | None: ... -def early_lint_expansions( - model_inputs: tuple[ModelInput, ...], -) -> dict[Path, CompiledSqlExpansion]: ... class EarlySqlLint: """Own speculative SQL checks for one compilation invocation.""" @@ -7131,15 +6929,14 @@ class EarlySqlLint: def __init__(self, *, enabled: bool) -> None: ... @property def enabled(self) -> bool: ... - @property - def preparation(self) -> PreparedSqlLint | None: ... - @property - def expansion_reuse(self) -> SqlExpansionReuse | None: ... def __enter__(self) -> EarlySqlLint: ... def __exit__(self, *exception: object) -> None: ... def start(self, *, inputs: CompileInputsView, dialect: str) -> None: ... def stop(self) -> int: ... +class PreparedSqlLint: + """One early SQL lint running on a native thread, for one project and SQL rule codes.""" + def with_duplicate_literal_hints( *, findings: tuple[Finding, ...], project_dir: Path ) -> tuple[Finding, ...]: ... @@ -7170,16 +6967,12 @@ def lint_native_headers( def format_native_headers( *, contents: str, file_path: Path, config: LintConfig ) -> tuple[str, tuple[LintViolation, ...]]: ... -def prepare_native_header_cache(*, files: dict[Path, str]) -> None: ... def safe_format_files( *, files: dict[Path, str], config: LintConfig ) -> tuple[dict[Path, str], dict[Path, str]]: ... def reject_unparseable_header_rewrites( *, updated: dict[Path, str], config: LintConfig, faults: list[LintViolation] ) -> tuple[dict[Path, str], list[LintViolation]]: ... -def external_identifiers_for_headers( - *, contents: str, headers: tuple[HeaderSpan, ...] -) -> tuple[str, ...]: ... def prepare_model_header_tokens(headers: Sequence[str], /) -> None: ... def parse_header_values( *, @@ -7276,14 +7069,6 @@ def migration_destination_fingerprints( ) -> dict[str, tuple[str, ...]]: ... def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... -def plan_old_name_views( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - migration_entries: tuple[ModelMigrationPlanEntry, ...], - column_entries: tuple[ColumnMigrationPlanEntry, ...], -) -> OldNameViewPlanning: ... def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: ... def discover_model_migrations( *, @@ -7306,15 +7091,6 @@ def decide_model_migration( state: MigrationStateInspection, snapshots_config: SnapshotsConfig, ) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: ... -def plan_model_migrations( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - overrides: PlannerOverrides, - deferral: DeferralInputs, - fingerprints: MigrationFingerprintCache, -) -> ModelMigrationPlanning: ... def plan_column_migrations( *, runtime: PlannerRuntime, @@ -7325,7 +7101,7 @@ def plan_column_migrations( overrides: PlannerOverrides, deferral: DeferralInputs, source_columns: Mapping[str, tuple[ColumnInfo, ...]], -) -> ColumnMigrationPlanning: ... +) -> None: ... def old_name_schemas_to_read(*, snapshot: WarehouseSnapshot, needed: set[str]) -> set[str]: ... def old_name_claim_message(*, model_name: str, view: OldNameView, now: datetime) -> str: ... def old_name_claim_errors( @@ -7438,33 +7214,6 @@ def resolve_microbatch_lookback( *, model: CompiledModelView, effective_grain: str | None = None ) -> str | None: ... def extract_seed_columns(seed: CompiledSeedView) -> tuple[ColumnInfo, ...]: ... -def assemble_model_plan_entry( - *, - model: CompiledModelView, - adapter: BaseAdapter, - snapshot: WarehouseSnapshot, - change_result: ChangeDetectionResult, - action: PlanAction, - reason: PlanReason, - materialization_type: MaterializationType, - resolved_sql: str, - sql_analysis_enabled: bool, - pre_hooks: object, - post_hooks: object, - cursor_column: str | None, - cursor_input_relations: tuple[CursorInputRelation, ...], - runtime_owned_cursor_bounds: bool, - cursor_bounds: CursorBounds | None, - microbatch_range: CursorBounds | None, - empty_cursor_inputs: tuple[str, ...], - waiting_on_empty_inputs: bool, - backfill: BackfillResult, - start_cursor_override: str | None, - end_cursor_override: str | None, - future_cursor_config: FutureCursorsConfig | None, - start_cursor_config: StartCursorsConfig | None, - invocation_time: datetime | None, -) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: ... def plan_audit( *, audit: CompiledAuditView, @@ -7836,28 +7585,6 @@ def check_selected_scope_buildability( warehouse: PlannerWarehouseState, deferral: DeferralInputs, ) -> None: ... -def prune_planner_execution_scope( - *, scopes: PlannerScopeResolution, resolved_actions: PlannerResolvedActions -) -> PlannerScopePruningResult: ... -def reconcile_execution_changes( - *, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - pruning: PlannerScopePruningResult, - changes: PlannerChangeResults, -) -> PlannerChangeReconciliation: ... -def build_planner_entry_results( - *, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - overrides: PlannerOverrides, - policies: PlannerPolicies, - deferral: DeferralInputs, - pruning: PlannerScopePruningResult, - reconciliation: PlannerChangeReconciliation, - source_freshness: DirectSourceFreshnessPlanningResult, -) -> PlannerEntryResults: ... def resolve_model_full_refresh(*, model: CompiledModelView, cli_full_refresh: bool) -> bool: ... def effectively_full_refreshed_model_names( *, project: CompiledProjectView, cli_full_refresh: bool @@ -10842,61 +10569,6 @@ class LintBody(NativeRecord): dependency_relations: tuple[tuple[str, str], ...] = ..., ) -> None: ... -class PendingLintBody(NativeRecord): - """One expanded authored body awaiting its native lexical preparation.""" - - file_path: Path - body_start: int - body_end: int - expansion_input: str - expanded: str - expansion_passes: tuple[tuple[ExpansionSpan, ...], ...] - pre_expansion_sites: tuple[InterpolationSite, ...] - external_identifiers: tuple[str, ...] - allows_ceremonial_select: bool - allows_dynamic_output_star: bool - allows_empty_fixture_star: bool - request: NativeLintPreparationRequest - def __init__( - self, - file_path: Path, - body_start: int, - body_end: int, - expansion_input: str, - expanded: str, - expansion_passes: tuple[tuple[ExpansionSpan, ...], ...], - pre_expansion_sites: tuple[InterpolationSite, ...], - external_identifiers: tuple[str, ...], - allows_ceremonial_select: bool, - allows_dynamic_output_star: bool, - allows_empty_fixture_star: bool, - request: NativeLintPreparationRequest, - ) -> None: ... - -class ExpandedLintFile(NativeRecord): - """The expanded bodies of one file, up to the first body that could not be expanded.""" - - file_path: Path - bodies: tuple[PendingLintBody, ...] - failure: ProjectCompileError | None - def __init__( - self, - file_path: Path, - bodies: tuple[PendingLintBody, ...], - failure: ProjectCompileError | None, - ) -> None: ... - -class PreparedLintFiles(NativeRecord): - """Lint bodies of every expandable file, and why the skipped files could not expand.""" - - bodies: tuple[LintBody, ...] - unexpandable: dict[Path, str] - def __init__( - self, - bodies: tuple[LintBody, ...], - unexpandable: dict[Path, str], - ) -> None: ... - class LintEdit(NativeRecord): """One deterministic authored-source edit proposed by a lint rule.""" @@ -13145,41 +12817,6 @@ class SqlTestRulesConfig(NativeRecord): pipeline_directory: str = ..., ) -> None: ... -class PreparedSqlLintResult(NativeRecord): - """Reusable expansion context and completed preliminary SQL checks.""" - - result: LintRunResult - context: SqlExpansionContext - def __init__( - self, - result: LintRunResult, - context: SqlExpansionContext, - ) -> None: ... - -class SqlExpansionReuse(NativeRecord): - """Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.""" - - discovered_inputs: DiscoveredProjectInputs - declaration_scope: DeclarationScopeBuild - def __init__( - self, - discovered_inputs: DiscoveredProjectInputs, - declaration_scope: DeclarationScopeBuild, - ) -> None: ... - -class PreparedSqlLint(NativeRecord): - """Process-local SQL checks started after compiler expansion has completed.""" - - codes: tuple[str, ...] - future: Future[PreparedSqlLintResult] - project_dir: Path | None - def __init__( - self, - codes: tuple[str, ...], - future: Future[PreparedSqlLintResult], - project_dir: Path | None = ..., - ) -> None: ... - class SqlRulesEvaluation(NativeRecord): """SQL rule findings of one run, their cache accounting and elapsed time.""" diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 3e22a919a5..c5c4bc144c 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -290,6 +290,9 @@ "sqlbuild.runtime.event_exporting.main.get_lifecycle_event_sink_definition", "get_lifecycle_event_sink_definition", ), + "ProjectRelation": ("sqlbuild.compiler.references.models", "ProjectRelation"), + "ProjectRelationIndex": ("sqlbuild.compiler.references.models", "ProjectRelationIndex"), + "RelationName": ("sqlbuild.compiler.references.models", "RelationName"), "CostResourceContext": ("sqlbuild.cost.models", "CostResourceContext"), "RunCostSummary": ("sqlbuild.cost.models", "RunCostSummary"), "CostCapability": ("sqlbuild.cost.types", "CostCapability"), @@ -714,6 +717,16 @@ ), "startup_inputs": ("sqlbuild.cli.compile_reuse._helpers.startup_inputs", "startup_inputs"), "CompileReuseRequest": ("sqlbuild.cli.compile_reuse.models", "CompileReuseRequest"), + "AuditEvaluationMode": ("sqlbuild.compiler.auditing.types", "AuditEvaluationMode"), + "AuditOutcome": ("sqlbuild.compiler.auditing.types", "AuditOutcome"), + "AuditSeverity": ("sqlbuild.compiler.auditing.types", "AuditSeverity"), + "ThresholdOperator": ("sqlbuild.compiler.auditing.types", "ThresholdOperator"), + "AttachedAuditTargetKind": ("sqlbuild.compiler.compile.types", "AttachedAuditTargetKind"), + "CompiledResourceType": ("sqlbuild.compiler.compile.types", "CompiledResourceType"), + "FunctionLanguage": ("sqlbuild.compiler.compile.types", "FunctionLanguage"), + "SqlTestMode": ("sqlbuild.compiler.compile.types", "SqlTestMode"), + "CursorAlgebraError": ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"), + "SharedInputError": ("sqlbuild.errors.contracts.exceptions", "SharedInputError"), } __all__ = ("host_targets", *_TARGETS) diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 562d33924e..2a97ba8825 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -83,6 +83,10 @@ from sqlbuild.cli.compile_reuse._helpers.provider_settings import ( ) from sqlbuild.cli.compile_reuse._helpers.startup_inputs import startup_inputs as startup_inputs from sqlbuild.cli.compile_reuse.models import CompileReuseRequest as CompileReuseRequest +from sqlbuild.compiler.auditing.types import AuditEvaluationMode as AuditEvaluationMode +from sqlbuild.compiler.auditing.types import AuditOutcome as AuditOutcome +from sqlbuild.compiler.auditing.types import AuditSeverity as AuditSeverity +from sqlbuild.compiler.auditing.types import ThresholdOperator as ThresholdOperator from sqlbuild.compiler.compile._helpers.diagnostics.python_resources import ( python_resource_rows as python_resource_rows, ) @@ -98,6 +102,10 @@ from sqlbuild.compiler.compile._helpers.render.user_macros import ( from sqlbuild.compiler.compile._helpers.render.user_macros import ( macro_invocation_context as macro_invocation_context, ) +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind as AttachedAuditTargetKind +from sqlbuild.compiler.compile.types import CompiledResourceType as CompiledResourceType +from sqlbuild.compiler.compile.types import FunctionLanguage as FunctionLanguage +from sqlbuild.compiler.compile.types import SqlTestMode as SqlTestMode from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( discover_command_output_sink_functions as discover_command_output_sink_functions, ) @@ -172,10 +180,14 @@ from sqlbuild.compiler.python_nodes.main._hook_identities import ( ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind as PythonNodeKind from sqlbuild.compiler.python_nodes.types import PythonNodeStatus as PythonNodeStatus +from sqlbuild.compiler.references.models import ProjectRelation as ProjectRelation +from sqlbuild.compiler.references.models import ProjectRelationIndex as ProjectRelationIndex +from sqlbuild.compiler.references.models import RelationName as RelationName from sqlbuild.cost.models import CostResourceContext as CostResourceContext from sqlbuild.cost.models import RunCostSummary as RunCostSummary from sqlbuild.cost.types import CostCapability as CostCapability from sqlbuild.cost.types import CostStatus as CostStatus +from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError as CursorAlgebraError from sqlbuild.cursor_algebra.types import BoundSentinel as BoundSentinel from sqlbuild.diagnostics._helpers.process_resources import ( read_process_resources as read_process_resources, @@ -189,6 +201,7 @@ from sqlbuild.diagnostics.main.process_resource_reporting import ( ) from sqlbuild.diagnostics.models import ProcessResourceUsage as ProcessResourceUsage from sqlbuild.errors.contracts.exceptions import ExecutorInputError as ExecutorInputError +from sqlbuild.errors.contracts.exceptions import SharedInputError as SharedInputError from sqlbuild.executor.auditing.main._current_audit_result_projection import ( current_audit_result_projection as current_audit_result_projection, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index dfa12101fa..f618ca5b23 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -575,6 +575,16 @@ class EarlyLintCompileTestCase: expected_diagnostics: tuple[tuple[str, str, int, int], ...] +@dataclass(frozen=True) +class EarlyLintInterruptTestCase: + """A rules-enabled project whose lint-thread macro expansions are slow enough to interrupt.""" + + description: str + model_count: int + macro_seconds: float + expected_max_stop_seconds: float + + @dataclass(frozen=True) class CompileReuseInvalidationTestCase: """One compile input change that must make the stored compile miss.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index fd248cbac8..2fd96350e6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4806,7 +4806,6 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( - (native_module, "assemble_native_project_resources"), (native_module, "infer_native_expression_source_shapes"), (native_module, "analyze_native_model_sql"), (native_module, "assemble_native_sql_tests"), @@ -4970,3 +4969,46 @@ def compile_with_binding_error(project_dir: Path) -> dict[str, Any]: assert result.returncode == 1, result.stdout + result.stderr payload: dict[str, Any] = json.loads(result.stdout) return payload + + +_SLOW_LINT_MACRO: str = ( + "import os\n" + "import threading\n" + "import time\n\n\n" + "def slow_total(amount: str) -> str:\n" + ' """Return the amount, pausing when the background SQL lint expands it."""\n' + " if threading.current_thread() is not threading.main_thread():\n" + " with open(os.environ['SLOW_MACRO_MARKER'], 'a', encoding='utf-8') as marker:\n" + " marker.write('expanding\\n')\n" + " time.sleep(float(os.environ['SLOW_MACRO_SECONDS']))\n" + " return amount\n" +) + + +def slow_lint_model_files(*, model_count: int) -> dict[str, str]: + """A macro used by a model, a function, and macro tests only background SQL lint expands.""" + + return { + "macros/slow.py": _SLOW_LINT_MACRO, + "functions/sql/slow_amount.sql": ( + 'FUNCTION (description "Slow amount", arguments (p_amount INTEGER), returns INTEGER);\n' + 'SELECT @slow_total("p_amount")\n' + ), + "models/slow_total.sql": ( + 'MODEL (description "Slow total");\n' + 'SELECT @slow_total("amount") AS amount FROM (SELECT 5 AS amount) AS input_values\n' + ), + **{ + f"tests/slow_total_{index}.sql": ( + f'TEST (mode macro, name "slow_total_{index}");\n\n' + "WITH\n" + "input_values AS (SELECT 5 AS amount),\n" + "__macro_actual__ AS (\n" + ' SELECT @slow_total("amount") AS amount FROM input_values\n' + "),\n" + "__macro_expected__ AS (SELECT 5 AS amount)\n" + "SELECT 1\n" + ) + for index in range(model_count) + }, + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py index e350906c26..8ca339f922 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py @@ -2,6 +2,11 @@ from __future__ import annotations +import os +import signal +import subprocess +import sys +import time from pathlib import Path from typing import Any @@ -9,10 +14,12 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( EarlyLintCompileTestCase, + EarlyLintInterruptTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( compile_json_payload, diagnostic_location_keys, + slow_lint_model_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project @@ -87,3 +94,53 @@ def test_given_rules_enabled_project_when_compiling_cold_and_warm_then_lint_wait wait_ms: object = payload["compile_timings"]["early_lint_wait_ms"] assert isinstance(wait_ms, int) assert wait_ms >= 0 + + +@pytest.mark.parametrize( + "test_case", + ( + EarlyLintInterruptTestCase( + description="ctrl-c stops after the file being expanded", + model_count=8, + macro_seconds=3.0, + expected_max_stop_seconds=8.0, + ), + ), + ids=lambda case: case.description, +) +def test_given_slow_background_lint_when_interrupted_then_compile_stops_promptly( + test_case: EarlyLintInterruptTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="commerce", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + **slow_lint_model_files(model_count=test_case.model_count), + }, + ) + marker: Path = tmp_path / "slow_macro_marker.txt" + process: subprocess.Popen[str] = subprocess.Popen( + [str(Path(sys.executable).with_name("sqb")), "--project-dir", str(project_dir), "compile"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env={ + **os.environ, + "SLOW_MACRO_MARKER": str(marker), + "SLOW_MACRO_SECONDS": str(test_case.macro_seconds), + }, + ) + deadline: float = time.monotonic() + 60.0 + while not marker.exists() and time.monotonic() < deadline and process.poll() is None: + time.sleep(0.05) + interrupted_at: float = time.monotonic() + process.send_signal(signal.SIGINT) + output: str = process.communicate(timeout=120)[0] + stop_seconds: float = time.monotonic() - interrupted_at + + assert process.returncode in (-signal.SIGINT, 130), output + assert stop_seconds < test_case.expected_max_stop_seconds, output + assert marker.read_text(encoding="utf-8").count("expanding") < test_case.model_count diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 4986554e52..b9b48cb311 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -11,7 +11,6 @@ from sqlbuild._native import ( ContractValidationResult, NativeModelAnalyses, - NativeProjectResources, NativeSqlTestArtifact, NativeSqlTestAssembly, ProjectColumnLineage, @@ -31,7 +30,6 @@ NativeAnalysisSeamTestCase( description="models_contracts_lineage_and_sql_tests", expected_native_returns={ - "assemble_native_project_resources": [ANY], "infer_native_expression_source_shapes": [ANY, ANY, ANY, ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [ANY], @@ -65,7 +63,6 @@ def test_given_project_when_compiling_then_native_seams_answer( planned_counts: list[int] = [] for seams in (native_seams,): assert isinstance(seams["build_project_column_lineage"][0], ProjectColumnLineage) - assert isinstance(seams["assemble_native_project_resources"][0], NativeProjectResources) assert isinstance(seams["analyze_native_model_sql"][0], NativeModelAnalyses) assert seams["analyze_native_model_sql"][0].session is not None assert isinstance(seams["infer_native_expression_source_shapes"][0], tuple) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 90b0042362..95eefdc761 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3,10 +3,11 @@ from __future__ import annotations import json +import sys import time from collections.abc import Callable from pathlib import Path -from threading import Event +from types import SimpleNamespace from typing import Any, cast import pytest @@ -19,10 +20,8 @@ DeclarationScopeBuild, DiscoveredMacroFile, DiscoveredProjectInputs, - LintRunResult, LoadedMacro, SqlLexicalSyntax, - active_lint_stop, rules_bulk_cache_path, ) from sqlbuild._native import run_cli as main @@ -77,7 +76,9 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: _CUSTOM_CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT 1 AS order_id\n' _CUSTOM_CACHE_EDITED_SQL: str = _CUSTOM_CACHE_MODEL_SQL.replace("SELECT 1", "SELECT 2") -_EARLY_LINT_STOP_TIMEOUT_S: float = 30.0 +_EARLY_LINT_SLOW_HOOKS: int = 8 +_EARLY_LINT_SLOW_EXPANSION_S: float = 0.5 +_EARLY_LINT_EXPANSION_DELAYS_S: dict[str, float] = {"orders": 0.0} _EARLY_LINT_FINDINGS: tuple[tuple[str, str, int | None, int | None], ...] = ( ("SQBRSQL005", "models/order_cents.sql", 2, 116), ) @@ -916,14 +917,13 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("cold SQL rules reuse the early file read", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("early file read gives the cached findings", 1, "SQBRSQL004")], ids=lambda case: case.description, ) -def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_file_read( +def test_given_cold_non_model_sql_finding_when_compiling_twice_then_warm_findings_match( test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', @@ -939,25 +939,11 @@ def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_ b"__expected__orders AS (SELECT order_id FROM __ref__orders LIMIT 1)\r\n" b"SELECT 1\r\n" ) - collections: list[frozenset[Path] | None] = [] - original_collect: Callable[..., dict[Path, str]] = native_module.collect_lint_project_files - - def counting_collect( - *, project_dir: Path, selected_paths: frozenset[Path] | None - ) -> dict[Path, str]: - collections.append(selected_paths) - return original_collect(project_dir=project_dir, selected_paths=selected_paths) - - monkeypatch.setattr(native_module, "collect_lint_project_files", counting_collect) - cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) - cold_collections: int = len(collections) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) assert f"tests/unit/test_orders.sql:{test_case.expected_code}" in cold assert warm == cold - assert cold_collections == 0 - assert collections == [None] @pytest.mark.parametrize( @@ -1017,7 +1003,7 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ @pytest.mark.parametrize( "test_case", - [RulePassIntegrationTestCase("graph error stops the unused early lint", 1, ("B002",))], + [RulePassIntegrationTestCase("graph error stops the unused early lint", 1, ("B002", "S024"))], ids=lambda case: case.description, ) def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_awaited( @@ -1033,7 +1019,8 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ models_dir: Path = tmp_path / "models" models_dir.mkdir() (models_dir / "orders.sql").write_text( - 'MODEL (description "Orders");\nSELECT 1 AS order_id\n', encoding="utf-8" + 'MODEL (description "Orders");\nSELECT @slow_total("orders") AS order_id\n', + encoding="utf-8", ) (models_dir / "order_totals.sql").write_text( 'MODEL (description "Order totals");\n' @@ -1041,17 +1028,28 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ 'SELECT order_id, missing_column FROM __ref("orders")\n', encoding="utf-8", ) - lint_started: Event = Event() - stop_observed: list[bool] = [] - original_run_lint: Callable[..., LintRunResult] = native_module.run_lint + macros_dir: Path = tmp_path / "macros" + macros_dir.mkdir() + (tmp_path / "hooks").mkdir() + (macros_dir / "slow_total.py").write_text( + "import orders_lint_probe\n\n\n" + "def slow_total(name: str) -> str:\n" + ' """Record the expansion and take a while."""\n' + " return orders_lint_probe.expanded(name)\n", + encoding="utf-8", + ) + for index in range(_EARLY_LINT_SLOW_HOOKS): + (tmp_path / "hooks" / f"total_{index:02d}.sql").write_text( + f'SELECT @slow_total("total_{index:02d}") AS order_total\n', encoding="utf-8" + ) + expanded: list[str] = [] - def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: - lint_started.set() - stop: Event | None = active_lint_stop() - stop_observed.append(stop is not None and stop.wait(timeout=_EARLY_LINT_STOP_TIMEOUT_S)) - return original_run_lint(**kwargs) + def expand_slowly(name: str) -> str: + expanded.append(name) + time.sleep(_EARLY_LINT_EXPANSION_DELAYS_S.get(name, _EARLY_LINT_SLOW_EXPANSION_S)) + return "1" - monkeypatch.setattr(native_module, "run_lint", stop_awaiting_run_lint) + monkeypatch.setitem(sys.modules, "orders_lint_probe", SimpleNamespace(expanded=expand_slowly)) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) @@ -1061,8 +1059,7 @@ def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: assert tuple(diagnostic["code"] for diagnostic in diagnostics) == ( test_case.expected_diagnostics ) - assert lint_started.is_set() - assert stop_observed == [True] + assert len(set(expanded) - set(_EARLY_LINT_EXPANSION_DELAYS_S)) < _EARLY_LINT_SLOW_HOOKS @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py index 9be0333b4d..d32e7d0435 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py @@ -3,11 +3,11 @@ @dataclass(frozen=True) class SqlRulePublicationTestCase: - """Increasing path counts and the linear conversion budget.""" + """Paths whose lint findings are published to the SQL rule cache within a time limit.""" description: str sizes: tuple[int, ...] - expected_counts: list[tuple[int, int]] + expected_max_seconds: float @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py index 6fb27cb42a..00059e1f38 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py @@ -1,7 +1,7 @@ import json import shutil from pathlib import Path -from typing import Any, ClassVar, cast +from typing import Any, cast import pytest @@ -14,16 +14,6 @@ type FindingKey = tuple[str, str, int | None, int | None, str] -class CountingRulePath(Path): - """Count finding path conversions across the real native cache publication.""" - - conversions: ClassVar[int] = 0 - - def as_posix(self) -> str: - CountingRulePath.conversions += 1 - return super().as_posix() - - def write_rules_fixture(*, project_dir: Path) -> None: """Copy the neutral fixture and select every built-in rule.""" diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index 91ab02ce99..033d1dc48d 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -1,7 +1,6 @@ """Bound native SQL rule cache publication work while preserving persisted findings.""" import json -from concurrent.futures import Future from pathlib import Path from time import monotonic from types import SimpleNamespace @@ -14,28 +13,34 @@ from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( SqlRulePublicationTestCase, ) -from tests.integration.src.sqlbuild.rule_engine.native_rows.helpers import CountingRulePath + +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n' @pytest.mark.parametrize( "test_case", - [ - SqlRulePublicationTestCase( - "doubling paths and findings", (4, 8, 16), [(4, 12), (8, 24), (16, 48)] - ) - ], + [SqlRulePublicationTestCase("doubling paths and findings", (4, 8, 16), 30.0)], ids=lambda case: case.description, ) -def test_given_increasing_findings_when_publishing_cache_then_work_is_linear_and_order_preserved( - test_case: SqlRulePublicationTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_given_increasing_findings_when_publishing_cache_then_findings_are_stored_by_path( + test_case: SqlRulePublicationTestCase, tmp_path: Path ) -> None: - """Exercise planning, finding conversion, encoding and disk publication through the binding.""" - monkeypatch.setattr(native, "compiled_relation_keys", lambda project: {}) - counts: list[tuple[int, int]] = [] + """Lint each model natively, then publish its findings to the cache file in finding order.""" started: float = monotonic() for size in test_case.sizes: project_dir: Path = tmp_path / str(size) + (project_dir / "models").mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") paths: list[Path] = [Path(f"models/orders_{index}.sql") for index in range(size)] + for index, path in enumerate(paths[:-1]): + (project_dir / path).write_text( + f'MODEL (description "Orders {index}");\nSELECT 1 AS order_id LIMIT 1\n', + encoding="utf-8", + ) + (project_dir / paths[-1]).write_text( + 'MODEL (description "Orders without a limit");\nSELECT 1 AS order_id\n', + encoding="utf-8", + ) models: list[SimpleNamespace] = [ SimpleNamespace( relative_path=path, @@ -43,11 +48,16 @@ def test_given_increasing_findings_when_publishing_cache_then_work_is_linear_and query_sql="SELECT 1 AS order_id", config=SimpleNamespace(values={}), dynamic_column_contract=None, + relation_key_row=("model", path.stem, "", "", "", False, None, [], []), ) for path in paths ] project: SimpleNamespace = SimpleNamespace( - settings=SimpleNamespace(sql_analysis=True), models=models, sql_expansions={} + settings=SimpleNamespace(sql_analysis=True), + models=models, + seeds=(), + sources=(), + sql_expansions={}, ) plan: SimpleNamespace = SimpleNamespace( project_dir=project_dir, @@ -58,46 +68,26 @@ def test_given_increasing_findings_when_publishing_cache_then_work_is_linear_and thresholds={}, allow_exceptions=True, cache=SimpleNamespace(enabled=True) ), ) - violations: list[SimpleNamespace] = [ - SimpleNamespace( - file_path=CountingRulePath(project_dir / paths[index % (size - 1)]), - code="SQBRSQL004", - line=index + 1, - column=1, - message=f"Finding {index}", - remediation="Use an explicit projection.", - fix=None, - ) - for index in range(size) - ] - future: Future[Any] = Future() - future.set_result(SimpleNamespace(result=SimpleNamespace(violations=violations))) - CountingRulePath.conversions = 0 result: SqlRulesEvaluation = native.run_planned_sql_rules( plan=cast(Any, plan), graph=cast(Any, SimpleNamespace(project=project)), dialect="duckdb", discovered_inputs=cast(Any, None), - prepared_sql=cast(Any, SimpleNamespace(codes=("SQBRSQL004",), future=future)), - expansion_reuse=None, + early_lint=None, ) - conversions: int = CountingRulePath.conversions assert result.cache_misses == size - assert len(result.findings) == size + assert len(result.findings) == size - 1 stored: dict[str, tuple[str, str]] = native.load_sql_rule_cache( path=native.rules_bulk_cache_path(project_dir=project_dir, file_name="sql.json") ) assert set(stored) == {path.as_posix() for path in paths} - expected: list[list[Finding]] = [[] for _ in paths] - for position, finding in enumerate(result.findings): - expected[position % (size - 1)].append(finding) - for path, own in zip(paths, expected, strict=True): - assert stored[path.as_posix()][1] == native.encode_sql_rule_findings(findings=own) + by_path: dict[str, list[Finding]] = {path.as_posix(): [] for path in paths} + for finding in result.findings: + by_path[finding.path.as_posix()].append(finding) + for path, own in by_path.items(): + assert stored[path][1] == native.encode_sql_rule_findings(findings=own) assert json.loads(stored[paths[-1].as_posix()][1]) == [] - # One conversion each for grouping, encoding, and duplicate-literal hints. - counts.append((size, conversions)) - assert counts == test_case.expected_counts - assert monotonic() - started < 30 + assert monotonic() - started < test_case.expected_max_seconds if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index 02f7d206a5..69a15cd0b7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -424,8 +424,8 @@ def test_given_changes_only_current_model_when_planning_type_then_selection_stil declared=True, ), ) - warehouse = replace( - warehouse, + warehouse = PlannerWarehouseState( + inspection_relations=warehouse.inspection_relations, snapshot=replace( warehouse.snapshot, fingerprints=WarehouseFingerprints( @@ -753,8 +753,11 @@ def test_given_missing_bigquery_dataset_when_planning_then_defers_retention_unti existing_relations={}, config_values={}, ) - runtime: PlannerRuntime = replace( - runtime, + runtime: PlannerRuntime = PlannerRuntime( + project=runtime.project, + adapter=runtime.adapter, + connection=runtime.connection, + invocation_time=runtime.invocation_time, project_config=ProjectConfig( name="test", adapter=BuiltinAdapter.BIGQUERY.value, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index 43b1eff0ca..51fcfc51c6 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -219,7 +219,7 @@ class ManyViolationLocationTestCase: @dataclass(frozen=True) class BatchedPreparationTestCase: - """A project whose lint is compared between batched and per-body native preparation.""" + """A project whose bodies of every kind are prepared natively in one batch.""" description: str files: dict[str, str] @@ -227,13 +227,20 @@ class BatchedPreparationTestCase: @dataclass(frozen=True) -class BatchedPreparationFailureTestCase: - """A native preparation failure and an unexpandable file, in a given file order.""" +class UnexpandableFailureTestCase: + """Unexpandable files linted without skipping, and the file whose failure is raised.""" description: str - unexpandable_file: str - skip_unexpandable: bool - expected_error: type[Exception] | None + expected_error_file: str + + +@dataclass(frozen=True) +class UnexpandableSkipTestCase: + """Unexpandable files linted with skipping, and the files reported as skipped.""" + + description: str + expected_unexpandable: tuple[str, ...] + expected_codes: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index fedae56d34..f54c0e2632 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -4,7 +4,6 @@ from dataclasses import astuple from pathlib import Path -from typing import TYPE_CHECKING from sqlbuild._native import ( HeaderSpan, @@ -19,9 +18,6 @@ sql_body_ranges, ) -if TYPE_CHECKING: - from sqlbuild._native import NativeLintPreparationRequest, NativePreparedSql - def lint_bodies_for(*, file_path: Path, contents: str) -> tuple[LintBody, ...]: """Build unexpanded lint bodies for every SQL body in the contents.""" @@ -93,20 +89,6 @@ def write_lint_project(*, root: Path, files: dict[str, str]) -> None: path.write_text(contents, encoding="utf-8") -def prepare_nothing_in_batch( - requests: list[NativeLintPreparationRequest], -) -> list[tuple[bool, NativePreparedSql | None]]: - """Report every body as unprepared so each one takes the per-body native call.""" - - return [(False, None) for _request in requests] - - -def refuse_preparation(request: NativeLintPreparationRequest) -> NativePreparedSql | None: - """Fail the per-body native preparation of any body.""" - - raise ValueError(f"cannot prepare {request['dialect']} body") - - def write_orders_lint_project(*, project_dir: Path, model_bytes: bytes) -> Path: """Write a DuckDB project with one model and one SQL test; return the model path.""" diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py index 754ff94dc8..5912f01143 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py @@ -1,4 +1,4 @@ -"""Lint bodies are prepared natively in one batch with per-body results and failure order.""" +"""Lint bodies are prepared natively in one batch; unexpandable files fail or skip in file order.""" from __future__ import annotations @@ -6,17 +6,13 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import LintConfig, LintRunResult, ProjectCompileError, run_lint from tests.unit.src.sqlbuild.lint._helpers._test_types import ( - BatchedPreparationFailureTestCase, BatchedPreparationTestCase, + UnexpandableFailureTestCase, + UnexpandableSkipTestCase, ) -from tests.unit.src.sqlbuild.lint._helpers.helpers import ( - prepare_nothing_in_batch, - refuse_preparation, - write_lint_project, -) +from tests.unit.src.sqlbuild.lint._helpers.helpers import write_lint_project _ORDERS_MODEL: str = ( 'MODEL (description "Orders");\n' @@ -30,6 +26,13 @@ "__expected__orders AS (SELECT 1 AS customer_id)\nSELECT 1\n" ) _NOT_NULL_AUDIT: str = "AUDIT ();\nSELECT 1 FROM @relation WHERE @column IS NULL\n" +_BROKEN_MODEL: str = 'MODEL (description "Broken");\nSELECT @unknown()\n' +_UNEXPANDABLE_FILES: dict[str, str] = { + "models/customers.sql": _CUSTOMERS_MODEL, + "models/orders.sql": _ORDERS_MODEL, + "models/aa_broken.sql": _BROKEN_MODEL, + "models/zz_broken.sql": _BROKEN_MODEL, +} @pytest.mark.parametrize( @@ -48,66 +51,65 @@ ], ids=lambda case: case.description, ) -def test_given_project_when_linting_with_batched_preparation_then_result_matches_per_body( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: BatchedPreparationTestCase +def test_given_project_when_linting_then_bodies_of_every_kind_are_prepared_in_one_batch( + tmp_path: Path, test_case: BatchedPreparationTestCase ) -> None: write_lint_project(root=tmp_path, files=test_case.files) config: LintConfig = LintConfig(dialect="duckdb") - batched: LintRunResult = run_lint(project_dir=tmp_path, config=config) - monkeypatch.setattr(native_module, "prepare_lint_sql_batch", prepare_nothing_in_batch) - per_body: LintRunResult = run_lint(project_dir=tmp_path, config=config) + first: LintRunResult = run_lint(project_dir=tmp_path, config=config) + second: LintRunResult = run_lint(project_dir=tmp_path, config=config) - assert batched == per_body - assert tuple(violation.code for violation in batched.violations) == test_case.expected_codes + assert first == second + assert tuple(violation.code for violation in first.violations) == test_case.expected_codes @pytest.mark.parametrize( "test_case", [ - BatchedPreparationFailureTestCase( - description="native failure before a later unexpandable file", - unexpandable_file="models/zz_broken.sql", - skip_unexpandable=False, - expected_error=ValueError, - ), - BatchedPreparationFailureTestCase( - description="unexpandable file before a later native failure", - unexpandable_file="models/aa_broken.sql", - skip_unexpandable=False, - expected_error=ProjectCompileError, - ), - BatchedPreparationFailureTestCase( - description="skipped unexpandable file before a later native failure", - unexpandable_file="models/aa_broken.sql", - skip_unexpandable=True, - expected_error=ValueError, - ), + UnexpandableFailureTestCase( + description="first unexpandable file in path order is raised", + expected_error_file="models/aa_broken.sql", + ) ], ids=lambda case: case.description, ) -def test_given_native_and_expansion_failures_when_linting_then_first_in_file_order_is_raised( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: BatchedPreparationFailureTestCase +def test_given_unexpandable_files_when_linting_then_first_in_file_order_is_raised( + tmp_path: Path, test_case: UnexpandableFailureTestCase ) -> None: - write_lint_project( - root=tmp_path, - files={ - "models/customers.sql": _CUSTOMERS_MODEL, - "models/orders.sql": _ORDERS_MODEL, - test_case.unexpandable_file: 'MODEL (description "Broken");\nSELECT @unknown()\n', - }, - ) - monkeypatch.setattr(native_module, "prepare_lint_sql_batch", prepare_nothing_in_batch) - monkeypatch.setattr(native_module, "prepare_lint_sql", refuse_preparation) - - with pytest.raises(Exception) as raised: - _ = run_lint( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - skip_unexpandable=test_case.skip_unexpandable, + write_lint_project(root=tmp_path, files=_UNEXPANDABLE_FILES) + + with pytest.raises(ProjectCompileError) as raised: + _ = run_lint(project_dir=tmp_path, config=LintConfig(dialect="duckdb")) + + assert str(raised.value).startswith(str(tmp_path / test_case.expected_error_file)) + + +@pytest.mark.parametrize( + "test_case", + [ + UnexpandableSkipTestCase( + description="skipped unexpandable files are reported and the rest is linted", + expected_unexpandable=("models/aa_broken.sql", "models/zz_broken.sql"), + expected_codes=("SQBRSQL001",), ) + ], + ids=lambda case: case.description, +) +def test_given_unexpandable_files_when_linting_with_skipping_then_they_are_reported( + tmp_path: Path, test_case: UnexpandableSkipTestCase +) -> None: + write_lint_project(root=tmp_path, files=_UNEXPANDABLE_FILES) + + result: LintRunResult = run_lint( + project_dir=tmp_path, config=LintConfig(dialect="duckdb"), skip_unexpandable=True + ) - assert type(raised.value) is test_case.expected_error + assert ( + tuple(path.relative_to(tmp_path).as_posix() for path in result.unexpandable) + == test_case.expected_unexpandable + ) + assert tuple(violation.code for violation in result.violations) == test_case.expected_codes if __name__ == "__main__":