From 5ec53d0f6c645c3449e239c72970aa67f7279147 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:51:24 +0100 Subject: [PATCH 01/12] refactor(compile): attach compile inputs through native calls and render constants natively - Compile-input attachment and assembly call their native stages directly instead of through _native lookups, functools.partial and keyword dictionaries; the frontier produces native stages without Python stage callables. - Declaration expansion renders constants with a native ConstantRendering instead of a per-model pathlib.Path, functools.partial and Python callback host. - Lane module strings for compile, auditing, reference and shared-error types resolve through the host table. - SQL test classification runs without the GIL on a bounded worker pool. --- .../compiler/_helpers/sql_tests/extraction.rs | 30 +- .../bindings/_helpers/auditing/measurement.rs | 8 +- .../_helpers/authored_values/project_vars.rs | 9 +- .../_helpers/authored_values/python_nodes.rs | 6 +- .../compile_attachment/model_inputs.rs | 62 ++-- .../sql_expansion_context.rs | 51 ++-- .../_helpers/compile_attachment/target.rs | 35 +-- .../_helpers/compile_driver/assembly.rs | 44 ++- .../compile_driver/assembly_checks.rs | 19 +- .../_helpers/compile_driver/assembly_facts.rs | 40 +-- .../compile_driver/assembly_project.rs | 14 +- .../compile_driver/config_validation.rs | 14 +- .../_helpers/compile_driver/discovery.rs | 39 +-- .../_helpers/compile_driver/inputs.rs | 274 ++++++++++-------- .../_helpers/compile_driver/project.rs | 160 +++++----- .../compile_driver/target_defaults.rs | 47 +-- .../_helpers/compile_driver/targets.rs | 114 ++++---- .../_helpers/compile_records/constants.rs | 23 +- .../_helpers/compiled_project/attachment.rs | 30 +- .../_helpers/compiled_project/loop_classes.rs | 15 +- .../_helpers/compiled_project/model_loop.rs | 60 ++-- .../compiled_project/model_loop_hooks.rs | 14 +- .../_helpers/compiled_project/project.rs | 16 +- .../_helpers/compiled_project/source_loop.rs | 13 +- .../compiled_project/sql_test_loop.rs | 7 +- .../bindings/_helpers/cursor/conversion.rs | 6 +- .../_helpers/discovery/declaration_records.rs | 11 +- .../bindings/_helpers/discovery/guidance.rs | 5 +- .../_helpers/discovery/imported_modules.rs | 11 +- .../discovery/imported_python_nodes.rs | 10 +- .../bindings/_helpers/discovery/locations.rs | 2 +- .../_helpers/discovery/python_hook_files.rs | 2 +- .../_helpers/discovery/sql_test_blocks.rs | 8 +- .../_helpers/frontier/compile_frontier.rs | 44 ++- .../_helpers/hand_built/compiled_model.rs | 10 +- .../_helpers/model_config/contract_classes.rs | 7 +- .../bindings/_helpers/native_host/executor.rs | 18 ++ .../bindings/_helpers/native_host/facade.rs | 60 ++++ .../project_assembly/native_resources.rs | 2 +- .../project_assembly/syntax_checks.rs | 4 +- .../bindings/_helpers/references/relations.rs | 12 +- .../_helpers/relation_naming/fitting.rs | 7 +- .../bindings/_helpers/render/authored_sql.rs | 21 +- .../_helpers/render/declaration_expansion.rs | 13 +- .../_helpers/render/declaration_indexes.rs | 6 +- .../_helpers/render/declaration_values.rs | 51 +++- .../bindings/_helpers/scopes/scope_loading.rs | 70 ++--- .../_helpers/sqltext/sql_references.rs | 6 +- src/sqlbuild/_native_host.py | 13 + src/sqlbuild/_native_host.pyi | 13 + 50 files changed, 811 insertions(+), 745 deletions(-) diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs index ced6725397..f98885a529 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs @@ -2,6 +2,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; use crate::compiler::_helpers::sql_tests::authored_macro_calls::calls_macro; @@ -15,6 +16,10 @@ use sqlbuild_sqltext::sql_scan::main::dialect_matching_paren::dialect_matching_p use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed}; +/// Batches smaller than this are extracted on the calling thread. +const PARALLEL_EXTRACTION_MINIMUM: usize = 64; +const EXTRACTION_WORKERS: usize = 4; +const EXTRACTION_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; const MODEL_PREFIXES: [&str; 8] = [ "__macro__", "__ref__", @@ -134,10 +139,29 @@ pub(crate) fn extract_tests( tests: &[SqlTestRequest], syntax: &LexicalSyntax, ) -> Result, (usize, Failure)> { - tests - .iter() + let extracted: Vec> = + if tests.len() < PARALLEL_EXTRACTION_MINIMUM { + tests + .iter() + .map(|test| extract_test(test, syntax)) + .collect() + } else { + let pool: rayon::ThreadPool = rayon::ThreadPoolBuilder::new() + .num_threads(EXTRACTION_WORKERS) + .stack_size(EXTRACTION_WORKER_STACK_BYTES) + .build() + .map_err(|error| (0, Failure::from(error.to_string())))?; + pool.install(|| { + tests + .par_iter() + .map(|test| extract_test(test, syntax)) + .collect() + }) + }; + extracted + .into_iter() .enumerate() - .map(|(index, test)| extract_test(test, syntax).map_err(|failure| (index, failure))) + .map(|(index, result)| result.map_err(|failure| (index, failure))) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs b/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs index cd764d371c..fc6ded71a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs @@ -8,11 +8,11 @@ use sqlbuild_analysis::auditing::main::evaluate_measurement::evaluate_measuremen use sqlbuild_analysis::auditing::models::{MeasurementOutcome, Thresholds}; use crate::bindings::_helpers::auditing::identity::thresholds as read_thresholds; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; - fn measurement_error(py: Python<'_>, message: &str) -> PyErr { match registry::error(py, ErrorId::MeasurementAudit).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), @@ -40,9 +40,7 @@ fn evaluate_measurement<'py>( MeasurementOutcome::Error => "ERROR", MeasurementOutcome::Insufficient => "INSUFFICIENT", }; - py.import(AUDITING_TYPES_MODULE)? - .getattr("AuditOutcome")? - .getattr(member) + host::entry(py, FacadeHost::AuditOutcome)?.getattr(member) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs index 9ecb1ae789..727dc3ce7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/project_vars.rs @@ -1,12 +1,13 @@ //! Project variable values rendered as the text SQL and config interpolation inserts. +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyList, PyString, PyTypeMethods}; use pyo3::{pyfunction, wrap_pyfunction}; -const ERRORS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; const JSON_MODULE: &str = "json"; /// Characters of a structured value's JSON preview before it is cut short. const PREVIEW_LIMIT: usize = 120; @@ -14,11 +15,7 @@ const PREVIEW_ELLIPSIS: &str = "..."; /// `SharedInputError(message)`. fn shared_input_error(py: Python<'_>, message: String) -> PyErr { - match py - .import(ERRORS_MODULE) - .and_then(|module| module.getattr("SharedInputError")) - .and_then(|class| class.call1((message,))) - { + match host::entry(py, FacadeHost::SharedInputError).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), Err(error) => error, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs index 55bdbb7ed1..ddaa1338cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/authored_values/python_nodes.rs @@ -13,11 +13,10 @@ use sqlbuild_core::identifier_text::main::is_lowercase_identifier::is_lowercase_ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -const ERRORS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; -const SHARED_INPUT_ERROR: &str = "SharedInputError"; const RANDOM_MODULE: &str = "random"; const MAX_ATTEMPTS_TOO_LOW: &str = "RetryPolicy max_attempts must be at least 1"; const INITIAL_DELAY_NEGATIVE: &str = "RetryPolicy initial_delay_seconds must be non-negative"; @@ -36,8 +35,7 @@ const HOOK_READS_INVALID: &str = /// `SharedInputError(message)`. fn shared_input_error(py: Python<'_>, message: &str) -> PyErr { - match imported(py, ERRORS_MODULE, SHARED_INPUT_ERROR).and_then(|class| class.call1((message,))) - { + match host::entry(py, FacadeHost::SharedInputError).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), Err(error) => error, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 96063ddc63..2f0c7060a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use sqlbuild_analysis::assembly::project::main::sql_analysis_opt_out_help::sql_analysis_opt_out_help; use crate::bindings::_helpers::attachments::cursor_intrinsics::reserved_cursor_markers; @@ -11,9 +11,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; -use crate::bindings::_helpers::render::declaration_values::constant_declaration_sql; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::macro_expansion_result::macro_expansion_classes; use crate::bindings::_helpers::native_host::host; @@ -51,31 +55,15 @@ fn opt_out_help(py: Python<'_>, model_owned: bool) -> PyResult> Ok(PyString::new(py, &sql_analysis_opt_out_help(model_owned)).into_any()) } -/// The host the loop calls back into: `constant_text(*, declaration, file_path)`. -fn loop_host<'py>(context: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = context.py(); - let context: Py = context.clone().unbind(); - let constant_text = move |arguments: &Bound<'_, PyTuple>, - keywords: Option<&Bound<'_, PyDict>>| - -> PyResult> { - let py: Python<'_> = arguments.py(); - let context: &Bound<'_, PyAny> = context.bind(py); - let passed: Bound<'_, PyDict> = - keywords.map_or_else(|| PyDict::new(py), |keywords| keywords.clone()); - let file_path: Bound<'_, PyAny> = passed.as_any().get_item("file_path")?; - let rendered: String = constant_declaration_sql( - &passed.as_any().get_item("declaration")?, - &context.getattr("value_renderer")?, - (&context.getattr("collection_rendering")?, Some(&file_path)), - )?; - Ok(PyString::new(py, &rendered).into_any().unbind()) - }; - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item( - "constant_text", - PyCFunction::new_closure(py, None, None, constant_text)?, - )?; - imported(py, "types", "SimpleNamespace")?.call((), Some(&keywords)) +/// The adapter's constant rendering every model, source and hook expansion shares. +fn constant_rendering<'py>(context: &Bound<'py, PyAny>) -> PyResult> { + Bound::new( + context.py(), + ConstantRendering::new( + context.getattr("value_renderer")?.unbind(), + context.getattr("collection_rendering")?.unbind(), + ), + ) } fn macro_inputs<'py>(context: &Bound<'py, PyAny>) -> PyResult> { @@ -283,14 +271,16 @@ pub(crate) fn build_model_inputs<'py>( project_config_path.str()?, ), ))?; - let files: Bound<'py, PyList> = PyList::new( - py, - discovered_inputs - .getattr("model_files")? - .try_iter()? - .collect::>>()?, + let files: Vec> = discovered_inputs + .getattr("model_files")? + .try_iter()? + .map(|file| file?.extract()) + .collect::>()?; + let inputs: Vec> = NativeModelLoop::run( + native_loop.cast::()?, + native_project.cast::()?.clone(), + constant_rendering(context)?, + files, )?; - let inputs: Bound<'py, PyAny> = - native_loop.call_method1("run", (native_project, loop_host(context)?, files))?; - PyTuple::new(py, inputs.try_iter()?.collect::>>()?) + PyTuple::new(py, inputs) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index e2588c9eea..262ba423fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -4,11 +4,16 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use crate::bindings::_helpers::compile_attachment::declaration_scope::rebind_declaration_scope; +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_vars; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; /// Build the declaration scope of `discovered`; a static scope lends only its index to macros. fn declaration_scope<'py>( @@ -20,14 +25,9 @@ fn declaration_scope<'py>( let loaded_macros: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadProjectMacros)? .call1((discovered.getattr("macro_files")?,))?; if !static_scope.is_none() { - let rebound: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::RebindDeclarationScope)?, - &[ - ("scope", static_scope), - ("discovered_inputs", discovered.clone()), - ("loaded_macros", loaded_macros.clone()), - ], - )?; + let rebound: Bound<'py, PyAny> = + rebind_declaration_scope(&static_scope, &discovered.clone(), &loaded_macros.clone()) + .and_then(|value| value.into_bound_py_any(py))?; if !rebound.is_none() { return Ok(rebound); } @@ -115,15 +115,13 @@ pub(crate) fn build_sql_expansion_context<'py>( if cli_vars.is_none() { cli_vars = PyDict::new(py).into_any(); } - let effective_vars: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildEffectiveVars)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("target_config", py.None().into_bound(py)), - ("cli_vars", cli_vars), - ], - )?; + let effective_vars: Bound<'py, PyAny> = build_effective_vars( + &project_config.clone(), + &local_config.clone(), + &py.None().into_bound(py), + &cli_vars, + ) + .and_then(|value| value.into_bound_py_any(py))?; let mut scope: Bound<'py, PyAny> = arguments.value("declaration_scope")?; if scope.is_none() { scope = declaration_scope( @@ -136,25 +134,16 @@ pub(crate) fn build_sql_expansion_context<'py>( registry::native(py, NativeAttr::PublicDeclarationIndexes)? .call1((&discovered,))? .extract()?; - let collection_rendering: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveCollectionRendering)?, - &[ - ("project_config", project_config.clone()), - ("declaration_override", py.None().into_bound(py)), - ], - )?; + let collection_rendering: Bound<'py, PyAny> = + resolve_effective_collection_rendering(&project_config.clone(), &py.None().into_bound(py)) + .and_then(|value| value.into_bound_py_any(py))?; let macro_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::MacroContext)?, &[ ( "adapter_name", - call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config), - ], - )?, + resolve_effective_adapter_name(&project_config.clone(), &local_config) + .and_then(|value| value.into_bound_py_any(py))?, ), ( "sql_analysis_enabled", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs index 3100046009..21c49a298a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -8,16 +8,15 @@ use crate::bindings::_helpers::compile_attachment::effective_config::project_var use crate::bindings::_helpers::compile_command::constants::{ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, RETIRED_CACHE_DIRECTORY_NAMES, }; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; use crate::bindings::_helpers::frontier::constants::{ COMPILER_CACHE_DIRECTORY_NAME, RETIRED_CACHE_NAMESPACE_SUFFIXES, }; use crate::bindings::_helpers::model_config::template_data::{early_model_templates, run_id}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; const DATABASE_KEY: &str = "database"; const SCHEMA_KEY: &str = "schema"; @@ -30,25 +29,19 @@ fn resolved_target<'py>( let py: Python<'py> = discovered.py(); let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; - let target_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("selected_target", selected_target.clone()), - ], - )?; + let target_name: Bound<'py, PyAny> = resolve_target_name( + py, + &project_config.clone(), + &local_config.clone(), + &selected_target.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; if target_name.is_none() { return Ok((target_name, py.None().into_bound(py))); } - let target_config: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetConfig)?, - &[ - ("project_config", project_config), - ("local_config", local_config), - ("target_name", target_name.clone()), - ], - )?; + let target_config: Bound<'py, PyAny> = + resolve_target_config(py, &project_config, &local_config, &target_name.clone()) + .and_then(|value| value.into_bound_py_any(py))?; Ok((target_name, target_config)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index a25a389139..2bd86cf8e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -6,9 +6,12 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::assembly_project::assembled_project; use crate::bindings::_helpers::compile_driver::project::lineage_mode; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, within, }; +use crate::bindings::_helpers::compiled_project::records::project_with; +use crate::bindings::_helpers::compiled_project::records::record_model_analyses; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::NativeAttr; @@ -35,7 +38,7 @@ pub(crate) struct AssemblyRequest { /// Convert compile inputs into the planner-ready project view. #[pyfunction] #[pyo3(signature = (**options))] -fn assemble_project<'py>( +pub(crate) fn assemble_project<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -60,9 +63,9 @@ fn assemble_project<'py>( return Ok(built); } let diagnostics: Bound<'py, PyTuple> = chained(&built.getattr("diagnostics")?, &collected)?; - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[("project", built), ("diagnostics", diagnostics.into_any())], + project_with( + &built, + Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), ) } @@ -125,13 +128,15 @@ fn referenced_project(py: Python<'_>, request: AssemblyRequest) -> PyResult = chained(&project.getattr("diagnostics")?, &reference_diagnostics)?; - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[ - ("project", project), - ("diagnostics", diagnostics.into_any()), - ("unmatched_literal_sql_relations", unmatched), - ], + project_with( + &project, + Some(keywords( + py, + &[ + ("diagnostics", diagnostics.into_any()), + ("unmatched_literal_sql_relations", unmatched), + ], + )?), ) } @@ -144,22 +149,13 @@ fn with_native_project<'py>( if native_project.is_none() { return Ok(project.clone()); } - call_with( - ®istry::native(py, NativeAttr::RecordModelAnalyses)?, - &[ - ("project", native_project.clone()), - ("models", project.getattr("models")?), - ], - )?; + record_model_analyses(native_project.cast()?, &project.getattr("models")?)?; if project.getattr("native_project")?.is(native_project) { return Ok(project.clone()); } - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[ - ("project", project.clone()), - ("native_project", native_project.clone()), - ], + project_with( + &project.clone(), + Some(keywords(py, &[("native_project", native_project.clone())])?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs index 05c6331f5d..8e9a68e439 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_checks.rs @@ -5,11 +5,14 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::attachments::cursor_intrinsics::validated_model_cursor_sql; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_values::declaration_scope_resolver; +use pyo3::IntoPyObjectExt; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; /// The `CompiledSqlScenario` fields copied from its input, in Python's keyword order. const SCENARIO_FIELDS: [&str; 12] = [ "sql_body", @@ -110,13 +113,11 @@ fn model_macro_context<'py>(inputs: &Bound<'py, PyAny>) -> PyResult = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", inputs.getattr("project_config")?), - ("local_config", inputs.getattr("local_config")?), - ], - )?; + let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( + &inputs.getattr("project_config")?, + &inputs.getattr("local_config")?, + ) + .and_then(|value| value.into_bound_py_any(py))?; call_with( ®istry::class(py, RecordId::MacroContext)?, &[ @@ -141,7 +142,7 @@ pub(crate) fn compiled_sql_scenarios<'py>( let scenario_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledSqlScenario)?; let key_class: Bound<'py, PyAny> = registry::class(py, RecordId::CompiledObjectKey)?; let resource_type: Bound<'py, PyAny> = - imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("SQL_SCENARIO")?; + host::entry(py, FacadeHost::CompiledResourceType)?.getattr("SQL_SCENARIO")?; let mut scenarios: Vec> = Vec::new(); for scenario_input in inputs.getattr("scenario_inputs")?.try_iter()? { let scenario_input: Bound<'py, PyAny> = scenario_input?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index 81d7e22eac..aef011a80c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -7,8 +7,12 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyListMethods, PyTuple}; use crate::bindings::_helpers::compile_driver::assembly_analysis::AnalysisContext; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::project_assembly::models::{NativeModelFacts, NativeProjectFacts}; +use crate::bindings::_helpers::project_assembly::native_resources::assemble_native_project_resources; +use crate::bindings::_helpers::project_assembly::syntax_checks::model_syntax_checks; +use crate::bindings::_helpers::project_assembly::syntax_checks::standalone_pivot_models; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; /// `NativeProjectFacts` of `inputs`: resources, syntax validity, deps and standalone pivot proofs. pub(crate) fn native_project_facts<'py>( @@ -31,31 +35,19 @@ pub(crate) fn native_project_facts<'py>( } else { PyDict::new(py).into_any() }; - let syntax_checks: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ModelSyntaxChecks)?, - &[ - ("model_inputs", model_inputs.clone()), - ("analysis_model_names", names.clone()), - ("analysis_succeeded", succeeded), - ], - )?; - let resources: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AssembleNativeProjectResources)?, - &[ - ("inputs", inputs.clone()), - ("dialect", dialect.clone()), - ("syntax_checks", syntax_checks), - ], - )?; + let syntax_checks: Bound<'py, PyAny> = + model_syntax_checks(&model_inputs.clone(), &names.clone(), &succeeded) + .and_then(|value| value.into_bound_py_any(py))?; + let resources: Bound<'py, PyAny> = assemble_native_project_resources( + &inputs.clone(), + dialect.clone().extract()?, + &syntax_checks, + ) + .and_then(|value| value.into_bound_py_any(py))?; let proofs: Bound<'py, PyDict> = pivot_proofs( - &call_with( - ®istry::native(py, NativeAttr::StandalonePivotModels)?, - &[ - ("model_inputs", model_inputs.clone()), - ("analysed", positions), - ], - )? - .cast_into::()?, + &standalone_pivot_models(&model_inputs.clone(), &positions) + .and_then(|value| value.into_bound_py_any(py))? + .cast_into::()?, context, session, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 1a57642e0f..0e57c1791c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ }; use crate::bindings::_helpers::compile_driver::assembly_sql_tests::assembled_sql_tests; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::model_config::template_data::early_model_templates; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -23,6 +24,7 @@ use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; +use pyo3::IntoPyObjectExt; const SOURCE_SLOT: &str = "source"; const SEED_SLOT: &str = "seed"; @@ -164,13 +166,11 @@ fn connection_database_fallback<'py>( connection: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", inputs.getattr("project_config")?), - ("local_config", inputs.getattr("local_config")?), - ], - )?; + let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( + &inputs.getattr("project_config")?, + &inputs.getattr("local_config")?, + ) + .and_then(|value| value.into_bound_py_any(py))?; if adapter_name.eq(host::entry(py, AdapterHost::BuiltinAdapter)?.getattr("DUCKDB")?)? { return Ok(py.None().into_bound(py)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index 4eba2d7bfc..1dd915b13f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -6,10 +6,12 @@ use sqlbuild_config::project::main::unsupported_target_warehouse::unsupported_ta use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; const SQL_ANALYSIS_KEY: &str = "sql_analysis"; @@ -143,13 +145,11 @@ fn reject_unsupported_target_warehouses(config: &ProjectConfiguration<'_, '_>) - let Some(group) = first_configured_group(config.local_config)? else { return Ok(()); }; - let adapter_name: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", config.project_config.clone()), - ("local_config", config.local_config.clone()), - ], - )?; + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &config.project_config.clone(), + &config.local_config.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; if session_warehouse_support(py, &adapter_name, config.project_dir)? { return Ok(()); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 09b049f239..d422e28f3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -1,7 +1,7 @@ //! Project discovery: configuration, declaration files and imported project Python, validated. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PySet, PySetMethods, PyTuple}; +use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::config_validation::{ @@ -16,6 +16,7 @@ use crate::bindings::_helpers::discovery::declaration_records::{ discover_macro_files, discover_model_schema_files, discover_seed_files, discover_sql_function_files, discover_sql_hook_files, retained_discovery_session, }; +use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; use crate::bindings::_helpers::discovery::model_records::{ ModelFileSelection, discovered_model_files, }; @@ -29,11 +30,13 @@ use crate::bindings::_helpers::discovery::yaml_records::{ discover_schema_files, discover_source_files, }; use crate::bindings::_helpers::event_exporting::scope::configure_discovered_event_exporters; +use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, produced}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; @@ -185,29 +188,11 @@ fn discover_project_inputs<'py>( || arguments.flag("extract_output_column_locations")?, }; within(&lifecycle(py, "project_discovery")?, |_| { - let stage: Bound<'py, PyCFunction> = PyCFunction::new_closure( - py, - None, - None, - move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { - let py: Python<'_> = arguments.py(); - within(&lifecycle(py, "discovery_project_assembly")?, |_| { - assembled_inputs(py, &request) - }) - .map(Bound::unbind) - }, - )?; - call_with( - ®istry::native(py, NativeAttr::CompileFrontier)?, - &[ - ( - "until", - registry::class(py, RecordId::CompilerStage)? - .getattr("DISCOVERED_PROJECT_INPUTS")?, - ), - ("python_stage", stage.into_any()), - ], - ) + let assembled: Bound<'py, PyAny> = + within(&lifecycle(py, "discovery_project_assembly")?, |_| { + assembled_inputs(py, &request) + })?; + produced(FrontierStage::DiscoveredProjectInputs, assembled) }) } @@ -371,10 +356,8 @@ fn declaration_files<'py>( /// Providers, then every kind of imported project Python that may use them. fn imported_python<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { let py: Python<'py> = project_dir.py(); - let providers: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverProviderClasses)?, - &[("project_dir", project_dir.clone())], - )?; + let providers: Bound<'py, PyAny> = discover_provider_classes(project_dir.clone()) + .and_then(|value| value.into_bound_py_any(py))?; let python: Bound<'py, PyDict> = PyDict::new(py); python.set_item("providers", &providers)?; for (field, discoverer) in PYTHON_DISCOVERERS { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index 9b41b1d58b..a5d9d34675 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -1,17 +1,26 @@ //! The first attached compile input snapshot: target, vars, macros, scopes, models and resources. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_attachment::effective_config::{ + build_effective_connection, build_effective_settings, build_effective_vars, +}; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, keywords, within, }; +use crate::bindings::_helpers::compiled_project::attachment::{attach_checks, attach_resources}; +use crate::bindings::_helpers::compiled_project::records::{compile_inputs_view, inputs_with}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::model_config::template_data::run_id; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::render::declaration_indexes::{ + public_declaration_indexes, public_model_schema_index, +}; const REQUIRED: [&str; 2] = ["discovered_inputs", "adapter_context"]; const OPTIONAL: [&str; 8] = [ @@ -25,6 +34,42 @@ const OPTIONAL: [&str; 8] = [ "no_cache", ]; +/// The arguments of one compile input attachment; unset flags are `False`, other options `None`. +pub(crate) struct InputsRequest { + pub(crate) discovered_inputs: Py, + pub(crate) adapter_context: Py, + pub(crate) selected_target: Py, + pub(crate) cli_vars: Py, + pub(crate) run_id: Py, + pub(crate) no_sql_validation: Py, + pub(crate) defer_model_sql_validation: Py, + pub(crate) external_sql_reference_resolver: Py, + pub(crate) resolved_connection: Py, + pub(crate) no_cache: Py, +} + +impl InputsRequest { + /// The request `build_compile_inputs` was called with. + fn read(arguments: &KeywordArguments<'_>) -> PyResult { + Ok(Self { + discovered_inputs: arguments.value("discovered_inputs")?.unbind(), + adapter_context: arguments.value("adapter_context")?.unbind(), + selected_target: arguments.value("selected_target")?.unbind(), + cli_vars: arguments.value("cli_vars")?.unbind(), + run_id: arguments.value("run_id")?.unbind(), + no_sql_validation: arguments.value_or_false("no_sql_validation")?.unbind(), + defer_model_sql_validation: arguments + .value_or_false("defer_model_sql_validation")? + .unbind(), + external_sql_reference_resolver: arguments + .value("external_sql_reference_resolver")? + .unbind(), + resolved_connection: arguments.value("resolved_connection")?.unbind(), + no_cache: arguments.value_or_false("no_cache")?.unbind(), + }) + } +} + /// The effective target, vars, settings and macro context every later stage shares. struct TargetContext<'py> { target_name: Bound<'py, PyAny>, @@ -61,11 +106,19 @@ fn build_compile_inputs<'py>( ) -> PyResult> { let arguments: KeywordArguments<'py> = KeywordArguments::read(py, "build_compile_inputs", options, (&REQUIRED, &OPTIONAL))?; + attached_inputs(py, &InputsRequest::read(&arguments)?) +} + +/// The first compile input snapshot, with diagnostics collected while attaching appended. +pub(crate) fn attached_inputs<'py>( + py: Python<'py>, + request: &InputsRequest, +) -> PyResult> { let collector: Bound<'py, PyAny> = registry::native(py, NativeAttr::CollectCompileDiagnostics)?.call0()?; let (built, violations): (Bound<'py, PyAny>, Bound<'py, PyAny>) = within(&collector, |violations| { - Ok((compile_inputs(py, &arguments)?, violations.clone())) + Ok((compile_inputs(py, request)?, violations.clone())) })?; let collected: Bound<'py, PyAny> = violations.getattr("diagnostics")?; if !collected.is_truthy()? { @@ -79,21 +132,18 @@ fn build_compile_inputs<'py>( .chain(collected.try_iter()?) .collect::>>()?, )?; - call_with( - ®istry::native(py, NativeAttr::InputsWith)?, - &[("inputs", built), ("diagnostics", diagnostics.into_any())], + inputs_with( + &built, + Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), ) } /// The undecorated `build_compile_inputs`. -fn compile_inputs<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, -) -> PyResult> { - let discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - let adapter_context: Bound<'py, PyAny> = arguments.value("adapter_context")?; - let target: TargetContext<'py> = target_context(arguments, &discovered, &adapter_context)?; - let run_id: Bound<'py, PyAny> = run_id(&arguments.value("run_id")?)?; +fn compile_inputs<'py>(py: Python<'py>, request: &InputsRequest) -> PyResult> { + let discovered: Bound<'py, PyAny> = request.discovered_inputs.bind(py).clone(); + let adapter_context: Bound<'py, PyAny> = request.adapter_context.bind(py).clone(); + let target: TargetContext<'py> = target_context(request, &discovered, &adapter_context)?; + let run_id: Bound<'py, PyAny> = run_id(&request.run_id.bind(py).clone())?; let loaded_macros: Bound<'py, PyAny> = host::entry(py, UserCodeHost::LoadProjectMacros)? .call1((discovered.getattr("macro_files")?,))?; let syntax: Bound<'py, PyAny> = adapter_context.getattr("sql_lexical_syntax")?; @@ -129,9 +179,9 @@ fn compile_inputs<'py>( ), ], )?; - let models: ModelBuild<'py> = model_build(arguments, &discovered, &model_context)?; + let models: ModelBuild<'py> = model_build(request, &discovered, &model_context)?; let attached: AttachedResources<'py> = attached_resources( - arguments, + request, &adapter_context, (&discovered, &declaration_scope), &models, @@ -149,12 +199,12 @@ fn compile_inputs<'py>( ("compile_cache_dir", target.compile_cache_dir.clone()), ( "effective_connection", - effective_connection(arguments, &discovered, &target)?, + effective_connection(request, &discovered, &target)?, ), ("effective_settings", target.settings.clone()), ( "no_sql_validation", - arguments.value_or_false("no_sql_validation")?, + request.no_sql_validation.bind(py).clone(), ), ("effective_vars", target.vars.clone()), ("macro_context", target.macro_context.clone()), @@ -171,24 +221,18 @@ fn compile_inputs<'py>( ("diagnostics", attached.checks.get_item(3)?), ( "external_sql_reference_resolver", - arguments.value("external_sql_reference_resolver")?, + request.external_sql_reference_resolver.bind(py).clone(), ), ("scope_index", declaration_scope.getattr("index")?), ("declaration_scope", declaration_scope), ], )?; - call_with( - ®istry::native(py, NativeAttr::CompileInputsView)?, - &[ - ("fields", fields.into_any()), - ("native_project", models.native_project), - ], - ) + compile_inputs_view(&fields, Some(models.native_project.cast_into()?)) } /// The selected target with the macro call store, and the effective vars, settings and macro context. fn target_context<'py>( - arguments: &KeywordArguments<'py>, + request: &InputsRequest, discovered: &Bound<'py, PyAny>, adapter_context: &Bound<'py, PyAny>, ) -> PyResult> { @@ -197,43 +241,30 @@ fn target_context<'py>( &host::entry(py, UserCodeHost::TargetContextWithMacroCallStore)?, &[ ("discovered_inputs", discovered.clone()), - ("selected_target", arguments.value("selected_target")?), - ("no_cache", arguments.value_or_false("no_cache")?), + ("selected_target", request.selected_target.bind(py).clone()), + ("no_cache", request.no_cache.bind(py).clone()), ], )?; let target_name: Bound<'py, PyAny> = selected.get_item(0)?; let target: Bound<'py, PyAny> = selected.get_item(1)?; let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; - let cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; + let cli_vars: Bound<'py, PyAny> = request.cli_vars.bind(py).clone(); let cli_vars: Bound<'py, PyAny> = if cli_vars.is_none() { PyDict::new(py).into_any() } else { cli_vars }; - let vars: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildEffectiveVars)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("target_config", target.clone()), - ("cli_vars", cli_vars), - ], - )?; - let settings: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildEffectiveSettings)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ], - )?; - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config), - ], + let vars: Bound<'py, PyAny> = build_effective_vars( + &project_config.clone(), + &local_config.clone(), + &target.clone(), + &cli_vars, )?; + let settings: Bound<'py, PyAny> = + build_effective_settings(&project_config.clone(), &local_config.clone())?; + let adapter_name: Bound<'py, PyAny> = + resolve_effective_adapter_name(&project_config.clone(), &local_config)?; let macro_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::MacroContext)?, &[ @@ -269,17 +300,15 @@ fn target_context<'py>( /// `_build_models_with_declarations`: attach every model with the public declaration indexes. fn model_build<'py>( - arguments: &KeywordArguments<'py>, + request: &InputsRequest, discovered: &Bound<'py, PyAny>, context: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = discovered.py(); - let declarations: Bound<'py, PyAny> = - registry::native(py, NativeAttr::PublicDeclarationIndexes)?.call1((discovered,))?; - let enums: Bound<'py, PyAny> = declarations.get_item(0)?; - let constants: Bound<'py, PyAny> = declarations.get_item(1)?; - let schemas: Bound<'py, PyAny> = - registry::native(py, NativeAttr::PublicModelSchemaIndex)?.call1((discovered,))?; + let (enums, constants) = public_declaration_indexes(discovered.clone())?; + let (enums, constants): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + (enums.into_any(), constants.into_any()); + let schemas: Bound<'py, PyAny> = public_model_schema_index(discovered.clone())?.into_any(); let native_project: Bound<'py, PyAny> = registry::native(py, NativeAttr::NativeCompiledProject)?.call0()?; let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; @@ -301,15 +330,15 @@ fn model_build<'py>( ("context", model_context), ( "no_sql_validation", - arguments.value_or_false("no_sql_validation")?, + request.no_sql_validation.bind(py).clone(), ), ( "defer_model_sql_validation", - arguments.value_or_false("defer_model_sql_validation")?, + request.defer_model_sql_validation.bind(py).clone(), ), ( "external_sql_reference_resolver", - arguments.value("external_sql_reference_resolver")?, + request.external_sql_reference_resolver.bind(py).clone(), ), ("native_project", native_project.clone()), ], @@ -335,7 +364,7 @@ fn model_build<'py>( /// The seeds, functions, sources and checks attached to the models. fn attached_resources<'py>( - arguments: &KeywordArguments<'py>, + request: &InputsRequest, adapter_context: &Bound<'py, PyAny>, (discovered, declaration_scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), models: &ModelBuild<'py>, @@ -343,55 +372,63 @@ fn attached_resources<'py>( let py: Python<'py> = discovered.py(); let context: &Bound<'py, PyAny> = &models.context; let settings: Bound<'py, PyAny> = context.getattr("effective_settings")?; - let resources: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AttachResources)?, - &[ - ("discovered_inputs", discovered.clone()), - ("native_project", models.native_project.clone()), - ( - "settings", - PyTuple::new( - py, - [ - settings.clone(), - context.getattr("target_config")?, - context.getattr("macro_context")?.getattr("adapter_name")?, - ], - )? - .into_any(), - ), - ( - "no_sql_validation", - arguments.value_or_false("no_sql_validation")?, - ), - ( - "python_functions_inherit_default_namespace", - adapter_context.getattr("python_functions_inherit_default_namespace")?, - ), - ], - )?; + let resources: Bound<'py, PyAny> = attach_resources( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("native_project", models.native_project.clone()), + ( + "settings", + PyTuple::new( + py, + [ + settings.clone(), + context.getattr("target_config")?, + context.getattr("macro_context")?.getattr("adapter_name")?, + ], + )? + .into_any(), + ), + ( + "no_sql_validation", + request.no_sql_validation.bind(py).clone(), + ), + ( + "python_functions_inherit_default_namespace", + adapter_context.getattr("python_functions_inherit_default_namespace")?, + ), + ], + )?), + )? + .into_any(); let sources: Bound<'py, PyAny> = resources.get_item(2)?; - let checks: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AttachChecks)?, - &[ - ("discovered_inputs", discovered.clone()), - ("native_project", models.native_project.clone()), - ( - "settings", - PyTuple::new( - py, - [ - settings, - adapter_context.getattr("value_renderer")?, - context.getattr("sql_lexical_syntax")?, - ], - )? - .into_any(), - ), - ("scan_cache", declaration_scope.getattr("sql_test_scans")?), - ("source_inputs", sources.clone()), - ], - )?; + let checks: Bound<'py, PyAny> = attach_checks( + py, + Some(keywords( + py, + &[ + ("discovered_inputs", discovered.clone()), + ("native_project", models.native_project.clone()), + ( + "settings", + PyTuple::new( + py, + [ + settings, + adapter_context.getattr("value_renderer")?, + context.getattr("sql_lexical_syntax")?, + ], + )? + .into_any(), + ), + ("scan_cache", declaration_scope.getattr("sql_test_scans")?), + ("source_inputs", sources.clone()), + ], + )?), + )? + .into_any(); Ok(AttachedResources { seeds: resources.get_item(0)?, functions: resources.get_item(1)?, @@ -402,22 +439,19 @@ fn attached_resources<'py>( /// The resolved connection when given, otherwise the configured one for the target. fn effective_connection<'py>( - arguments: &KeywordArguments<'py>, + request: &InputsRequest, discovered: &Bound<'py, PyAny>, target: &TargetContext<'py>, ) -> PyResult> { - let resolved: Bound<'py, PyAny> = arguments.value("resolved_connection")?; + let resolved: Bound<'py, PyAny> = request.resolved_connection.bind(discovered.py()).clone(); if !resolved.is_none() { return Ok(resolved); } - call_with( - ®istry::native(discovered.py(), NativeAttr::BuildEffectiveConnection)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ("target_config", target.target.clone()), - ("effective_vars", target.vars.clone()), - ], + build_effective_connection( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + &target.target.clone(), + &target.vars.clone(), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index ce6ebd9b9e..276d018fa5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -5,11 +5,10 @@ use pyo3::types::{PyBool, PyCFunction, PyDict, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::compile_timings::record_compile_timing; +use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; use crate::bindings::_helpers::compile_driver::phases::monotonic; use crate::bindings::_helpers::compile_driver::pipeline_entries::project_graph; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::compile_driver::target_defaults::{ TargetDefaults, apply_target_defaults, }; @@ -17,9 +16,16 @@ use crate::bindings::_helpers::compile_driver::targets::{ managed_loader_target_isolation, managed_write_schemas, named_target_schema_strategy, project_targets, }; +use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; +use crate::bindings::_helpers::compile_driver::assembly::assemble_project; +use crate::bindings::_helpers::compile_driver::python_calls::keywords; +use crate::bindings::_helpers::compiled_project::records::project_with; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::TABLE_PROMOTION_MODE_SETTING_KEY; const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; @@ -112,23 +118,9 @@ impl ProjectRequest { /// The compiled project produced through the `COMPILED_PROJECT` frontier. fn compiled_project(py: Python<'_>, request: ProjectRequest) -> PyResult> { - let stage: Bound<'_, PyCFunction> = PyCFunction::new_closure( - py, - None, - None, - move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { - python_compiled_project(arguments.py(), &request).map(Bound::unbind) - }, - )?; - call_with( - ®istry::native(py, NativeAttr::CompileFrontier)?, - &[ - ( - "until", - registry::class(py, RecordId::CompilerStage)?.getattr("COMPILED_PROJECT")?, - ), - ("python_stage", stage.into_any()), - ], + produced( + FrontierStage::CompiledProject, + python_compiled_project(py, &request)?, ) } @@ -162,26 +154,30 @@ fn python_compiled_project<'py>( ("selection", selection.clone()), ], )?; - let project: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::AssembleProject)?, - &[ - ("inputs", compile_inputs.clone()), - ("inference_profile", inference_profile), - ( - "skip_column_inference", - py_bool(py, request.skip_column_inference), - ), - ( - "column_lineage_mode", - request.column_lineage_mode.bind(py).clone(), - ), - ( - "analysis_cache_dir", - compile_inputs.getattr("compile_cache_dir")?, - ), - ("analysis_model_names", analysis_model_names), - ], - )?; + let project: Bound<'py, PyAny> = assemble_project( + py, + Some(keywords( + py, + &[ + ("inputs", compile_inputs.clone()), + ("inference_profile", inference_profile), + ( + "skip_column_inference", + py_bool(py, request.skip_column_inference), + ), + ( + "column_lineage_mode", + request.column_lineage_mode.bind(py).clone(), + ), + ( + "analysis_cache_dir", + compile_inputs.getattr("compile_cache_dir")?, + ), + ("analysis_model_names", analysis_model_names), + ], + )?), + ) + .and_then(|value| value.into_bound_py_any(py))?; targeted_project(py, request, &project) } @@ -210,13 +206,9 @@ fn targeted_project<'py>( let project: Bound<'py, PyAny> = apply_target_defaults(project, &defaults)?; let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", project_config), - ("local_config", local_config.clone()), - ], - )?; + let adapter_name: Bound<'py, PyAny> = + resolve_effective_adapter_name(&project_config, &local_config.clone()) + .and_then(|value| value.into_bound_py_any(py))?; project_targets(&adapter_name, &project)?; let promotion_key: Bound<'py, PyAny> = pyo3::types::PyString::new(py, TABLE_PROMOTION_MODE_SETTING_KEY).into_any(); @@ -247,12 +239,9 @@ fn targeted_project<'py>( .chain(conflicts.try_iter()?) .collect::>>()?, )?; - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[ - ("project", project), - ("diagnostics", diagnostics.into_any()), - ], + project_with( + &project, + Some(keywords(py, &[("diagnostics", diagnostics.into_any())])?), ) } @@ -264,13 +253,11 @@ fn frontier_compile_inputs<'py>( ) -> PyResult> { let discovered: &Bound<'py, PyAny> = request.discovered_inputs.bind(py); let adapter: &Bound<'py, PyAny> = request.adapter.bind(py); - let collection_rendering: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveCollectionRendering)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("declaration_override", py.None().into_bound(py)), - ], - )?; + let collection_rendering: Bound<'py, PyAny> = resolve_effective_collection_rendering( + &discovered.getattr("project_config")?, + &py.None().into_bound(py), + ) + .and_then(|value| value.into_bound_py_any(py))?; let adapter_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompileAdapterContext)?, &[ @@ -283,38 +270,29 @@ fn frontier_compile_inputs<'py>( ("sql_lexical_syntax", adapter.getattr("sql_lexical_syntax")?), ], )?; - let stage: Bound<'py, PyAny> = imported(py, "functools", "partial")?.call( - (registry::native(py, NativeAttr::BuildCompileInputs)?,), - Some(&keywords( - py, - &[ - ("discovered_inputs", discovered.clone()), - ("adapter_context", adapter_context), - ("selected_target", request.selected_target.bind(py).clone()), - ("no_sql_validation", py_bool(py, request.no_sql_validation)), - ("defer_model_sql_validation", py_bool(py, true)), - ("cli_vars", request.cli_vars.bind(py).clone()), - ( - "resolved_connection", - request.resolved_connection.bind(py).clone(), - ), - ( - "external_sql_reference_resolver", - request.external_sql_reference_resolver.bind(py).clone(), - ), - ("no_cache", py_bool(py, no_cache)), - ], - )?), + let inputs: InputsRequest = InputsRequest { + discovered_inputs: discovered.clone().unbind(), + adapter_context: adapter_context.unbind(), + selected_target: request.selected_target.clone_ref(py), + cli_vars: request.cli_vars.clone_ref(py), + run_id: py.None(), + no_sql_validation: py_bool(py, request.no_sql_validation).unbind(), + defer_model_sql_validation: py_bool(py, true).unbind(), + external_sql_reference_resolver: request.external_sql_reference_resolver.clone_ref(py), + resolved_connection: request.resolved_connection.clone_ref(py), + no_cache: py_bool(py, no_cache).unbind(), + }; + let stage: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + attached_inputs(arguments.py(), &inputs).map(Bound::unbind) + }, )?; - call_with( - ®istry::native(py, NativeAttr::CompileFrontier)?, - &[ - ( - "until", - registry::class(py, RecordId::CompilerStage)?.getattr("COMPILE_PROJECT_INPUTS")?, - ), - ("python_stage", stage), - ], + produced( + FrontierStage::CompileProjectInputs, + bridged(stage.as_any())?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs index 27a3ec9e39..e0cc7f661d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyTuple; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiled_project::records::project_with; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -96,28 +97,30 @@ pub(crate) fn apply_target_defaults<'py>( ), )?); } - call_with( - ®istry::native(py, NativeAttr::ProjectWith)?, - &[ - ("project", project.clone()), - ( - "effective_target_database", - or_else( - project.getattr("effective_target_database")?, - &defaults.default_database, - )?, - ), - ( - "effective_target_schema", - or_else( - project.getattr("effective_target_schema")?, - &defaults.default_schema, - )?, - ), - ("models", PyTuple::new(py, models)?.into_any()), - ("seeds", PyTuple::new(py, seeds)?.into_any()), - ("functions", PyTuple::new(py, functions)?.into_any()), - ], + project_with( + &project.clone(), + Some(keywords( + py, + &[ + ( + "effective_target_database", + or_else( + project.getattr("effective_target_database")?, + &defaults.default_database, + )?, + ), + ( + "effective_target_schema", + or_else( + project.getattr("effective_target_schema")?, + &defaults.default_schema, + )?, + ), + ("models", PyTuple::new(py, models)?.into_any()), + ("seeds", PyTuple::new(py, seeds)?.into_any()), + ("functions", PyTuple::new(py, functions)?.into_any()), + ], + )?), ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs index 3bdd7d99b4..9e1e0ecd0d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/targets.rs @@ -20,9 +20,7 @@ use sqlbuild_core::text::main::python_strip::python_strip; use sqlbuild_model_config::templates::models::TemplateOptions; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::model_config::template_data::expanded_template_data; /// Connection settings that identify the warehouse two targets write to. @@ -39,8 +37,14 @@ const CONNECTION_IDENTITY_KEYS: [&str; 7] = [ const EXPLICIT_TARGET_ADAPTERS: [&str; 3] = ["snowflake", "bigquery", "databricks"]; const LOADER_NAMESPACE_LABEL: &str = "managed loader target namespace"; const DEFAULTS_HELP_PURPOSE: &str = "to set project-wide defaults"; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_name; +use crate::bindings::_helpers::discovery::guidance::setting_help; +use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const DEFAULTS_SECTION: &str = "defaults"; @@ -126,25 +130,19 @@ pub(crate) fn named_target_schema_strategy( } let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; - let target_name: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetName)?, - &[ - ("project_config", project_config.clone()), - ("local_config", local_config.clone()), - ("selected_target", selected_target.clone()), - ], - )?; + let target_name: Bound<'_, PyAny> = resolve_target_name( + py, + &project_config.clone(), + &local_config.clone(), + &selected_target.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; if target_name.is_none() { return Ok(()); } - let target_config: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetConfig)?, - &[ - ("project_config", project_config), - ("local_config", local_config), - ("target_name", target_name.clone()), - ], - )?; + let target_config: Bound<'_, PyAny> = + resolve_target_config(py, &project_config, &local_config, &target_name.clone()) + .and_then(|value| value.into_bound_py_any(py))?; let schema: Option = text_or_none(&target_config.getattr("schema")?)?; if schema.is_some_and(|schema| !python_strip(&schema).is_empty()) { return Ok(()); @@ -261,31 +259,21 @@ fn target_namespaces( ) -> PyResult { let py: Python<'_> = discovered.py(); let name: Bound<'_, PyAny> = PyString::new(py, target_name).into_any(); - let target_config: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveTargetConfig)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ("target_name", name.clone()), - ], - )?; - let runtime: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::BuildEffectiveRuntimeConfig)?, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", name.clone()), - ], - )?; + let target_config: Bound<'_, PyAny> = resolve_target_config( + py, + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + &name.clone(), + ) + .and_then(|value| value.into_bound_py_any(py))?; + let runtime: Bound<'_, PyAny> = + build_effective_runtime_config(&discovered.clone(), Some(name.clone()), None) + .and_then(|value| value.into_bound_py_any(py))?; let (_, effective_vars, _): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = runtime.extract()?; let connection: Bound<'_, PyAny> = if resolved_connection.is_none() { - call_with( - ®istry::native(py, NativeAttr::BuildEffectiveConnectionConfig)?, - &[ - ("discovered_inputs", discovered.clone()), - ("selected_target", name), - ], - )? + build_effective_connection_config(&discovered.clone(), Some(name), None) + .and_then(|value| value.into_bound_py_any(py))? } else { resolved_connection.clone() }; @@ -447,25 +435,29 @@ fn required_target_parts( let Some(first) = missing.first() else { return Ok(()); }; - let help: String = call_with( - ®istry::native(py, NativeAttr::SettingHelp)?, - &[ - ( - "purpose", - PyString::new(py, DEFAULTS_HELP_PURPOSE).into_any(), - ), - ( - "file_name", - PyString::new(py, PROJECT_CONFIG_FILENAME).into_any(), - ), - ("section", PyString::new(py, DEFAULTS_SECTION).into_any()), - ("key", PyString::new(py, first).into_any()), - ( - "value", - PyString::new(py, &format!("my_{first}")).into_any(), - ), - ], - )? + let help: String = setting_help( + py, + Some(keywords( + py, + &[ + ( + "purpose", + PyString::new(py, DEFAULTS_HELP_PURPOSE).into_any(), + ), + ( + "file_name", + PyString::new(py, PROJECT_CONFIG_FILENAME).into_any(), + ), + ("section", PyString::new(py, DEFAULTS_SECTION).into_any()), + ("key", PyString::new(py, first).into_any()), + ( + "value", + PyString::new(py, &format!("my_{first}")).into_any(), + ), + ], + )?), + ) + .and_then(|value| value.into_bound_py_any(py))? .extract()?; Err(planner_error( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs index 3903924712..fc12fc6178 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_records/constants.rs @@ -1,5 +1,6 @@ //! Layouts of the compile and discovery value records hosted natively. +use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; @@ -383,7 +384,7 @@ const R_COMPILED_FUNCTION: RecordSpec = RecordSpec { FieldSpec::new("references", FieldDefault::EmptyTuple), FieldSpec::new( "language", - FieldDefault::Member("sqlbuild.compiler.compile.types", "FunctionLanguage", "SQL"), + FieldDefault::HostMember(FacadeHost::FunctionLanguage.id(), "SQL"), ), FieldSpec::new("source_file_path", FieldDefault::None), FieldSpec::new("runtime_version", FieldDefault::None), @@ -446,7 +447,7 @@ const R_COMPILE_SQL_TEST_INPUT: RecordSpec = RecordSpec { FieldSpec::required("sql_body"), FieldSpec::new( "mode", - FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + FieldDefault::HostMember(FacadeHost::SqlTestMode.id(), "MODEL"), ), FieldSpec::new( "payload", @@ -528,7 +529,7 @@ const R_COMPILED_DIRECT_LOGIC_SQL_TEST_PAYLOAD: RecordSpec = RecordSpec { FieldSpec::required("expected_cte"), FieldSpec::new( "mode", - FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + FieldDefault::HostMember(FacadeHost::SqlTestMode.id(), "MODEL"), ), FieldSpec::new("helper_ctes", FieldDefault::EmptyTuple), FieldSpec::new("tested_resource_names", FieldDefault::EmptyTuple), @@ -725,7 +726,7 @@ const R_DISCOVERED_SQL_TEST_BLOCK: RecordSpec = RecordSpec { FieldSpec::new("name", FieldDefault::None), FieldSpec::new( "mode", - FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + FieldDefault::HostMember(FacadeHost::SqlTestMode.id(), "MODEL"), ), FieldSpec::new("parameters", FieldDefault::EmptyTuple), FieldSpec::new("cases", FieldDefault::EmptyTuple), @@ -765,11 +766,7 @@ const R_DISCOVERED_AUDIT_BLOCK: RecordSpec = RecordSpec { FieldSpec::new("name", FieldDefault::None), FieldSpec::new( "evaluation_mode", - FieldDefault::Member( - "sqlbuild.compiler.auditing.types", - "AuditEvaluationMode", - "VIOLATIONS", - ), + FieldDefault::HostMember(FacadeHost::AuditEvaluationMode.id(), "VIOLATIONS"), ), FieldSpec::new("measurement_contract", FieldDefault::None), FieldSpec::new("measure_sql", FieldDefault::None), @@ -1448,11 +1445,7 @@ const R_COMPILED_AUDIT: RecordSpec = RecordSpec { FieldSpec::required("sql_body"), FieldSpec::new( "evaluation_mode", - FieldDefault::Member( - "sqlbuild.compiler.auditing.types", - "AuditEvaluationMode", - "VIOLATIONS", - ), + FieldDefault::HostMember(FacadeHost::AuditEvaluationMode.id(), "VIOLATIONS"), ), FieldSpec::new("measurement_contract", FieldDefault::None), FieldSpec::new("thresholds", FieldDefault::None), @@ -1577,7 +1570,7 @@ const R_COMPILED_SQL_TEST: RecordSpec = RecordSpec { FieldSpec::required("sql_body"), FieldSpec::new( "mode", - FieldDefault::Member("sqlbuild.compiler.compile.types", "SqlTestMode", "MODEL"), + FieldDefault::HostMember(FacadeHost::SqlTestMode.id(), "MODEL"), ), FieldSpec::new( "payload", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index c1da8fb1c9..5ac9ebb48a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -23,14 +23,13 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ TestBlockRow, TestCaseRow, TestFileRow, }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_CTE_SCAN_ALGORITHM; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const YAML_FILE_SUFFIXES: [&str; 2] = [".yml", ".yaml"]; const PATHLIB_MODULE: &str = "pathlib"; const TEST_TIMING_KEY: &str = "test_input_compile_ms"; @@ -57,18 +56,6 @@ fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult( - py: Python<'py>, - (module, class, member): (&str, &str, &str), -) -> PyResult> { - imported(py, module, class)?.getattr(member) -} - -/// `{member.value: member for member in enum_class}`. -fn by_value<'py>(py: Python<'py>, module: &str, class: &str) -> PyResult> { - members_by_value(&imported(py, module, class)?) -} - /// `{member.value: member for member in enum_class}` of a native string enum. fn native_by_value(py: Python<'_>, class: RecordId) -> PyResult> { members_by_value(®istry::class(py, class)?) @@ -350,7 +337,10 @@ fn sql_test_settings<'py>( ] { settings.set_item(key, registry::class(py, class)?)?; } - settings.set_item("modes", by_value(py, COMPILE_TYPES_MODULE, "SqlTestMode")?)?; + settings.set_item( + "modes", + members_by_value(&host::entry(py, FacadeHost::SqlTestMode)?)?, + )?; settings.set_item( "test_kind", registry::member(py, RecordId::ResourceKind, "TEST")?, @@ -379,14 +369,8 @@ fn sql_test_settings<'py>( fn audit_block_row(block: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = block.py(); - let measurement: Bound<'_, PyAny> = enum_member( - py, - ( - AUDITING_TYPES_MODULE, - "AuditEvaluationMode", - MEASUREMENT_MODE, - ), - )?; + let measurement: Bound<'_, PyAny> = + host::entry(py, FacadeHost::AuditEvaluationMode)?.getattr(MEASUREMENT_MODE)?; Ok(( block.clone().unbind(), attr(&block, "name")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index f7fe29395d..5610e7f072 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -5,20 +5,16 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; /// The materializations a model's storage policies are inherited for, besides none. const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; /// The declaration kinds whose project-owned records the loop names. const NAMED_KINDS: [&str; 4] = ["AUDIT", "SCHEMA", "SQL_HOOK", "PYTHON_HOOK"]; -fn class<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { - imported(py, module, name) -} - fn dict<'py>( py: Python<'py>, entries: Vec<(&str, Bound<'py, PyAny>)>, @@ -301,11 +297,11 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { vec![ ( "audit_severity", - class(py, AUDITING_TYPES_MODULE, "AuditSeverity")?, + host::entry(py, FacadeHost::AuditSeverity)?, ), ( "attached_target_kind", - class(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?, + host::entry(py, FacadeHost::AttachedAuditTargetKind)?, ), ], )? @@ -326,7 +322,7 @@ fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { ), ( "function_language", - class(py, COMPILE_TYPES_MODULE, "FunctionLanguage")?, + host::entry(py, FacadeHost::FunctionLanguage)?, ), ], )? @@ -436,7 +432,6 @@ pub(crate) fn model_loop_classes<'py>( "reference_kind", registry::class(py, RecordId::SqlReferenceKind)?, ), - ("partial", imported(py, "functools", "partial")?), ( "config", native_config_classes(&host("config_session")?, &host("record_template_reads")?)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index a9a455c55b..52967e92af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -62,6 +62,7 @@ use crate::bindings::_helpers::render::declaration_expansion::{ DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, expand_declarations, }; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; @@ -98,8 +99,6 @@ const UNMANAGED_ATTRIBUTE: &str = "unmanaged"; const TABLE_TYPE_ATTRIBUTE: &str = "table_type"; const DECLARED_ATTRIBUTE: &str = "declared"; const VALIDATE_METHOD: &str = "validate"; -const FILE_PATH_ARGUMENT: &str = "file_path"; -const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; const HELP_ARGUMENT: &str = "help"; const CODE_ARGUMENT: &str = "code"; const ANALYSIS_CONFLICT: &str = "MODEL sql_analysis conflicts with legacy sql_validation"; @@ -142,8 +141,6 @@ pub(crate) struct LoopClasses<'py> { pub(crate) external_resolver: Bound<'py, PyAny>, /// `SqlReferenceKind`. pub(crate) reference_kind: Bound<'py, PyAny>, - /// `functools.partial`, binding the model's file to its constant renderer. - pub(crate) partial: Bound<'py, PyAny>, /// What a model's effective config is built from. pub(crate) config: ConfigClasses<'py>, /// What a model's visible declarations are resolved from. @@ -180,7 +177,7 @@ pub(crate) struct NativeModelLoop { /// What one run reads beside each model: the host, Python classes and header contract classes. pub(crate) struct LoopContext<'a, 'py> { - pub(crate) host: &'a Bound<'py, PyAny>, + pub(crate) host: &'a Bound<'py, ConstantRendering>, pub(crate) classes: &'a LoopClasses<'py>, pub(crate) contracts: &'a ContractClasses<'py>, } @@ -230,22 +227,6 @@ impl NativeModelLoop { loop_classes, }) } - - /// Build every model in order into `project`; return each model's input view. - fn run<'py>( - slf: &Bound<'py, Self>, - project: Bound<'py, NativeCompiledProject>, - host: Bound<'py, PyAny>, - model_files: Vec>, - ) -> PyResult>> { - let py: Python<'py> = slf.py(); - let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; - let models: Vec = prepared_rows(py, &classes, model_files)?; - project - .borrow_mut() - .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); - slf.get().run_models(py, project, host, models) - } } /// Replace the SQL hook classes' discovered `hook_files` with their `definitions` by name. @@ -324,7 +305,7 @@ impl NativeModelLoop { &self, py: Python<'py>, project: Bound<'py, NativeCompiledProject>, - host: Bound<'py, PyAny>, + host: Bound<'py, ConstantRendering>, models: Vec, ) -> PyResult>> { let contracts: ContractClasses<'py> = self.contract_classes.bind(py).extract()?; @@ -390,11 +371,27 @@ impl NativeModelLoop { } impl NativeModelLoop { + /// Build every model in order into `project`; return each model's input view. + pub(crate) fn run<'py>( + slf: &Bound<'py, Self>, + project: Bound<'py, NativeCompiledProject>, + host: Bound<'py, ConstantRendering>, + model_files: Vec>, + ) -> PyResult>> { + let py: Python<'py> = slf.py(); + let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; + let models: Vec = prepared_rows(py, &classes, model_files)?; + project + .borrow_mut() + .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); + slf.get().run_models(py, project, host, models) + } + /// Attach every discovered source in order into `project`; return each source's view. pub(crate) fn run_sources<'py>( &self, project: &Bound<'py, NativeCompiledProject>, - host: &Bound<'py, PyAny>, + host: &Bound<'py, ConstantRendering>, rows: Vec, settings: SourceSettings, ) -> PyResult> { @@ -413,7 +410,7 @@ impl NativeModelLoop { /// Attach every singular audit, then each owner's attached audits. pub(crate) fn run_audits<'py>( &self, - host: &Bound<'py, PyAny>, + host: &Bound<'py, ConstantRendering>, files: &[AuditFileRow], owners: Vec>, settings: &AuditSettings, @@ -432,7 +429,7 @@ impl NativeModelLoop { /// Attach every SQL test case, then every scenario, against the project's resources. pub(crate) fn run_sql_tests<'py>( &self, - host: &Bound<'py, PyAny>, + host: &Bound<'py, ConstantRendering>, rows: (&[TestFileRow], &[ScenarioRow]), settings: &SqlTestSettings, table_function_arities: HashMap, @@ -499,7 +496,7 @@ impl NativeModelLoop { /// Attach every SQL, then Python, function in order, checking table function call arities. pub(crate) fn run_functions<'py>( &self, - host: &Bound<'py, PyAny>, + host: &Bound<'py, ConstantRendering>, rows: (Vec, Vec), settings: FunctionSettings, models: &[(String, Vec)], @@ -567,15 +564,6 @@ impl NativeModelLoop { }; let interpolated: String = applied_interpolation(context, interpolation)?; let (references, stop) = scan.clone().unwrap_or_default(); - let file_keyword: Bound<'py, PyDict> = PyDict::new(py); - file_keyword.set_item( - FILE_PATH_ARGUMENT, - context.classes.path_class.call1((file_path,))?, - )?; - let render_constant: Bound<'py, PyAny> = context.classes.partial.call( - (context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), - Some(&file_keyword), - )?; let expansion_classes: ExpansionClasses<'py> = ExpansionClasses { usage_record: context.classes.usage_record.clone(), compile_error: context.classes.compile_error.clone(), @@ -590,7 +578,7 @@ impl NativeModelLoop { relationship_visibility: false, path: file_path, }, - render_constant: &render_constant, + rendering: context.host.get(), classes: &expansion_classes, })?; let macros: Option = if calls_macros(&declarations.spliced.sql) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs index a0a5504022..8fdd710adf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs @@ -54,8 +54,6 @@ const FILE_PATH_ATTRIBUTE: &str = "file_path"; const SQL_BODY_ATTRIBUTE: &str = "sql_body"; const READS_ATTRIBUTE: &str = "reads"; const KIND_ATTRIBUTE: &str = "kind"; -const FILE_PATH_ARGUMENT: &str = "file_path"; -const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; const MACRO_TOKEN: char = '@'; /// `_VisibleModelDeclarations` fields an unscoped hook's `DeclarationResolutionContext` reads. const UNSCOPED_FIELDS: [&str; 4] = [ @@ -468,17 +466,7 @@ impl<'py> HookRun<'_, '_, 'py> { declarations: &Bound<'py, PyAny>, scope: &Scope<'_, 'py>, ) -> PyResult> { - let py: Python<'py> = scope.consumer.py(); let (references, stop) = declaration_scan(sql); - let file_keyword: Bound<'py, PyDict> = PyDict::new(py); - file_keyword.set_item( - FILE_PATH_ARGUMENT, - self.context.classes.path_class.call1((scope.file_path,))?, - )?; - let render_constant: Bound<'py, PyAny> = self.context.classes.partial.call( - (self.context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), - Some(&file_keyword), - )?; let classes: ExpansionClasses<'py> = ExpansionClasses { usage_record: self.context.classes.usage_record.clone(), compile_error: self.context.classes.compile_error.clone(), @@ -493,7 +481,7 @@ impl<'py> HookRun<'_, '_, 'py> { relationship_visibility: false, path: scope.file_path, }, - render_constant: &render_constant, + rendering: self.context.host.get(), classes: &classes, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index e74ac57b62..c02bcdf032 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -55,6 +55,7 @@ use crate::bindings::_helpers::compiled_project::source_views::{ AttachedSource, NativeCompiledSource, NativeSourceInput, SourceNamespace, SourceRecord, }; use crate::bindings::_helpers::compiled_project::values::{object_value, plain_value}; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; pub(crate) const CONTRACT_KEY: &str = "contract"; const INPUT_CLASSES_UNSET: &str = "no model loop recorded inputs into this project"; @@ -133,8 +134,8 @@ pub(crate) struct NativeCompiledProject { functions: Vec, /// Each singular, then model-, source- and seed-attached, audit in attachment order. audits: Vec, - /// The model loop and its host, which attach the project's other resources. - attachment: Option<(Py, Py)>, + /// The model loop and its constant rendering, which attach the project's other resources. + attachment: Option<(Py, Py)>, } impl NativeCompiledProject { @@ -229,19 +230,22 @@ impl NativeCompiledProject { Ok(owners) } - /// The kept model loop and its host. - fn attachment_loop(&self, py: Python<'_>) -> PyResult<(Py, Py)> { + /// The kept model loop and its constant rendering. + fn attachment_loop( + &self, + py: Python<'_>, + ) -> PyResult<(Py, Py)> { match &self.attachment { Some((model_loop, host)) => Ok((model_loop.clone_ref(py), host.clone_ref(py))), None => Err(PyRuntimeError::new_err(ATTACHMENT_LOOP_UNSET)), } } - /// Keep the model loop and its host for attaching the project's other resources. + /// Keep the model loop and its constant rendering for attaching the project's other resources. pub(crate) fn keep_attachment_loop( &mut self, model_loop: Py, - host: Py, + host: Py, ) { self.attachment = Some((model_loop, host)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs index 01ab413136..e7e0122c61 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs @@ -30,8 +30,6 @@ use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolated}; const MACRO_TOKEN: char = '@'; -const FILE_PATH_ARGUMENT: &str = "file_path"; -const CONSTANT_TEXT_CALLBACK: &str = "constant_text"; const NATIVE_CONTEXTS_ATTRIBUTE: &str = "native_contexts"; const GENERIC_DIALECT: &str = "generic"; const EXPRESSION_STATEMENTS: [&str; 3] = ["select", "with", "values"]; @@ -226,15 +224,6 @@ pub(crate) fn expanded_authored_sql<'py>( let mut argument_references: Vec = Vec::new(); let mut expanded_sql: String = interpolated_sql.clone(); if !references.is_empty() || stop.is_some() { - let file_keyword: Bound<'py, PyDict> = PyDict::new(py); - file_keyword.set_item( - FILE_PATH_ARGUMENT, - context.classes.path_class.call1((file_path,))?, - )?; - let render_constant: Bound<'py, PyAny> = context.classes.partial.call( - (context.host.getattr(CONSTANT_TEXT_CALLBACK)?,), - Some(&file_keyword), - )?; let classes: ExpansionClasses<'py> = ExpansionClasses { usage_record: context.classes.usage_record.clone(), compile_error: context.classes.compile_error.clone(), @@ -249,7 +238,7 @@ pub(crate) fn expanded_authored_sql<'py>( relationship_visibility: false, path: file_path, }, - render_constant: &render_constant, + rendering: context.host.get(), classes: &classes, })?; expanded_sql = expanded.spliced.sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs index 23bb0332fe..947a084832 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs @@ -34,6 +34,7 @@ use sqlbuild_render::macro_calls::main::macro_call_names::macro_call_names; use sqlbuild_sqltext::compiler::models::DeclarationReferenceStop; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::{ReferenceExtraction, SqlReference}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::bindings::_helpers::cache::sql_test_scan_store::SqlTestScanStore; use crate::bindings::_helpers::compiled_project::model_loop::{ @@ -662,8 +663,10 @@ impl<'py> TestScope<'_, 'py> { scenario: false, }) .collect(); - let classified: Vec = - extract_sql_tests(&requests, &self.model_loop.syntax).map_err(|failure| { + let syntax: &LexicalSyntax = &self.model_loop.syntax; + let classified: Vec = py + .detach(|| extract_sql_tests(&requests, syntax)) + .map_err(|failure| { let test: &ExpandedTest<'_, 'py> = &tests[pending[failure.index]]; self.located(&failure, test.file, (test.block, &test.sql)) })?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index 1a8778bef5..d28ff6345e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -18,10 +18,11 @@ use crate::bindings::_helpers::cursor::values::{ DATETIME_MODULE, civil_date, local_date_time, naive_date_time, plain_date, replaced, timedelta_micros, }; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -const EXCEPTIONS_MODULE: &str = "sqlbuild.cursor_algebra.exceptions"; /// Cursor grain names, finest first. pub(crate) const GRAIN_NAMES: [(&str, Grain); 6] = [ ("second", Grain::Second), @@ -34,8 +35,7 @@ pub(crate) const GRAIN_NAMES: [(&str, Grain); 6] = [ /// `CursorAlgebraError(message)`. pub(crate) fn algebra_error(py: Python<'_>, message: &str) -> PyErr { - match imported(py, EXCEPTIONS_MODULE, "CursorAlgebraError") - .and_then(|class| class.call1((message,))) + match host::entry(py, FacadeHost::CursorAlgebraError).and_then(|class| class.call1((message,))) { Ok(error) => PyErr::from_value(error), Err(error) => error, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 1759145de6..3f427818eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -27,12 +27,13 @@ use crate::bindings::_helpers::discovery::snapshot::{ }; use crate::bindings::_helpers::discovery::sorted_keys::sorted_keys; use crate::bindings::_helpers::model_config::header_columns::schema_columns; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const PATHLIB_MODULE: &str = "pathlib"; const SESSION_MEMO_KEY: &str = "native_discovery_session"; const MEASUREMENT_MODE: &str = "MEASUREMENT"; @@ -283,12 +284,10 @@ fn hook_file<'py>(context: &FileContext<'_, 'py>, file: &HookFile) -> PyResult(py: Python<'py>, block: &AuditBlock) -> PyResult> { let header_values: Bound<'py, PyDict> = authored(py, &block.header_values)?; - let mode: Bound<'py, PyAny> = imported(py, AUDITING_TYPES_MODULE, "AuditEvaluationMode")? - .call1((block.evaluation_mode,))?; + let mode: Bound<'py, PyAny> = + host::entry(py, FacadeHost::AuditEvaluationMode)?.call1((block.evaluation_mode,))?; let measurement: Bound<'py, PyAny> = - if mode.is(&imported(py, AUDITING_TYPES_MODULE, "AuditEvaluationMode")? - .getattr(MEASUREMENT_MODE)?) - { + if mode.is(&host::entry(py, FacadeHost::AuditEvaluationMode)?.getattr(MEASUREMENT_MODE)?) { call_with( ®istry::class(py, RecordId::MeasurementContract)?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs index 5ed0391081..4fadf5da43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/guidance.rs @@ -77,7 +77,10 @@ fn setting_text(value: &Bound<'_, PyAny>) -> PyResult { /// Render `, set this in :` followed by the exact TOML section and key. #[pyfunction] #[pyo3(signature = (**options))] -fn setting_help<'py>(py: Python<'py>, options: Option>) -> PyResult { +pub(crate) fn setting_help<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { let arguments: KeywordArguments<'py> = KeywordArguments::read( py, "setting_help", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs index ca782ab0ae..72f5ae26dd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs @@ -5,11 +5,13 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::discovery::locations::project_relative_path; use crate::bindings::_helpers::event_exporting::scope::current_scope; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; /// One project directory under discovery. pub(crate) struct ProjectDir<'py> { @@ -28,13 +30,8 @@ impl<'py> ProjectDir<'py> { /// `project_relative_path(path=file_path, project_dir=project_dir)`. pub(crate) fn relative(&self, file_path: &Bound<'py, PyAny>) -> PyResult> { - call_with( - ®istry::native(self.py(), NativeAttr::ProjectRelativePath)?, - &[ - ("path", file_path.clone()), - ("project_dir", self.path.clone()), - ], - ) + project_relative_path(&file_path.clone(), &self.path.clone()) + .and_then(|value| value.into_bound_py_any(self.py())) } /// `str(file_path.relative_to(project_dir))`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs index 2aacc4c22e..9974b14861 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs @@ -13,10 +13,12 @@ use crate::bindings::_helpers::discovery::imported_modules::{ }; use crate::bindings::_helpers::discovery::imported_providers; +use crate::bindings::_helpers::discovery::python_hook_files::discover_python_hook_files; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; +use pyo3::IntoPyObjectExt; use sqlbuild_core::constants::PYTHON_NODE_ROOT; const NODE_ERROR: ErrorId = ErrorId::PythonNodeDiscovery; const AUDIT_CASE: UserCodeHost = UserCodeHost::AuditCase; @@ -449,10 +451,8 @@ pub(crate) fn discover_hook_functions<'py>( Some(providers) => provider_by_name(&providers)?, None => PyDict::new(py), }; - let hook_files: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::DiscoverPythonHookFiles)?, - &[("project_dir", project.path.clone())], - )?; + let hook_files: Bound<'py, PyAny> = discover_python_hook_files(&project.path.clone()) + .and_then(|value| value.into_bound_py_any(py))?; let mut hooks: Vec> = Vec::new(); let mut seen: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); for hook_file in hook_files.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs index db24afda82..976d68d0ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/locations.rs @@ -17,7 +17,7 @@ fn yaml_entry_line(contents: &str, name: &str) -> Option { /// `path.relative_to(project_dir)` without pathlib's per-parent comparison scan. #[pyfunction] #[pyo3(signature = (*, path, project_dir))] -fn project_relative_path<'py>( +pub(crate) fn project_relative_path<'py>( path: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs index 4afa8b43dd..4caad201fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_hook_files.rs @@ -20,7 +20,7 @@ use crate::bindings::models::{NativeDiscoverySession, NativeProjectTree}; /// Each public Python hook file as `file_path`, `relative_path` and its role's scope fields. #[pyfunction] #[pyo3(signature = (*, project_dir))] -fn discover_python_hook_files<'py>( +pub(crate) fn discover_python_hook_files<'py>( project_dir: &Bound<'py, PyAny>, ) -> PyResult> { in_snapshot(project_dir, || { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 21d9b44d28..0c7ff037c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -9,11 +9,11 @@ use sqlbuild_model_config::sql_tests::models::{SqlTestBlock, SqlTestCase}; use sqlbuild_model_config::sql_values::models::{RawKey, RawSqlValue}; use crate::bindings::_helpers::contracts::sql_values::{SqlValueClasses, raw_value, sql_value}; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; -const TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - pub(crate) type NativeBlock<'py> = (Bound<'py, PyDict>, String, Bound<'py, PyAny>); fn parse_error(py: Python<'_>, message: String) -> PyResult { @@ -82,9 +82,7 @@ fn block_record<'py>( keywords.set_item("name", block.name)?; keywords.set_item( "mode", - py.import(TYPES_MODULE)? - .getattr("SqlTestMode")? - .call1((block.mode,))?, + host::entry(py, FacadeHost::SqlTestMode)?.call1((block.mode,))?, )?; keywords.set_item("parameters", PyTuple::new(py, parameters)?)?; keywords.set_item("cases", PyTuple::new(py, cases)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs index 086309d934..9c05ed2d96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/frontier/compile_frontier.rs @@ -44,17 +44,47 @@ fn native_frontier<'py>( if !native_stage.is_none() { return native_stage.call0(); } - let inputs_stage: Bound<'py, PyAny> = - registry::class(py, RecordId::CompilerStage)?.getattr("COMPILE_PROJECT_INPUTS")?; - if until.is(&inputs_stage) { - return call_with( - &host::entry(py, UserCodeHost::RunWithMacroBridge)?, - &[("stage", python_stage.clone())], - ); + if until.is(&frontier_stage(py, FrontierStage::CompileProjectInputs)?) { + return bridged(python_stage); } python_stage.call0() } +/// The compiler stages native callers produce through the frontier. +#[derive(Clone, Copy)] +pub(crate) enum FrontierStage { + DiscoveredProjectInputs, + CompileProjectInputs, + CompiledProject, +} + +/// The `CompilerStage` member of `stage`. +pub(crate) fn frontier_stage(py: Python<'_>, stage: FrontierStage) -> PyResult> { + let member: &str = match stage { + FrontierStage::DiscoveredProjectInputs => "DISCOVERED_PROJECT_INPUTS", + FrontierStage::CompileProjectInputs => "COMPILE_PROJECT_INPUTS", + FrontierStage::CompiledProject => "COMPILED_PROJECT", + }; + registry::class(py, RecordId::CompilerStage)?.getattr(member) +} + +/// `stage()` run inside the macro bridge user macros render through. +pub(crate) fn bridged<'py>(stage: &Bound<'py, PyAny>) -> PyResult> { + call_with( + &host::entry(stage.py(), UserCodeHost::RunWithMacroBridge)?, + &[("stage", stage.clone())], + ) +} + +/// The frontier object `produced` for `stage`, captured on request. +pub(crate) fn produced<'py>( + stage: FrontierStage, + produced: Bound<'py, PyAny>, +) -> PyResult> { + write_stage_capture(&frontier_stage(produced.py(), stage)?, &produced)?; + Ok(produced) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(compile_frontier, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index ccf852eb11..556a280393 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,12 +1,12 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; - type Object<'py> = Bound<'py, PyAny>; /// The compiled model of `model_input` from its session outcome, fallback rows and proof. @@ -17,7 +17,6 @@ pub(crate) fn compiled_model<'py>( macro_deps: &Object<'py>, ) -> PyResult> { let py: Python<'py> = model_input.py(); - let types = py.import(COMPILE_TYPES_MODULE)?; let model_file: Object<'py> = model_input.getattr("model_file")?; let model_name: Object<'py> = model_file.getattr("file_path")?.getattr("stem")?; let none: Object<'py> = py.None().into_bound(py); @@ -68,7 +67,7 @@ pub(crate) fn compiled_model<'py>( inferred = proof.getattr("fixed_columns")?; has_star = true_object(py); } - let resource_type: Object<'py> = types.getattr("CompiledResourceType")?; + let resource_type: Object<'py> = host::entry(py, FacadeHost::CompiledResourceType)?; let key_class: Object<'py> = registry::class(py, RecordId::CompiledObjectKey)?; let mut dependency_keys: Vec> = Vec::new(); for dependency in deps.try_iter()? { @@ -253,11 +252,10 @@ fn located<'py>( }; let located: Object<'py> = registry::native(py, NativeAttr::LocateModelBindingDiagnostics)? .call1((PyList::new(py, rows)?, texts, expansion_row))?; - let types = py.import(COMPILE_TYPES_MODULE)?; let diagnostic_class: Object<'py> = registry::class(py, RecordId::CompilerDiagnostic)?; let location_class: Object<'py> = registry::class(py, RecordId::SourceLocation)?; let phase: Object<'py> = registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?; - let model: Object<'py> = types.getattr("CompiledResourceType")?.getattr("MODEL")?; + let model: Object<'py> = host::entry(py, FacadeHost::CompiledResourceType)?.getattr("MODEL")?; let name: Object<'py> = model_file.getattr("file_path")?.getattr("stem")?; let mut built: Vec> = Vec::new(); for row in located.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs index a6dc231cfd..2ff6218008 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/contract_classes.rs @@ -4,13 +4,12 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods}; -use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::third_party::ThirdPartyHost; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const CONTRACT_CLASSES: [(&str, RecordId); 6] = [ ("schema_column", RecordId::SchemaColumn), ("schema_audit_instance", RecordId::SchemaAuditInstance), @@ -79,11 +78,11 @@ fn built_contract_classes(py: Python<'_>) -> PyResult> { } classes.set_item( "severities", - members_by_value(&imported(py, AUDITING_TYPES_MODULE, "AuditSeverity")?)?, + members_by_value(&host::entry(py, FacadeHost::AuditSeverity)?)?, )?; classes.set_item( "threshold_operators", - members_by_value(&imported(py, AUDITING_TYPES_MODULE, "ThresholdOperator")?)?, + members_by_value(&host::entry(py, FacadeHost::ThresholdOperator)?)?, )?; Ok(classes) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs index d9daaee8a8..a2d1968c07 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/executor.rs @@ -5,6 +5,9 @@ use crate::bindings::_helpers::native_host::models::{HostEdge, HostEntry, HostId /// One executor entry of `sqlbuild._native_host`, indexing `ENTRIES`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum ExecutorHost { + ProjectRelation, + ProjectRelationIndex, + RelationName, CostResourceContext, RunCostSummary, CostCapability, @@ -166,6 +169,21 @@ pub(crate) enum ExecutorHost { /// Every executor entry, in `ExecutorHost` order. pub(crate) const ENTRIES: &[HostEntry] = &[ + HostEntry { + name: "ProjectRelation", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, + HostEntry { + name: "ProjectRelationIndex", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, + HostEntry { + name: "RelationName", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, HostEntry { name: "CostResourceContext", kind: HostKind::Class, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs index 09f2323f5e..d40ae9e79f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/facade.rs @@ -10,6 +10,16 @@ pub(crate) enum FacadeHost { ProviderSettingsInputs, StartupInputs, CompileReuseRequest, + AuditEvaluationMode, + AuditOutcome, + AuditSeverity, + ThresholdOperator, + AttachedAuditTargetKind, + CompiledResourceType, + FunctionLanguage, + SqlTestMode, + CursorAlgebraError, + SharedInputError, } /// Every facade entry, in `FacadeHost` order. @@ -39,6 +49,56 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Facade, }, + HostEntry { + name: "AuditEvaluationMode", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "AuditOutcome", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "AuditSeverity", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "ThresholdOperator", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "AttachedAuditTargetKind", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "CompiledResourceType", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "FunctionLanguage", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "SqlTestMode", + kind: HostKind::Class, + edge: HostEdge::Facade, + }, + HostEntry { + name: "CursorAlgebraError", + kind: HostKind::Exception, + edge: HostEdge::Facade, + }, + HostEntry { + name: "SharedInputError", + kind: HostKind::Exception, + edge: HostEdge::Facade, + }, ]; impl FacadeHost { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 6df67839b2..1a40da4c06 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -449,7 +449,7 @@ fn resources_object<'py>( /// Return the resource facts and the resource Python's assembly raises at, if any. #[pyfunction] #[pyo3(signature = (*, inputs, dialect, syntax_checks))] -fn assemble_native_project_resources<'py>( +pub(crate) fn assemble_native_project_resources<'py>( inputs: &Bound<'py, PyAny>, dialect: Option, syntax_checks: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 3ba20fef2e..822dc5426d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -90,7 +90,7 @@ fn syntax_checks<'py>( /// Each model's `(sql, placeholders)` that assembly passes to syntax validation. #[pyfunction] #[pyo3(signature = (*, model_inputs, analysis_model_names, analysis_succeeded))] -fn model_syntax_checks<'py>( +pub(crate) fn model_syntax_checks<'py>( model_inputs: &Bound<'py, PyAny>, analysis_model_names: &Bound<'py, PyAny>, analysis_succeeded: &Bound<'py, PyAny>, @@ -105,7 +105,7 @@ fn model_syntax_checks<'py>( /// `(sql, families)` by model index for each pivot model no analysis covered. #[pyfunction] #[pyo3(signature = (*, model_inputs, analysed))] -fn standalone_pivot_models<'py>( +pub(crate) fn standalone_pivot_models<'py>( model_inputs: &Bound<'py, PyAny>, analysed: &Bound<'py, PyAny>, ) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs index f93d22b5f8..4ef70d6c7a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/relations.rs @@ -8,8 +8,9 @@ use sqlbuild_core::text::main::python_casefold::python_casefold; use crate::bindings::_helpers::analysis::relation_names::{ RelationRow, extract_sql_relation_names, }; +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; -const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; const RUNTIME_RELATION_PREFIX: &str = "SELECT 1 FROM "; fn keyword_call<'py>( @@ -31,9 +32,7 @@ fn relation_name<'py>( keywords.set_item("name", name)?; keywords.set_item("schema", schema)?; keywords.set_item("database", database)?; - py.import(REFERENCE_MODELS_MODULE)? - .getattr("RelationName")? - .call((), Some(&keywords)) + host::entry(py, ExecutorHost::RelationName)?.call((), Some(&keywords)) } /// Relations named in `sql` and temporary tables it creates, or None if unparseable. @@ -111,8 +110,7 @@ fn runtime_project_relations<'py>( relations: &Bound<'py, PyAny>, dialect: Option<&str>, ) -> PyResult> { - let models: Bound<'py, PyAny> = py.import(REFERENCE_MODELS_MODULE)?.into_any(); - let relation_type: Bound<'py, PyAny> = models.getattr("ProjectRelation")?; + let relation_type: Bound<'py, PyAny> = host::entry(py, ExecutorHost::ProjectRelation)?; let resolved: Bound<'py, PyDict> = PyDict::new(py); for item in relations.call_method0("items")?.try_iter()? { let (reference, qualified): (Bound<'py, PyAny>, String) = item?.extract()?; @@ -133,7 +131,7 @@ fn runtime_project_relations<'py>( resolved.set_item(relation, py.None())?; } keyword_call( - &models.getattr("ProjectRelationIndex")?, + &host::entry(py, ExecutorHost::ProjectRelationIndex)?, &[("relations", PyTuple::new(py, resolved.keys())?.into_any())], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs index 4aa6770210..fe0ba7ee75 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/relation_naming/fitting.rs @@ -7,12 +7,11 @@ use sqlbuild_analysis::identifier_fitting::main::fit_artifact_logical_name::fit_ use sqlbuild_analysis::identifier_fitting::main::fit_auxiliary_relation_name::fit_auxiliary_relation_name as fit_auxiliary; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::imported; - -const SHARED_EXCEPTIONS_MODULE: &str = "sqlbuild.errors.contracts.exceptions"; +use crate::bindings::_helpers::native_host::facade::FacadeHost; +use crate::bindings::_helpers::native_host::host; pub(crate) fn shared_error(py: Python<'_>, message: String) -> PyErr { - imported(py, SHARED_EXCEPTIONS_MODULE, "SharedInputError") + host::entry(py, FacadeHost::SharedInputError) .and_then(|class| class.call1((message,))) .map_or_else(|error| error, PyErr::from_value) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 963b752559..3e35ac8ac3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; @@ -16,12 +16,12 @@ use crate::bindings::_helpers::render::declaration_expansion::{ DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, expand_declarations, span_rows, }; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; use crate::bindings::_helpers::render::macro_expansion_result::expand_sql_macros; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; -const FUNCTOOLS_MODULE: &str = "functools"; const DECLARATION_FIELDS: [&str; 4] = [ "enums", "constants", @@ -111,17 +111,10 @@ fn declarations_expanded<'py>( return Ok((sql.to_owned(), Vec::new())); } let context: Bound<'py, PyAny> = declaration_context(py, declarations)?; - let render_constant: Bound<'py, PyAny> = imported(py, FUNCTOOLS_MODULE, "partial")?.call( - (registry::native(py, NativeAttr::RenderConstantDeclaration)?,), - Some(&keywords( - py, - &[ - ("value_renderer", value_renderer.clone()), - ("collection_rendering", collection_rendering.clone()), - ("file_path", file_path.clone()), - ], - )?), - )?; + let rendering: ConstantRendering = ConstantRendering::new( + value_renderer.clone().unbind(), + collection_rendering.clone().unbind(), + ); let classes = ExpansionClasses { usage_record: registry::class(py, RecordId::UsageRecord)?, compile_error: registry::error(py, ErrorId::CompileInput)?, @@ -137,7 +130,7 @@ fn declarations_expanded<'py>( relationship_visibility: false, path: &path, }, - render_constant: &render_constant, + rendering: &rendering, classes: &classes, })?; let spans: Vec = span_rows(&expanded.spliced); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 599a082a4d..57f1fa829d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -11,6 +11,8 @@ use sqlbuild_analysis::compiled_project::models::{ DeclarationSplice, InaccessibleDeclaration, SplicedSql, }; +use crate::bindings::_helpers::render::declaration_values::ConstantRendering; + const ENUM_REFERENCE_KIND: u8 = 0; const ENUM_KIND: &str = "enum"; const CONSTANT_KIND: &str = "constant"; @@ -57,14 +59,14 @@ pub(crate) struct ExpansionConsumer<'a, 'py> { pub(crate) path: &'a str, } -/// One expansion: the SQL, its scan, the visible declarations and the adapter's constant renderer. +/// One expansion: the SQL, its scan, the visible declarations and the adapter's constant rendering. pub(crate) struct DeclarationExpansion<'a, 'py> { pub(crate) sql: &'a str, pub(crate) references: &'a [ScannedReference], pub(crate) stop: Option, pub(crate) context: &'a Bound<'py, PyAny>, pub(crate) consumer: ExpansionConsumer<'a, 'py>, - pub(crate) render_constant: &'a Bound<'py, PyAny>, + pub(crate) rendering: &'a ConstantRendering, pub(crate) classes: &'a ExpansionClasses<'py>, } @@ -145,12 +147,9 @@ fn constant_text<'py>( name: &str, ) -> PyResult<(String, Bound<'py, PyAny>)> { let declaration: Bound<'py, PyAny> = visible_declaration(request, CONSTANT_KIND, name)?; - let arguments: Bound<'py, PyDict> = PyDict::new(declaration.py()); - arguments.set_item(DECLARATION, declaration)?; let rendered: String = request - .render_constant - .call((), Some(&arguments))? - .extract()?; + .rendering + .render(&declaration, request.consumer.path)?; Ok((rendered, visibility(request, CONSTANT_VISIBILITY, name)?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs index 6bfd127f0e..2f2f6c537c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_indexes.rs @@ -54,7 +54,7 @@ fn index_declarations<'py>( /// Every public enum and constant by name, collision-checked. #[pyfunction] -fn public_declaration_indexes<'py>( +pub(crate) fn public_declaration_indexes<'py>( inputs: Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { let py: Python<'py> = inputs.py(); @@ -166,7 +166,9 @@ fn lowered_name(column: &Bound<'_, PyAny>) -> PyResult { /// Every public model schema by name, with its inherited columns first, collision-checked. #[pyfunction] -fn public_model_schema_index<'py>(inputs: Bound<'py, PyAny>) -> PyResult> { +pub(crate) fn public_model_schema_index<'py>( + inputs: Bound<'py, PyAny>, +) -> PyResult> { let py: Python<'py> = inputs.py(); let mut authored: Vec<(String, Bound<'py, PyAny>)> = Vec::new(); let mut by_name: HashMap = HashMap::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 13b5c6e3a5..58f0c57528 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -1,10 +1,10 @@ //! Declaration scope resolvers, per-path declaration contexts and adapter-rendered constants. use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyInt, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyclass, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -371,7 +371,7 @@ fn rendered_constant<'py>( pub(crate) fn constant_declaration_sql<'py>( declaration: &Bound<'py, PyAny>, value_renderer: &Bound<'py, PyAny>, - (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), + (collection_rendering, file_path): (&Bound<'py, PyAny>, Option<&str>), ) -> PyResult { let py: Python<'py> = declaration.py(); let render_as: Bound<'py, PyAny> = declaration.getattr("render_as")?; @@ -392,16 +392,16 @@ pub(crate) fn constant_declaration_sql<'py>( return Err(error); } let relative_path: Bound<'py, PyAny> = declaration.getattr("relative_path")?; - let location: Bound<'py, PyAny> = match file_path { - Some(file_path) if !file_path.is_none() => file_path.clone(), - _ => relative_path.clone(), + let location: String = match file_path { + Some(file_path) => file_path.to_owned(), + None => relative_path.str()?.to_string(), }; let compile_error: Bound<'py, PyAny> = registry::error(py, ErrorId::CompileInput)?.call1((format!( "{} constant '{}' could not be rendered in '{}' by adapter '{}' as {}: {}", relative_path.str()?, declaration.getattr("name")?.str()?, - location.str()?, + location, value_renderer.getattr("adapter_name")?.str()?, selected.getattr("value")?.str()?, raised.str()?, @@ -419,13 +419,48 @@ fn render_constant_declaration<'py>( collection_rendering: &Bound<'py, PyAny>, file_path: Option>, ) -> PyResult { + let location: Option = match file_path { + Some(file_path) => Some(file_path.str()?.to_string()), + None => None, + }; constant_declaration_sql( declaration, value_renderer, - (collection_rendering, file_path.as_ref()), + (collection_rendering, location.as_deref()), ) } +/// The adapter's constant rendering a compile's declaration expansions share. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ConstantRendering { + value_renderer: Py, + collection_rendering: Py, +} + +impl ConstantRendering { + /// Rendering with the adapter's value renderer and the project's collection rendering. + pub(crate) fn new(value_renderer: Py, collection_rendering: Py) -> Self { + Self { + value_renderer, + collection_rendering, + } + } + + /// One visible constant as SQL, its failures located at `file_path`. + pub(crate) fn render( + &self, + declaration: &Bound<'_, PyAny>, + file_path: &str, + ) -> PyResult { + let py: Python<'_> = declaration.py(); + constant_declaration_sql( + declaration, + self.value_renderer.bind(py), + (self.collection_rendering.bind(py), Some(file_path)), + ) + } +} + /// One enum member value as a SQL scalar literal. #[pyfunction] #[pyo3(signature = (*, value))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs index 05a24599f4..4464779cb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_loading.rs @@ -6,12 +6,16 @@ use pyo3::exceptions::{PyImportError, PyOSError, PyUnicodeError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyBool, PyCFunction, PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_driver::inputs::{InputsRequest, attached_inputs}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_collection_rendering; use crate::bindings::_helpers::discovery::scope_snapshot::discover_scope_snapshot; use crate::bindings::_helpers::frontier::cache_directory::cache_directory; +use crate::bindings::_helpers::frontier::compile_frontier::{FrontierStage, bridged, produced}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; @@ -24,6 +28,7 @@ use crate::bindings::_helpers::scopes::scope_index::open_native_scope_index; use crate::bindings::_helpers::scopes::scope_tolerant::scope_tolerant_index; use crate::bindings::_helpers::scopes::scope_validation::scope_index_with_compile_usages; use crate::bindings::_helpers::sql_test_glue::constants::SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM; +use pyo3::IntoPyObjectExt; const SCOPE_CACHE_DIRECTORY_NAME: &str = "declaration-scopes-v2"; const SCOPE_CACHE_FILENAME: &str = "scope-index.json"; @@ -53,13 +58,11 @@ fn project_adapter<'py>( discover_keywords.set_item("extract_output_column_locations", false)?; let discovered: Bound<'py, PyAny> = registry::native(py, NativeAttr::DiscoverProjectInputs)? .call((), Some(&discover_keywords))?; - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ], - )?; + let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + ) + .and_then(|value| value.into_bound_py_any(py))?; let adapter: Bound<'py, PyAny> = call_with( &host::entry(py, AdapterHost::ResolveAdapter)?, &[("adapter_name", adapter_name)], @@ -75,13 +78,9 @@ fn compiled_index<'py>( ) -> PyResult> { let py: Python<'py> = discovered.py(); let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let collection_rendering: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveCollectionRendering)?, - &[ - ("project_config", project_config), - ("declaration_override", py.None().into_bound(py)), - ], - )?; + let collection_rendering: Bound<'py, PyAny> = + resolve_effective_collection_rendering(&project_config, &py.None().into_bound(py)) + .and_then(|value| value.into_bound_py_any(py))?; let adapter_context: Bound<'py, PyAny> = call_with( ®istry::class(py, RecordId::CompileAdapterContext)?, &[ @@ -94,26 +93,29 @@ fn compiled_index<'py>( ("sql_lexical_syntax", syntax.clone()), ], )?; - let stage_keywords: Bound<'py, PyDict> = PyDict::new(py); - stage_keywords.set_item("discovered_inputs", discovered)?; - stage_keywords.set_item("adapter_context", adapter_context)?; - stage_keywords.set_item("resolved_connection", PyDict::new(py))?; - stage_keywords.set_item("no_sql_validation", true)?; - stage_keywords.set_item("defer_model_sql_validation", true)?; - stage_keywords.set_item("no_cache", true)?; - let python_stage: Bound<'py, PyAny> = py.import("functools")?.getattr("partial")?.call( - (registry::native(py, NativeAttr::BuildCompileInputs)?,), - Some(&stage_keywords), + let inputs: InputsRequest = InputsRequest { + discovered_inputs: discovered.unbind(), + adapter_context: adapter_context.unbind(), + selected_target: py.None(), + cli_vars: py.None(), + run_id: py.None(), + no_sql_validation: PyBool::new(py, true).to_owned().into_any().unbind(), + defer_model_sql_validation: PyBool::new(py, true).to_owned().into_any().unbind(), + external_sql_reference_resolver: py.None(), + resolved_connection: PyDict::new(py).into_any().unbind(), + no_cache: PyBool::new(py, true).to_owned().into_any().unbind(), + }; + let stage: Bound<'py, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + move |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| { + attached_inputs(arguments.py(), &inputs).map(Bound::unbind) + }, )?; - let compile_inputs: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::CompileFrontier)?, - &[ - ( - "until", - registry::class(py, RecordId::CompilerStage)?.getattr("COMPILE_PROJECT_INPUTS")?, - ), - ("python_stage", python_stage), - ], + let compile_inputs: Bound<'py, PyAny> = produced( + FrontierStage::CompileProjectInputs, + bridged(stage.as_any())?, )?; scope_index_with_compile_usages(&compile_inputs) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index e800067b7b..4650dca5ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; -const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; /// `(kind, name, package, call_argument_count)` for one reference. @@ -179,18 +178,17 @@ pub(crate) fn extract_references<'py>( let (references, invalid_calls) = scanned(py, sql, syntax)?; let report: Bound<'py, PyAny> = registry::native(py, NativeAttr::ReportCompileDiagnostic)?; let diagnostic_type: Bound<'py, PyAny> = registry::class(py, RecordId::CompilerDiagnostic)?; - let types = py.import(COMPILE_TYPES_MODULE)?; for call in invalid_calls { let diagnostic: Bound<'py, PyAny> = call_with( &diagnostic_type, &[ ( "phase", - registry::class(types.py(), RecordId::DiagnosticPhase)?.getattr("COMPILE")?, + registry::class(py, RecordId::DiagnosticPhase)?.getattr("COMPILE")?, ), ( "severity", - registry::class(types.py(), RecordId::DiagnosticSeverity)?.getattr("ERROR")?, + registry::class(py, RecordId::DiagnosticSeverity)?.getattr("ERROR")?, ), ("code", text(py, REFERENCE_CALL_SYNTAX_CODE)?), ("message", call.getattr("message")?), diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 3e22a919a5..c5c4bc144c 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -290,6 +290,9 @@ "sqlbuild.runtime.event_exporting.main.get_lifecycle_event_sink_definition", "get_lifecycle_event_sink_definition", ), + "ProjectRelation": ("sqlbuild.compiler.references.models", "ProjectRelation"), + "ProjectRelationIndex": ("sqlbuild.compiler.references.models", "ProjectRelationIndex"), + "RelationName": ("sqlbuild.compiler.references.models", "RelationName"), "CostResourceContext": ("sqlbuild.cost.models", "CostResourceContext"), "RunCostSummary": ("sqlbuild.cost.models", "RunCostSummary"), "CostCapability": ("sqlbuild.cost.types", "CostCapability"), @@ -714,6 +717,16 @@ ), "startup_inputs": ("sqlbuild.cli.compile_reuse._helpers.startup_inputs", "startup_inputs"), "CompileReuseRequest": ("sqlbuild.cli.compile_reuse.models", "CompileReuseRequest"), + "AuditEvaluationMode": ("sqlbuild.compiler.auditing.types", "AuditEvaluationMode"), + "AuditOutcome": ("sqlbuild.compiler.auditing.types", "AuditOutcome"), + "AuditSeverity": ("sqlbuild.compiler.auditing.types", "AuditSeverity"), + "ThresholdOperator": ("sqlbuild.compiler.auditing.types", "ThresholdOperator"), + "AttachedAuditTargetKind": ("sqlbuild.compiler.compile.types", "AttachedAuditTargetKind"), + "CompiledResourceType": ("sqlbuild.compiler.compile.types", "CompiledResourceType"), + "FunctionLanguage": ("sqlbuild.compiler.compile.types", "FunctionLanguage"), + "SqlTestMode": ("sqlbuild.compiler.compile.types", "SqlTestMode"), + "CursorAlgebraError": ("sqlbuild.cursor_algebra.exceptions", "CursorAlgebraError"), + "SharedInputError": ("sqlbuild.errors.contracts.exceptions", "SharedInputError"), } __all__ = ("host_targets", *_TARGETS) diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 562d33924e..2a97ba8825 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -83,6 +83,10 @@ from sqlbuild.cli.compile_reuse._helpers.provider_settings import ( ) from sqlbuild.cli.compile_reuse._helpers.startup_inputs import startup_inputs as startup_inputs from sqlbuild.cli.compile_reuse.models import CompileReuseRequest as CompileReuseRequest +from sqlbuild.compiler.auditing.types import AuditEvaluationMode as AuditEvaluationMode +from sqlbuild.compiler.auditing.types import AuditOutcome as AuditOutcome +from sqlbuild.compiler.auditing.types import AuditSeverity as AuditSeverity +from sqlbuild.compiler.auditing.types import ThresholdOperator as ThresholdOperator from sqlbuild.compiler.compile._helpers.diagnostics.python_resources import ( python_resource_rows as python_resource_rows, ) @@ -98,6 +102,10 @@ from sqlbuild.compiler.compile._helpers.render.user_macros import ( from sqlbuild.compiler.compile._helpers.render.user_macros import ( macro_invocation_context as macro_invocation_context, ) +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind as AttachedAuditTargetKind +from sqlbuild.compiler.compile.types import CompiledResourceType as CompiledResourceType +from sqlbuild.compiler.compile.types import FunctionLanguage as FunctionLanguage +from sqlbuild.compiler.compile.types import SqlTestMode as SqlTestMode from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( discover_command_output_sink_functions as discover_command_output_sink_functions, ) @@ -172,10 +180,14 @@ from sqlbuild.compiler.python_nodes.main._hook_identities import ( ) from sqlbuild.compiler.python_nodes.types import PythonNodeKind as PythonNodeKind from sqlbuild.compiler.python_nodes.types import PythonNodeStatus as PythonNodeStatus +from sqlbuild.compiler.references.models import ProjectRelation as ProjectRelation +from sqlbuild.compiler.references.models import ProjectRelationIndex as ProjectRelationIndex +from sqlbuild.compiler.references.models import RelationName as RelationName from sqlbuild.cost.models import CostResourceContext as CostResourceContext from sqlbuild.cost.models import RunCostSummary as RunCostSummary from sqlbuild.cost.types import CostCapability as CostCapability from sqlbuild.cost.types import CostStatus as CostStatus +from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError as CursorAlgebraError from sqlbuild.cursor_algebra.types import BoundSentinel as BoundSentinel from sqlbuild.diagnostics._helpers.process_resources import ( read_process_resources as read_process_resources, @@ -189,6 +201,7 @@ from sqlbuild.diagnostics.main.process_resource_reporting import ( ) from sqlbuild.diagnostics.models import ProcessResourceUsage as ProcessResourceUsage from sqlbuild.errors.contracts.exceptions import ExecutorInputError as ExecutorInputError +from sqlbuild.errors.contracts.exceptions import SharedInputError as SharedInputError from sqlbuild.executor.auditing.main._current_audit_result_projection import ( current_audit_result_projection as current_audit_result_projection, ) From d28088498aea0b3c94dbce4080dfed232119a802 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:04:48 +0100 Subject: [PATCH 02/12] refactor(compile): read attached schemas and references of model inputs natively Declared relations, standalone pivot checks, native resource facts and cursor roles read a native model input's attached schema and reference rows from its owner record instead of building its SchemaModelEntry and CompileSqlReference projections; hand-built inputs still read their record fields. --- .../compile_attachment/cursor_roles.rs | 5 +- .../compiled_project/model_input_view.rs | 64 +++++++++++++++++++ .../project_assembly/declared_relations.rs | 11 ++-- .../project_assembly/native_resources.rs | 14 ++-- .../project_assembly/syntax_checks.rs | 8 +-- 5 files changed, 81 insertions(+), 21 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs index fd1f70a8ff..39430e2020 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs @@ -5,6 +5,7 @@ use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -84,8 +85,8 @@ pub(crate) fn resolve_cursor_input_roles<'py>( Err(_) => { let cursor: Bound<'py, PyAny> = values.call_method1("get", ("cursor",))?; if cursor.is_instance_of::() { - for reference in model.getattr("references")?.try_iter()? { - filter.set_item(reference?.getattr("ref_name")?, &cursor)?; + for (_, name, _, _) in input_reference_rows(model)? { + filter.set_item(name, &cursor)?; } } if !rolling { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index dc0789f477..21c8b27c84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -447,3 +447,67 @@ fn expansion_object( fields.set_item("passes", PyTuple::new(py, pass_objects)?)?; Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) } + +/// A model input's attached schema: its name, columns and dynamic column families. +pub(crate) struct AttachedSchema<'py> { + pub(crate) name: Bound<'py, PyAny>, + pub(crate) columns: Bound<'py, PyAny>, + pub(crate) dynamic_columns: Bound<'py, PyAny>, +} + +/// The attached schema of `model_input`; a native input answers without building its entry. +pub(crate) fn attached_schema<'py>( + model_input: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native.get().read(py, |record, py| { + let Some(schema) = &record.schema else { + return Ok(None); + }; + Ok(Some(AttachedSchema { + name: record + .model_file + .bind(py) + .getattr("file_path")? + .getattr("stem")?, + columns: schema.columns.bind(py).clone().into_any(), + dynamic_columns: schema.dynamic_columns.bind(py).clone().into_any(), + })) + }); + } + let entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; + if entry.is_none() { + return Ok(None); + } + Ok(Some(AttachedSchema { + name: entry.getattr("name")?, + columns: entry.getattr("columns")?, + dynamic_columns: entry.getattr("dynamic_columns")?, + })) +} + +/// Each reference of `model_input` as `(kind, name, package, call argument count)`. +pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native + .get() + .read(py, |record, _| Ok(record.references.clone())); + } + model_input + .getattr("references")? + .try_iter()? + .map(|reference| reference_row_of(&reference?)) + .collect() +} + +/// One `CompileSqlReference` record as its row. +fn reference_row_of(reference: &Bound<'_, PyAny>) -> PyResult { + Ok(( + reference.getattr("ref_kind")?.str()?.extract()?, + reference.getattr("ref_name")?.extract()?, + reference.getattr("ref_package")?.extract()?, + reference.getattr("call_argument_count")?.extract()?, + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index 37cc938dd0..557af6ff50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -4,6 +4,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyBool, PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -72,12 +73,10 @@ fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult = PyDict::new(py); for model in inputs.getattr("model_inputs")?.try_iter()? { let model: Bound<'py, PyAny> = model?; - let entry: Bound<'py, PyAny> = model.getattr("schema_entry")?; - if entry.is_none() { + let Some(schema) = attached_schema(&model)? else { continue; - } - let name: Bound<'py, PyAny> = entry.getattr("name")?; - let columns: Bound<'py, PyAny> = entry.getattr("columns")?; + }; + let (name, columns): (Bound<'py, PyAny>, Bound<'py, PyAny>) = (schema.name, schema.columns); types.set_item(&name, column_types(&columns, false)?)?; nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; let contract: Option> = model @@ -89,7 +88,7 @@ fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult contract.eq(ENFORCED_CONTRACT)?, None => false, }; - if enforced && columns.is_truthy()? && !entry.getattr("dynamic_columns")?.is_truthy()? { + if enforced && columns.is_truthy()? && !schema.dynamic_columns.is_truthy()? { let stem: Bound<'py, PyAny> = model .getattr("model_file")? .getattr("file_path")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 1a40da4c06..5958548a0f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -19,6 +19,7 @@ use sqlbuild_model_config::templates::models::Scalar; use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::project_assembly::models::{ NativeProjectResources, NativeResourceFailure, @@ -162,15 +163,10 @@ fn tags(value: &Bound<'_, PyAny>) -> PyResult> { fn model_facts(model_input: &Bound<'_, PyAny>, checks: &Bound<'_, PyAny>) -> PyResult { let model_file: Bound<'_, PyAny> = model_input.getattr("model_file")?; - let mut reference_rows: Vec = Vec::new(); - for reference in model_input.getattr("references")?.try_iter()? { - let reference: Bound<'_, PyAny> = reference?; - reference_rows.push(( - reference.getattr("ref_kind")?.str()?.extract::()?, - reference.getattr("ref_name")?.extract()?, - reference.getattr("ref_package")?.extract()?, - )); - } + let reference_rows: Vec = input_reference_rows(model_input)? + .into_iter() + .map(|(kind, name, package, _)| (kind, name, package)) + .collect(); let mut syntax_checks: Vec = Vec::new(); for check in checks.try_iter()? { let (sql, placeholders): (String, Bound<'_, PyAny>) = check?.extract()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs index 822dc5426d..b63b199356 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs @@ -6,6 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; +use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::project_assembly::syntax_validation::hook_sql_statements; const HOOK_NAMES: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -113,11 +114,10 @@ pub(crate) fn standalone_pivot_models<'py>( let models: Bound<'py, PyDict> = PyDict::new(py); for (index, model_input) in model_inputs.try_iter()?.enumerate() { let model_input: Bound<'py, PyAny> = model_input?; - let schema_entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; - if schema_entry.is_none() { + let Some(schema) = attached_schema(&model_input)? else { continue; - } - let families: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; + }; + let families: Bound<'py, PyAny> = schema.dynamic_columns; if !families.is_truthy()? || analysed.contains(model_name(&model_input)?)? { continue; } From c6f19e906b3fb1b3a1a00e134b81c199ae3ac05d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:15:45 +0100 Subject: [PATCH 03/12] refactor(compile): compare header column locations with the model file's own path The schema facts of a model's header columns compare each column location against the discovered model file's relative path object instead of a pathlib.Path built per model, and skip the Python path comparison when the location carries that same object. --- .../_helpers/compiled_project/model_loop_contracts.rs | 6 ++++-- .../bindings/_helpers/compiled_project/model_loop_header.rs | 5 +---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs index fbc622549d..5ba537a17c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs @@ -231,8 +231,10 @@ pub(crate) fn schema_row( let mut column_rows: Vec = Vec::with_capacity(columns.len()); for column in columns.iter() { let location: Bound<'_, PyAny> = column.getattr(LOCATION_FIELD)?; - let declared_outside: bool = - !location.is_none() && location.getattr(PATH_FIELD)?.ne(model_path)?; + let declared_outside: bool = !location.is_none() && { + let path: Bound<'_, PyAny> = location.getattr(PATH_FIELD)?; + !path.is(model_path) && path.ne(model_path)? + }; column_rows.push(( column.getattr(NAME_FIELD)?.extract()?, column.getattr(TYPE_FIELD)?.extract()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index f73e9a239a..3b32a577ca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -155,10 +155,7 @@ pub(crate) fn header_schema<'py>( enum_columns(context, (&enums, source.visible), source.values, &columns)?; let fact: ModelSchemaFact = schema_fact(schema_row( &resolved.columns, - ( - audits.len(), - &context.classes.path_class.call1((source.relative_path,))?, - ), + (audits.len(), source.model_file.paths(py).1), &families, (type_enforcement, source.schema.is_some()), )?); From 85c299f0cad157da26ab829199bcec0cdcd733d3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:01:14 +0100 Subject: [PATCH 04/12] refactor(analysis): read attached schemas and audit references natively Dynamic-family indexes, model analysis rows and resource SQL checks read a native model input's attached schema and a native audit's reference rows from the owner record instead of building SchemaModelEntry and CompileSqlReference projections. --- .../analysis_session/model_analyses.rs | 16 ++++--- .../compile_driver/assembly_analysis.rs | 11 +++-- .../_helpers/compiled_project/audit_views.rs | 18 ++++++++ .../compiled_project/model_input_view.rs | 3 ++ .../_helpers/semantic_checks/resource_sql.rs | 43 ++++++++++++++----- 5 files changed, 69 insertions(+), 22 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index b6d43b56f8..c15924e2b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -13,6 +13,9 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_s use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + AttachedSchema, attached_schema, +}; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -299,9 +302,11 @@ fn model_rows<'py>( row.get_item(1)?.extract()?, )); } - let entry: Bound<'py, PyAny> = model_input.getattr("schema_entry")?; + let schema: Option> = attached_schema(model_input)?; let recovers: bool = get("contract")?.eq(&enforced)? - || (!entry.is_none() && entry.getattr("type_enforcement")?.is_truthy()?); + || schema + .as_ref() + .is_some_and(|schema| schema.type_enforcement); let snapshot_columns: Option<(String, String)> = if get("materialized")?.eq(&snapshot)? { Some(( or_default(&get("valid_from_column")?, VALID_FROM_DEFAULT)?, @@ -310,10 +315,9 @@ fn model_rows<'py>( } else { None }; - let families = if entry.is_none() { - Vec::new() - } else { - family_rows(&entry.getattr("dynamic_columns")?)? + let families = match &schema { + Some(schema) => family_rows(&schema.dynamic_columns)?, + None => Vec::new(), }; let row: Vec> = vec![ model_input diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index b1c5cf17fd..df161e7c3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::compile_driver::compile_timings::{ record_compile_cpu_timing, record_compile_timing, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -120,13 +121,11 @@ fn dynamic_families_by_table<'py>(inputs: &Bound<'py, PyAny>) -> PyResult = model_input.getattr("schema_entry")?; - if schema_entry.is_none() { + let Some(schema) = attached_schema(&model_input)? else { continue; - } - let dynamic_columns: Bound<'py, PyAny> = schema_entry.getattr("dynamic_columns")?; - if dynamic_columns.is_truthy()? { - families.set_item(schema_entry.getattr("name")?, dynamic_columns)?; + }; + if schema.dynamic_columns.is_truthy()? { + families.set_item(schema.name, schema.dynamic_columns)?; } } Ok(families) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs index 3f7ea723c5..0429bd3606 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs @@ -392,3 +392,21 @@ audit_getters!(NativeCompiledAudit, { }) } }); + +/// A native audit view's reference rows, or `None` for a hand-built audit record. +pub(crate) fn audit_reference_rows( + audit: &Bound<'_, PyAny>, +) -> PyResult>> { + let py: Python<'_> = audit.py(); + let (owner, index): (&Py, usize) = + if let Ok(view) = audit.cast::() { + (&view.get().owner, view.get().index) + } else if let Ok(view) = audit.cast::() { + (&view.get().owner, view.get().index) + } else { + return Ok(None); + }; + read_audit(owner, index, py, |record, _| { + Ok(Some(record.attached.references.clone())) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index 21c8b27c84..4acc68f18a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -453,6 +453,7 @@ pub(crate) struct AttachedSchema<'py> { pub(crate) name: Bound<'py, PyAny>, pub(crate) columns: Bound<'py, PyAny>, pub(crate) dynamic_columns: Bound<'py, PyAny>, + pub(crate) type_enforcement: bool, } /// The attached schema of `model_input`; a native input answers without building its entry. @@ -473,6 +474,7 @@ pub(crate) fn attached_schema<'py>( .getattr("stem")?, columns: schema.columns.bind(py).clone().into_any(), dynamic_columns: schema.dynamic_columns.bind(py).clone().into_any(), + type_enforcement: schema.type_enforcement, })) }); } @@ -484,6 +486,7 @@ pub(crate) fn attached_schema<'py>( name: entry.getattr("name")?, columns: entry.getattr("columns")?, dynamic_columns: entry.getattr("dynamic_columns")?, + type_enforcement: entry.getattr("type_enforcement")?.is_truthy()?, })) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 2b743d310b..00a9cddbbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -14,6 +14,8 @@ use sqlbuild_analysis::semantic_checks::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compiled_project::audit_views::audit_reference_rows; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::models::ProjectCatalog; @@ -162,7 +164,10 @@ fn audit_input( contents: Arc::from(file.getattr("contents")?.extract::()?), bodies, opts_out: opts_out(&block.getattr("header_values")?)?, - relations: relations(&audit.getattr("references")?, reference_class)?, + relations: match audit_reference_rows(audit)? { + Some(rows) => row_relations(rows), + None => relations(&audit.getattr("references")?, reference_class)?, + }, }) } @@ -253,6 +258,29 @@ fn opts_out(values: &Bound<'_, PyAny>) -> PyResult { .is(PyBool::new(values.py(), false))) } +/// `sorted(binding_relation_names(...))` of a native view's reference rows. +fn row_relations(rows: Vec) -> Vec { + let mut names: Vec = rows + .into_iter() + .filter_map(|(kind, name, _, _)| relation_name(&kind, name)) + .collect(); + names.sort_unstable(); + names.dedup(); + names +} + +/// The binding relation a reference of `kind` to `name` reads, or `None` for a UDF call. +fn relation_name(kind: &str, name: String) -> Option { + if kind == UDF_KIND { + return None; + } + Some(if kind == TABLE_FUNCTION_KIND { + format!("{TABLE_FUNCTION_STUB_PREFIX}{name}") + } else { + name + }) +} + /// `sorted(binding_relation_names(...))` of the `CompileSqlReference`s among `references`. fn relations( references: &Bound<'_, PyAny>, @@ -265,15 +293,10 @@ fn relations( continue; } let kind: String = reference.getattr("ref_kind")?.str()?.extract()?; - let name: String = reference.getattr("ref_name")?.extract()?; - if kind == UDF_KIND { - continue; - } - names.push(if kind == TABLE_FUNCTION_KIND { - format!("{TABLE_FUNCTION_STUB_PREFIX}{name}") - } else { - name - }); + names.extend(relation_name( + &kind, + reference.getattr("ref_name")?.extract()?, + )); } names.sort_unstable(); names.dedup(); From 58ebd1a950ef35e7271dd49b7066af05737a3ba0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:56:03 +0100 Subject: [PATCH 05/12] refactor(rules): parse DSL headers natively for header rules and formatting Header rules, header formatting and table-function argument names parse DSL header values with the native header parser and compare them as their Python projections compare, instead of calling parse_header_values through Python; the formatter no longer warms the Python header cache. --- .../bindings/_helpers/format_command/run.rs | 4 - .../src/bindings/_helpers/records/ids.rs | 1 - .../bindings/_helpers/rules/dsl_headers.rs | 154 ++--------- .../src/bindings/_helpers/rules/mod.rs | 1 + .../_helpers/rules/native_header_parser.rs | 243 ++++++++++++++++++ .../_helpers/sqltext/model_headers.rs | 14 +- src/sqlbuild/_native.pyi | 1 - 7 files changed, 277 insertions(+), 141 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index e1df98565a..ad821956c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -100,10 +100,6 @@ fn apply_fixes<'py>( return Ok((updated, faults)); } let current: Bound<'py, PyDict> = overlaid(files, &updated)?; - call_with( - &native(py, NativeAttr::PrepareNativeHeaderCache)?, - &[("files", current.clone().into_any())], - )?; for (path, contents) in sorted_items(¤t)? { let result: Bound<'py, PyAny> = call_with( &native(py, NativeAttr::FormatNativeHeaders)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index e23a03ebc1..ffe8ca87bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -1042,7 +1042,6 @@ native_ids! { PrepareLintSql => "prepare_lint_sql", PrepareLintSqlBatch => "prepare_lint_sql_batch", PrepareModelHeaderTokens => "prepare_model_header_tokens", - PrepareNativeHeaderCache => "prepare_native_header_cache", PrepareSqlRules => "prepare_sql_rules", PreparedCompileArtifacts => "PreparedCompileArtifacts", ProjectLineageViews => "project_lineage_views", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs index 41f064c542..12528c17f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs @@ -1,10 +1,6 @@ //! DSL header scanning, header lint rules and header formatting fixes over Python lint models. -use std::cell::RefCell; - -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::dsl_headers::constants::RULE_HEADER_PARSE; @@ -15,108 +11,23 @@ use sqlbuild_rules::dsl_headers::main::measurement_body_ranges::measurement_body use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers as scan; use sqlbuild_rules::dsl_headers::main::sql_body_ranges::sql_body_ranges as body_ranges; use sqlbuild_rules::dsl_headers::models::{ - HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, ParseFailure, ParseOutcome, + HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, ParseOutcome, }; use sqlbuild_rules::dsl_headers::types::HeaderParser; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::native_header_parser::{ + NativeHeaderParser, map_keys, map_value, +}; +use sqlbuild_sqltext::compiler::models::AuthoredValue; -const PATHLIB_MODULE: &str = "pathlib"; -const SENTINEL_PATH: &str = ""; const ENGINE: &str = "sqlbuild"; const LINE_FEED: &str = "\n"; const CARRIAGE_RETURN_LINE_FEED: &str = "\r\n"; const FUNCTION_KIND: &str = "FUNCTION"; - -/// Header values parsed by `parse_header_values`, with Python's error classes. -struct PyHeaderParser<'py> { - parse: Bound<'py, PyAny>, - path: Bound<'py, PyAny>, - syntax_error: Bound<'py, PyAny>, - discovery_error: Bound<'py, PyAny>, - /// The header syntax error of the last rejected parse. - rejected: RefCell>, -} - -impl<'py> PyHeaderParser<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - parse: registry::native(py, NativeAttr::ParseHeaderValues)?, - path: imported(py, PATHLIB_MODULE, "Path")?.call1((SENTINEL_PATH,))?, - syntax_error: registry::error(py, ErrorId::ModelSqlParse)?, - discovery_error: registry::error(py, ErrorId::Discovery)?, - rejected: RefCell::new(None), - }) - } - - fn failure(&self, error: &PyErr) -> PyResult { - let py: Python<'py> = self.parse.py(); - let value: Bound<'py, PyAny> = error.value(py).clone().into_any(); - let help: Option = if value.is_instance(&self.discovery_error)? { - value.getattr("help")?.extract()? - } else { - None - }; - Ok(ParseFailure { - message: value.str()?.extract()?, - help, - }) - } -} - -impl<'py> HeaderParser for PyHeaderParser<'py> { - type Values = Bound<'py, PyAny>; - type Error = PyErr; - - fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { - let py: Python<'py> = self.parse.py(); - let parsed: PyResult> = call_with( - &self.parse, - &[ - ("header", PyString::new(py, inner).into_any()), - ("file_path", self.path.clone()), - ("statement_name", PyString::new(py, kind).into_any()), - ("header_line", py.None().into_bound(py)), - ], - ); - let error: PyErr = match parsed { - Ok(values) => return ParseOutcome::Parsed(values), - Err(error) => error, - }; - let failure: ParseFailure = match self.failure(&error) { - Ok(failure) => failure, - Err(failed) => { - return ParseOutcome::Raised( - failed, - ParseFailure { - message: String::new(), - help: None, - }, - ); - } - }; - if error.is_instance(py, &self.syntax_error) { - self.rejected.replace(Some(error)); - ParseOutcome::Rejected(failure) - } else { - ParseOutcome::Raised(error, failure) - } - } - - fn description(&self, values: &Self::Values) -> PyResult> { - let description: Bound<'py, PyAny> = values.call_method1("get", ("description",))?; - if description.is_instance_of::() { - return description.extract().map(Some); - } - Ok(None) - } - - fn same(&self, left: &Self::Values, right: &Self::Values) -> PyResult { - left.eq(right) - } -} +const ARGUMENTS_KEY: &str = "arguments"; /// The models and enums header results are built as. struct LintClasses<'py> { @@ -251,7 +162,7 @@ fn header_violations<'py>( headers: &[HeaderSpan], config: &Bound<'py, PyAny>, ) -> PyResult> { - let parser: PyHeaderParser<'py> = PyHeaderParser::new(config.py())?; + let parser: NativeHeaderParser<'py> = NativeHeaderParser::new(config.py()); lint_headers(contents, headers, rules_config(config)?, &parser) } @@ -279,7 +190,7 @@ fn format_native_headers<'py>( config: &Bound<'py, PyAny>, ) -> PyResult<(String, Bound<'py, PyTuple>)> { let py: Python<'py> = contents.py(); - let parser: PyHeaderParser<'py> = PyHeaderParser::new(py)?; + let parser: NativeHeaderParser<'py> = NativeHeaderParser::new(py); let (formatted, faults) = format_headers(&text(contents)?, rules_config(config)?, &parser)?; let classes: LintClasses<'py> = LintClasses::new(py)?; Ok(( @@ -303,25 +214,6 @@ fn parse_faults<'py>( classes.violations(file_path, violations) } -/// Batch parse formatter-owned headers before per-file formatting and linting. -#[pyfunction] -#[pyo3(signature = (*, files))] -fn prepare_native_header_cache(files: &Bound<'_, PyDict>) -> PyResult<()> { - let mut headers: Vec = Vec::new(); - for contents in files.values() { - let contents: Vec = text(&contents)?; - for header in scan(&contents, false) { - headers.push( - inner_header_text(&contents[header.start..header.end]) - .iter() - .collect(), - ); - } - } - registry::native(files.py(), NativeAttr::PrepareModelHeaderTokens)?.call1((headers,))?; - Ok(()) -} - /// Parse-safe files, and every file's authored newline convention. #[pyfunction] #[pyo3(signature = (*, files, config))] @@ -389,7 +281,7 @@ fn external_identifiers_for_headers<'py>( ) -> PyResult> { let py: Python<'py> = contents.py(); let text: Vec = text(contents)?; - let parser: PyHeaderParser<'py> = PyHeaderParser::new(py)?; + let parser: NativeHeaderParser<'py> = NativeHeaderParser::new(py); for header in spans(headers)? { if header.kind != FUNCTION_KIND { continue; @@ -397,22 +289,21 @@ fn external_identifiers_for_headers<'py>( let inner: String = inner_header_text(&text[header.start..header.end]) .iter() .collect(); - let values: Bound<'py, PyAny> = match parser.parse(&header.kind, &inner) { + let values: AuthoredValue = match parser.parse(&header.kind, &inner) { ParseOutcome::Parsed(values) => values, - ParseOutcome::Rejected(_) => { - return Err(parser.rejected.take().unwrap_or_else(|| PyErr::fetch(py))); - } + ParseOutcome::Rejected(_) => return Err(parser.rejected_error()), ParseOutcome::Raised(error, _) => return Err(error), }; - let arguments: Bound<'py, PyAny> = values.call_method1("get", ("arguments",))?; - let Ok(arguments) = arguments.downcast::() else { + let Some(arguments) = map_value(&values, ARGUMENTS_KEY) else { continue; }; - let mut names: Vec = arguments - .keys() - .iter() - .map(|name| name.str()?.call_method0("lower")?.extract()) - .collect::>()?; + if !matches!(arguments, AuthoredValue::Map(_)) { + continue; + } + let mut names: Vec = map_keys(arguments) + .into_iter() + .map(str::to_lowercase) + .collect(); names.sort_unstable(); return PyTuple::new(py, names); } @@ -425,7 +316,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(measurement_body_ranges, module)?)?; module.add_function(wrap_pyfunction!(lint_native_headers, module)?)?; module.add_function(wrap_pyfunction!(format_native_headers, module)?)?; - module.add_function(wrap_pyfunction!(prepare_native_header_cache, module)?)?; module.add_function(wrap_pyfunction!(safe_format_files, module)?)?; module.add_function(wrap_pyfunction!( reject_unparseable_header_rewrites, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 5c3ad0aa26..c2a3ac6ee2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod lint_preparation; pub(crate) mod lint_sql_bodies; pub(crate) mod lint_stop; pub(crate) mod lint_text; +pub(crate) mod native_header_parser; pub(crate) mod project_tree_queries; pub(crate) mod registration; pub(crate) mod relation_keys; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs new file mode 100644 index 0000000000..fc4c16a1f9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs @@ -0,0 +1,243 @@ +//! DSL header values parsed natively for header rules, compared as Python compares their objects. + +use std::cell::RefCell; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_rules::dsl_headers::models::{ParseFailure, ParseOutcome}; +use sqlbuild_rules::dsl_headers::types::HeaderParser; +use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; +use sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch; +use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::records::ids::ErrorId; +use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sqltext::model_headers::header_failure; +use crate::bindings::types::CompilerDetach; + +/// The path header rules report header syntax errors at. +const SENTINEL_PATH: &str = ""; +const DESCRIPTION_KEY: &str = "description"; +const NEGATIVE_ZERO: &str = "-0"; +const ZERO: &str = "0"; + +/// Header values parsed natively; a rejected parse keeps its message for raising. +pub(crate) struct NativeHeaderParser<'py> { + py: Python<'py>, + rejected: RefCell>, +} + +impl<'py> NativeHeaderParser<'py> { + pub(crate) fn new(py: Python<'py>) -> Self { + Self { + py, + rejected: RefCell::new(None), + } + } + + /// The `ModelSqlParseError` of the last rejected parse. + pub(crate) fn rejected_error(&self) -> PyErr { + let Some(failure) = self.rejected.take() else { + return PyValueError::new_err("no header parse was rejected"); + }; + let raised: PyResult = (|| { + let keywords: Bound<'py, PyDict> = PyDict::new(self.py); + keywords.set_item("help", failure.help)?; + let error: Bound<'py, PyAny> = registry::error(self.py, ErrorId::ModelSqlParse)? + .call((failure.message,), Some(&keywords))?; + Ok(PyErr::from_value(error)) + })(); + raised.unwrap_or_else(|error| error) + } +} + +impl HeaderParser for NativeHeaderParser<'_> { + type Values = AuthoredValue; + type Error = PyErr; + + fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { + let headers: Vec = vec![inner.to_owned()]; + let parsed = match self.py.compiler_detach(|| parse_batch(&headers)) { + Ok(parsed) => parsed, + Err(message) => { + return ParseOutcome::Raised( + value_error(message.clone()), + ParseFailure { + message, + help: None, + }, + ); + } + }; + let (value, _, error) = parsed.into_iter().next().unwrap_or((None, None, None)); + if let Some(value) = value { + return ParseOutcome::Parsed(value); + } + let error: String = error.unwrap_or_else(|| "None".to_owned()); + let (detail, help, _) = header_failure(&error, inner, None); + let failure: ParseFailure = ParseFailure { + message: format!( + "{kind}(...) in '{SENTINEL_PATH}' contains invalid SQLBuild header syntax: {detail}" + ), + help, + }; + self.rejected.replace(Some(ParseFailure { + message: failure.message.clone(), + help: failure.help.clone(), + })); + ParseOutcome::Rejected(failure) + } + + fn description(&self, values: &AuthoredValue) -> PyResult> { + Ok(match map_value(values, DESCRIPTION_KEY) { + Some(AuthoredValue::String(text)) => Some(text.clone()), + Some(AuthoredValue::BareWord(word)) if header_word_value(word) == HeaderWord::Text => { + Some(word.clone()) + } + _ => None, + }) + } + + fn same(&self, left: &AuthoredValue, right: &AuthoredValue) -> PyResult { + Ok(python_equal(left, right)) + } +} + +/// The value a header map projects for `key`: its last entry, as a Python dict keeps it. +pub(crate) fn map_value<'a>(values: &'a AuthoredValue, key: &str) -> Option<&'a AuthoredValue> { + match values { + AuthoredValue::Map(entries) => entries + .iter() + .rev() + .find(|(name, _)| name == key) + .map(|(_, value)| value), + _ => None, + } +} + +/// A header map's keys once each, as the projected Python dict holds them. +pub(crate) fn map_keys(values: &AuthoredValue) -> Vec<&str> { + let mut keys: Vec<&str> = Vec::new(); + if let AuthoredValue::Map(entries) = values { + for (name, _) in entries { + if !keys.contains(&name.as_str()) { + keys.push(name); + } + } + } + keys +} + +/// A scalar as its projected Python object compares. +#[derive(Debug, PartialEq)] +enum Scalar<'a> { + None, + Integer(i128), + BigInteger(String), + Float(f64), + Text(&'a str), +} + +fn scalar(value: &AuthoredValue) -> Option> { + Some(match value { + AuthoredValue::Null => Scalar::None, + AuthoredValue::Boolean(flag) => Scalar::Integer(i128::from(*flag)), + AuthoredValue::String(text) => Scalar::Text(text), + AuthoredValue::BareWord(word) => match header_word_value(word) { + HeaderWord::Bool(flag) => Scalar::Integer(i128::from(flag)), + HeaderWord::Null => Scalar::None, + HeaderWord::Integer(digits) => match digits.parse::() { + Ok(number) => Scalar::Integer(number), + Err(_) => Scalar::BigInteger(digits), + }, + HeaderWord::Float(number) => Scalar::Float(number), + HeaderWord::Text => Scalar::Text(word), + }, + _ => return None, + }) +} + +fn scalars_equal(left: &Scalar<'_>, right: &Scalar<'_>) -> bool { + match (left, right) { + (Scalar::Integer(integer), Scalar::Float(float)) + | (Scalar::Float(float), Scalar::Integer(integer)) => { + integral_float_text(*float).is_some_and(|text| text == integer.to_string()) + } + (Scalar::BigInteger(digits), Scalar::Float(float)) + | (Scalar::Float(float), Scalar::BigInteger(digits)) => { + integral_float_text(*float).is_some_and(|text| text == digits.trim_start_matches('+')) + } + (Scalar::Float(left), Scalar::Float(right)) => left == right, + (left, right) => left == right, + } +} + +/// The exact decimal digits of an integral finite float, as Python compares it with an int. +fn integral_float_text(float: f64) -> Option { + if !float.is_finite() || float.trunc() != float { + return None; + } + let text: String = format!("{float:.0}"); + Some(if text == NEGATIVE_ZERO { + ZERO.to_owned() + } else { + text + }) +} + +/// Whether two parsed header values project to equal Python objects. +pub(crate) fn python_equal(left: &AuthoredValue, right: &AuthoredValue) -> bool { + if let (Some(left), Some(right)) = (scalar(left), scalar(right)) { + return scalars_equal(&left, &right); + } + match (left, right) { + (AuthoredValue::List(left), AuthoredValue::List(right)) + | (AuthoredValue::Tuple(left), AuthoredValue::Tuple(right)) + | (AuthoredValue::Set(left), AuthoredValue::Set(right)) => sequences_equal(left, right), + (AuthoredValue::Map(_), AuthoredValue::Map(_)) => maps_equal(left, right), + (AuthoredValue::TypedConstant(left), AuthoredValue::TypedConstant(right)) => { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| left.0 == right.0 && python_equal(&left.1, &right.1)) + } + (AuthoredValue::InlineSqlHook(left), AuthoredValue::InlineSqlHook(right)) => left == right, + ( + AuthoredValue::NamedSqlHook(left_name, left_kwargs), + AuthoredValue::NamedSqlHook(right_name, right_kwargs), + ) + | ( + AuthoredValue::PythonHook(left_name, left_kwargs), + AuthoredValue::PythonHook(right_name, right_kwargs), + ) => { + left_name == right_name + && maps_equal( + &AuthoredValue::Map(left_kwargs.clone()), + &AuthoredValue::Map(right_kwargs.clone()), + ) + } + _ => false, + } +} + +fn sequences_equal(left: &[AuthoredValue], right: &[AuthoredValue]) -> bool { + left.len() == right.len() + && left + .iter() + .zip(right) + .all(|(left, right)| python_equal(left, right)) +} + +fn maps_equal(left: &AuthoredValue, right: &AuthoredValue) -> bool { + let (left_keys, right_keys) = (map_keys(left), map_keys(right)); + left_keys.len() == right_keys.len() + && left_keys + .iter() + .all(|key| match (map_value(left, key), map_value(right, key)) { + (Some(left), Some(right)) => python_equal(left, right), + _ => false, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index 9840061ae3..2f7cec3bcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -105,9 +105,7 @@ fn model_header_column_locations( } /// A header syntax error's detail, help and file line (when `header_line` is known). -#[pyfunction] -#[pyo3(signature = (error, header, header_line))] -fn model_header_failure( +pub(crate) fn header_failure( error: &str, header: &str, header_line: Option, @@ -124,6 +122,16 @@ fn model_header_failure( } } +/// A header syntax error's detail, help and file line (when `header_line` is known). +#[pyfunction] +fn model_header_failure( + error: &str, + header: &str, + header_line: Option, +) -> (String, Option, Option) { + header_failure(error, header, header_line) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_model_headers, module)?)?; module.add_function(wrap_pyfunction!(match_model_headers, module)?)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bbe387de6a..952a6352e5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -7170,7 +7170,6 @@ def lint_native_headers( def format_native_headers( *, contents: str, file_path: Path, config: LintConfig ) -> tuple[str, tuple[LintViolation, ...]]: ... -def prepare_native_header_cache(*, files: dict[Path, str]) -> None: ... def safe_format_files( *, files: dict[Path, str], config: LintConfig ) -> tuple[dict[Path, str], dict[Path, str]]: ... From 6e105e0e02190577d35cfba97e17bb4aa01a5013 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:10:29 +0100 Subject: [PATCH 06/12] refactor(rules): run the lint pipeline on native data and early SQL lint on a native thread Lint now scans headers, plans and prepares bodies, lints SQL and applies suppressions over Rust rows, calling into Python only to expand user macros. Early SQL lint runs on a native thread with an atomic stop instead of a Python executor and context-variable stop event, and SQL rules consume its outcome, cache entries, keys and duplicate-literal hints as Rust rows, building Finding records once at the end. Expansion inputs, relation keys, rules config and compiled expansions are read natively; LintViolation, LintRunResult and SqlExpansionContext remain projections for the Python-facing functions. Seam tests that patched the removed native lookups now exercise the native paths. --- .../sql_expansion_context.rs | 206 ++- .../_helpers/compile_driver/analysis.rs | 20 +- .../compiled_project/compiled_model_view.rs | 2 +- .../compiled_project/model_input_view.rs | 2 +- .../_helpers/compiled_project/project_view.rs | 5 + .../_helpers/compiler_records/constants.rs | 38 - .../bindings/_helpers/format_command/run.rs | 38 +- .../src/bindings/_helpers/records/ids.rs | 40 - .../bindings/_helpers/render/authored_sql.rs | 209 ++-- .../_helpers/render/macro_expansion_result.rs | 41 +- .../_helpers/rule_records/constants.rs | 28 - .../bindings/_helpers/rules/dsl_headers.rs | 156 +-- .../bindings/_helpers/rules/early_sql_lint.rs | 462 ++++--- .../bindings/_helpers/rules/fix_session.rs | 99 +- .../src/bindings/_helpers/rules/lint.rs | 26 - .../bindings/_helpers/rules/lint_expansion.rs | 125 +- .../bindings/_helpers/rules/lint_headers.rs | 92 +- .../bindings/_helpers/rules/lint_inputs.rs | 115 ++ .../bindings/_helpers/rules/lint_pipeline.rs | 1107 +++++++++++++++++ .../_helpers/rules/lint_preparation.rs | 538 ++------ .../_helpers/rules/lint_sql_bodies.rs | 91 +- .../src/bindings/_helpers/rules/lint_stop.rs | 85 -- .../src/bindings/_helpers/rules/lint_text.rs | 80 +- .../src/bindings/_helpers/rules/mod.rs | 3 +- .../_helpers/rules/native_header_parser.rs | 118 +- .../bindings/_helpers/rules/registration.rs | 23 +- .../bindings/_helpers/rules/relation_keys.rs | 66 +- .../src/bindings/_helpers/rules/rule_cache.rs | 124 +- .../bindings/_helpers/rules/rule_dispatch.rs | 122 +- .../bindings/_helpers/rules/rules_config.rs | 52 +- .../src/bindings/_helpers/rules/run_lint.rs | 363 ++---- .../src/bindings/_helpers/rules/sql_rules.rs | 828 ++++++------ src/sqlbuild/_native.pyi | 178 +-- .../sqlbuild/cli/commands/main/test_rules.py | 65 +- .../rule_engine/native_rows/_test_types.py | 4 +- .../rule_engine/native_rows/helpers.py | 12 +- .../test_sql_rule_cache_publication.py | 72 +- .../src/sqlbuild/lint/_helpers/_test_types.py | 19 +- .../src/sqlbuild/lint/_helpers/helpers.py | 18 - .../sqlbuild/lint/_helpers/test_expansion.py | 106 +- 40 files changed, 3025 insertions(+), 2753 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs delete mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs index 262ba423fb..ca31f38e93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs @@ -1,6 +1,6 @@ //! The SQL expansion inputs lint uses to expand authored SQL outside a compile. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods}; @@ -74,70 +74,125 @@ fn local_declarations<'py>(discovered: &Bound<'py, PyAny>) -> PyResult( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = KeywordArguments::read( - py, - "build_sql_expansion_context", - options, - ( - &["project_dir", "value_renderer"], +/// The inputs authored SQL is expanded with outside a compile; the handles carry user values. +pub(crate) struct SqlExpansionInputs { + pub(crate) effective_vars: Py, + pub(crate) loaded_macros: Py, + pub(crate) macro_context: Py, + pub(crate) enums: Py, + pub(crate) constants: Py, + pub(crate) value_renderer: Py, + pub(crate) collection_rendering: Py, + /// Each model file's own declarations, by file path. + pub(crate) local_declarations: Py, + pub(crate) declaration_resolver: Py, +} + +impl SqlExpansionInputs { + /// The inputs a `SqlExpansionContext` record holds. + pub(crate) fn from_record(record: &Bound<'_, PyAny>) -> PyResult { + let field = |name: &str| -> PyResult> { Ok(record.getattr(name)?.unbind()) }; + Ok(Self { + effective_vars: field("effective_vars")?, + loaded_macros: field("loaded_macros")?, + macro_context: field("macro_context")?, + enums: field("enums")?, + constants: field("constants")?, + value_renderer: field("value_renderer")?, + collection_rendering: field("collection_rendering")?, + local_declarations: record + .getattr("local_declarations")? + .downcast_into::()? + .unbind(), + declaration_resolver: field("declaration_resolver")?, + }) + } + + /// These inputs as a `SqlExpansionContext` record. + pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { + call_with( + ®istry::class(py, RecordId::SqlExpansionContext)?, &[ - "cli_vars", - "discovered_inputs", - "declaration_scope", - "static_declaration_scope", + ("effective_vars", self.effective_vars.bind(py).clone()), + ("loaded_macros", self.loaded_macros.bind(py).clone()), + ("macro_context", self.macro_context.bind(py).clone()), + ("enums", self.enums.bind(py).clone()), + ("constants", self.constants.bind(py).clone()), + ("value_renderer", self.value_renderer.bind(py).clone()), + ( + "collection_rendering", + self.collection_rendering.bind(py).clone(), + ), + ( + "local_declarations", + self.local_declarations.bind(py).clone().into_any(), + ), + ( + "declaration_resolver", + self.declaration_resolver.bind(py).clone(), + ), ], - ), - )?; - let value_renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; - let mut discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - if discovered.is_none() { - discovered = call_with( + ) + } +} + +/// Where SQL expansion inputs come from: the project, its discovery and the scope to reuse. +pub(crate) struct ExpansionSources<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) value_renderer: Bound<'py, PyAny>, + pub(crate) cli_vars: Option>, + pub(crate) discovered_inputs: Option>, + pub(crate) declaration_scope: Option>, + pub(crate) static_declaration_scope: Option>, +} + +/// Assemble SQL expansion inputs; a static scope lends only its index to private macros. +pub(crate) fn sql_expansion_inputs(sources: ExpansionSources<'_>) -> PyResult { + let py: Python<'_> = sources.project_dir.py(); + let value_renderer = sources.value_renderer; + let discovered = match sources.discovered_inputs { + Some(discovered) => discovered, + None => call_with( ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, &[ - ("project_dir", arguments.value("project_dir")?), + ("project_dir", sources.project_dir.clone()), ( "sql_analysis_enabled_override", PyBool::new(py, false).to_owned().into_any(), ), ], - )?; - } - let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; - let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; - let mut cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; - if cli_vars.is_none() { - cli_vars = PyDict::new(py).into_any(); - } - let effective_vars: Bound<'py, PyAny> = build_effective_vars( + )?, + }; + let project_config: Bound<'_, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'_, PyAny> = discovered.getattr("local_config")?; + let cli_vars: Bound<'_, PyAny> = sources + .cli_vars + .unwrap_or_else(|| PyDict::new(py).into_any()); + let effective_vars: Bound<'_, PyAny> = build_effective_vars( &project_config.clone(), &local_config.clone(), &py.None().into_bound(py), &cli_vars, ) .and_then(|value| value.into_bound_py_any(py))?; - let mut scope: Bound<'py, PyAny> = arguments.value("declaration_scope")?; - if scope.is_none() { - scope = declaration_scope( + let scope: Bound<'_, PyAny> = match sources.declaration_scope { + Some(scope) => scope, + None => declaration_scope( &discovered, - arguments.value("static_declaration_scope")?, + sources + .static_declaration_scope + .unwrap_or_else(|| py.None().into_bound(py)), value_renderer.getattr("sql_lexical_syntax")?, - )?; - } - let (enums, constants): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + )?, + }; + let (enums, constants): (Bound<'_, PyAny>, Bound<'_, PyAny>) = registry::native(py, NativeAttr::PublicDeclarationIndexes)? .call1((&discovered,))? .extract()?; - let collection_rendering: Bound<'py, PyAny> = + let collection_rendering: Bound<'_, PyAny> = resolve_effective_collection_rendering(&project_config.clone(), &py.None().into_bound(py)) .and_then(|value| value.into_bound_py_any(py))?; - let macro_context: Bound<'py, PyAny> = call_with( + let macro_context: Bound<'_, PyAny> = call_with( ®istry::class(py, RecordId::MacroContext)?, &[ ( @@ -161,21 +216,52 @@ pub(crate) fn build_sql_expansion_context<'py>( ), ], )?; - call_with( - ®istry::class(py, RecordId::SqlExpansionContext)?, - &[ - ("effective_vars", effective_vars), - ("loaded_macros", scope.getattr("loaded_macros")?), - ("macro_context", macro_context), - ("enums", enums), - ("constants", constants), - ("value_renderer", value_renderer), - ("collection_rendering", collection_rendering), - ( - "local_declarations", - local_declarations(&discovered)?.into_any(), - ), - ("declaration_resolver", scope.getattr("resolver")?), - ], - ) + Ok(SqlExpansionInputs { + effective_vars: effective_vars.unbind(), + loaded_macros: scope.getattr("loaded_macros")?.unbind(), + macro_context: macro_context.unbind(), + enums: enums.unbind(), + constants: constants.unbind(), + value_renderer: value_renderer.unbind(), + collection_rendering: collection_rendering.unbind(), + local_declarations: local_declarations(&discovered)?.unbind(), + declaration_resolver: scope.getattr("resolver")?.unbind(), + }) +} + +/// `value`, or `None` when it is Python `None`. +pub(crate) fn present(value: Bound<'_, PyAny>) -> Option> { + (!value.is_none()).then_some(value) +} + +/// Assemble SQL expansion inputs; a static scope lends only its index to private macros. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_sql_expansion_context<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_sql_expansion_context", + options, + ( + &["project_dir", "value_renderer"], + &[ + "cli_vars", + "discovered_inputs", + "declaration_scope", + "static_declaration_scope", + ], + ), + )?; + sql_expansion_inputs(ExpansionSources { + project_dir: arguments.value("project_dir")?, + value_renderer: arguments.value("value_renderer")?, + cli_vars: present(arguments.value("cli_vars")?), + discovered_inputs: present(arguments.value("discovered_inputs")?), + declaration_scope: present(arguments.value("declaration_scope")?), + static_declaration_scope: present(arguments.value("static_declaration_scope")?), + })? + .record(py) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 8c038f3b60..b3fd172a9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -131,8 +131,10 @@ pub(crate) fn analyze<'py>( let selection: &Bound<'py, PyAny> = &options.analysis_selection; let enabled: bool = !(selection.getattr("select")?.is_truthy()? || selection.getattr("exclude")?.is_truthy()?); - let manager: Bound<'py, PyAny> = Bound::new(py, EarlySqlLint::new(py, enabled)?)?.into_any(); - within(&manager, |early_lint| analyzed(py, &options, early_lint)) + let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; + let analysis: PyResult> = analyzed(py, &options, &early_lint); + let _ = early_lint.borrow().stop(py); + analysis } /// The analysis phases sharing the command options, status line and early SQL lint. @@ -140,7 +142,7 @@ struct Driver<'a, 'py> { py: Python<'py>, options: &'a AnalysisOptions<'py>, status: Status<'py>, - early_lint: &'a Bound<'py, PyAny>, + early_lint: &'a Bound<'py, EarlySqlLint>, select: Bound<'py, PyAny>, exclude: Bound<'py, PyAny>, } @@ -156,7 +158,7 @@ struct ContractPhase<'py> { fn analyzed<'py>( py: Python<'py>, options: &AnalysisOptions<'py>, - early_lint: &Bound<'py, PyAny>, + early_lint: &Bound<'py, EarlySqlLint>, ) -> PyResult> { let driver: Driver<'_, 'py> = Driver { py, @@ -180,7 +182,7 @@ fn analyzed<'py>( let graph_failed: bool = any_error(&project.getattr("diagnostics")?)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { - early_lint_wait_ms = early_lint.call_method0("stop")?; + early_lint_wait_ms = early_lint.borrow().stop(py).into_pyobject(py)?.into_any(); } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, &dialect)?; @@ -208,7 +210,7 @@ fn analyzed<'py>( (&core_diagnostics, &selected_keys), &lint_dialect, )?; - early_lint_wait_ms = early_lint.call_method0("stop")?; + early_lint_wait_ms = early_lint.borrow().stop(py).into_pyobject(py)?.into_any(); result }; let diagnostics: Bound<'py, PyTuple> = @@ -483,11 +485,7 @@ impl<'py> Driver<'_, 'py> { ("project_dir", project_dir.clone()), ("dialect", lint_dialect.clone()), ("selected_keys", selected), - ("prepared_sql", self.early_lint.getattr("preparation")?), - ( - "expansion_reuse", - self.early_lint.getattr("expansion_reuse")?, - ), + ("early_lint", self.early_lint.clone().into_any()), ("no_sql_analysis", options.no_sql_validation.clone()), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index 511b6b8033..a6c7a57f8a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -236,7 +236,7 @@ impl NativeCompiledModel { Self::with_overlay(self.owner.clone_ref(py), self.index, overlay) } - fn read<'py, T>( + pub(crate) fn read<'py, T>( &self, py: Python<'py>, read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index 4acc68f18a..3d70d29b34 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -109,7 +109,7 @@ impl NativeModelInput { Self { owner, index } } - fn read<'py, T>( + pub(crate) fn read<'py, T>( &self, py: Python<'py>, read: impl FnOnce(&ModelInputRecord, Python<'py>) -> PyResult, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index 88bc987e5c..44bb8ad7ac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -250,6 +250,11 @@ impl NativeProjectView { )) } + /// The compiled project this view projects. + pub(crate) fn owner(&self) -> &Py { + &self.owner + } + fn with_fields(owner: Py, fields: ProjectFields) -> Self { Self { owner, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 85bd0efb7a..7d6bf00b85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -324,41 +324,6 @@ const R_LINT_BODY: RecordSpec = RecordSpec { FieldSpec::new("dependency_relations", FieldDefault::EmptyTuple), ], }; -const R_PENDING_LINT_BODY: RecordSpec = RecordSpec { - name: "PendingLintBody", - doc: "One expanded authored body awaiting its native lexical preparation.", - fields: &[ - FieldSpec::required("file_path"), - FieldSpec::required("body_start"), - FieldSpec::required("body_end"), - FieldSpec::required("expansion_input"), - FieldSpec::required("expanded"), - FieldSpec::required("expansion_passes"), - FieldSpec::required("pre_expansion_sites"), - FieldSpec::required("external_identifiers"), - FieldSpec::required("allows_ceremonial_select"), - FieldSpec::required("allows_dynamic_output_star"), - FieldSpec::required("allows_empty_fixture_star"), - FieldSpec::required("request"), - ], -}; -const R_EXPANDED_LINT_FILE: RecordSpec = RecordSpec { - name: "ExpandedLintFile", - doc: "The expanded bodies of one file, up to the first body that could not be expanded.", - fields: &[ - FieldSpec::required("file_path"), - FieldSpec::required("bodies"), - FieldSpec::required("failure"), - ], -}; -const R_PREPARED_LINT_FILES: RecordSpec = RecordSpec { - name: "PreparedLintFiles", - doc: "Lint bodies of every expandable file, and why the skipped files could not expand.", - fields: &[ - FieldSpec::required("bodies"), - FieldSpec::required("unexpandable"), - ], -}; const R_LINT_EDIT: RecordSpec = RecordSpec { name: "LintEdit", doc: "One deterministic authored-source edit proposed by a lint rule.", @@ -1007,9 +972,6 @@ pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ R_LINT_FILE_ROLE, R_INTERPOLATION_SITE, R_LINT_BODY, - R_PENDING_LINT_BODY, - R_EXPANDED_LINT_FILE, - R_PREPARED_LINT_FILES, R_LINT_EDIT, R_LINT_VIOLATION, R_RULE_FIX_RESULT, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index ad821956c7..728eb2b891 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -7,6 +7,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; +use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::discovery::description_inputs::externally_described_models; use crate::bindings::_helpers::format_command::files::{ @@ -14,6 +15,11 @@ use crate::bindings::_helpers::format_command::files::{ }; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::dsl_headers::{header_violations, rules_config}; +use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; +use crate::bindings::_helpers::rules::lint_text::apply_lint_suppressions; +use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers; +use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderViolation}; const DESCRIPTION_PRESENT: &str = "description-present"; @@ -154,23 +160,16 @@ fn lint_final_contents<'py>( ) -> PyResult>> { let py: Python<'py> = files.py(); let mut violations: Vec> = Vec::new(); + let header_rules: HeaderRulesConfig = rules_config(request.config)?; + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; for (path, contents) in sorted_items(files)? { - let final_contents: Bound<'py, PyAny> = updated.get_item(&path)?.unwrap_or(contents); - let headers: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::ScanHeaders)?, - &[("contents", final_contents.clone())], - )?; - let found: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::LintNativeHeaders)?, - &[ - ("contents", final_contents), - ("file_path", path), - ("headers", headers), - ("config", request.config.clone()), - ], - )?; - for violation in found.try_iter()? { - violations.push(violation?); + let final_contents: String = updated.get_item(&path)?.unwrap_or(contents).extract()?; + let text: Vec = final_contents.chars().collect(); + let found: Vec = + header_violations(&text, &scan_headers(&text, false), header_rules) + .map_err(value_error)?; + for violation in found { + violations.push(classes.violation(&path, &LintViolationRow::header(0, violation))?); } } Ok(violations) @@ -303,10 +302,9 @@ pub(crate) fn run_format<'py>(request: &FormatRequest<'_, 'py>) -> PyResult
> = drop_resolved_description_faults(request.project_dir, violations)?; let final_contents: Bound<'py, PyDict> = overlaid(&files, &updated)?; - let allow_suppressions: Bound<'py, PyAny> = request.config.getattr("allow_suppressions")?; - let violations: Vec> = native(py, NativeAttr::ApplyLintSuppressions)? - .call1((violations, &final_contents, allow_suppressions))? - .extract()?; + let allow_suppressions: bool = request.config.getattr("allow_suppressions")?.extract()?; + let violations: Vec> = + apply_lint_suppressions(py, violations, final_contents.clone(), allow_suppressions)?; Ok(FormatOutcome { files_checked, violations, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index ffe8ca87bd..7bfd4ebf72 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -243,7 +243,6 @@ native_ids! { EnumMetadata => "EnumMetadata", EventExportSeverity => "EventExportSeverity", ExecutionLimitsConfig => "ExecutionLimitsConfig", - ExpandedLintFile => "ExpandedLintFile", ExpansionSpan => "ExpansionSpan", Finding => "Finding", Fingerprint => "Fingerprint", @@ -383,7 +382,6 @@ native_ids! { ParsedSelector => "ParsedSelector", PartialBuildPhaseTimings => "PartialBuildPhaseTimings", PathSelector => "PathSelector", - PendingLintBody => "PendingLintBody", PlanAction => "PlanAction", PlanCommandRequest => "PlanCommandRequest", PlanEntryBuildInputs => "PlanEntryBuildInputs", @@ -413,9 +411,6 @@ native_ids! { PlaygroundTemplate => "PlaygroundTemplate", PolyglotAnalysisDialect => "PolyglotAnalysisDialect", PolyglotAnalysisResult => "PolyglotAnalysisResult", - PreparedLintFiles => "PreparedLintFiles", - PreparedSqlLint => "PreparedSqlLint", - PreparedSqlLintResult => "PreparedSqlLintResult", Project => "Project", ProjectConfig => "ProjectConfig", ProjectGraph => "ProjectGraph", @@ -558,7 +553,6 @@ native_ids! { SqlBindingDiagnostic => "SqlBindingDiagnostic", SqlBindingResult => "SqlBindingResult", SqlExpansionContext => "SqlExpansionContext", - SqlExpansionReuse => "SqlExpansionReuse", SqlHookEntry => "SqlHookEntry", SqlLexicalSyntax => "SqlLexicalSyntax", SqlLogicalType => "SqlLogicalType", @@ -691,7 +685,6 @@ native_ids! { ApplyCommandWarehouse => "apply_command_warehouse", ApplyConfiguredMicrobatchLimits => "apply_configured_microbatch_limits", ApplyDeferredLocations => "apply_deferred_locations", - ApplyLintSuppressions => "apply_lint_suppressions", ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", ApplyUnitTestSelectors => "apply_unit_test_selectors", @@ -738,7 +731,6 @@ native_ids! { BuildFunctionLocations => "build_function_locations", BuildIdentity => "BUILD_IDENTITY", BuildIntermediateSourceMap => "build_intermediate_source_map", - BuildLintExpansionContext => "build_lint_expansion_context", BuildManifest => "build_manifest", BuildMergedDbtExecutionArgv => "build_merged_dbt_execution_argv", BuildMigrationFingerprint => "build_migration_fingerprint", @@ -791,7 +783,6 @@ native_ids! { ClassifyCursorSqlType => "classify_cursor_sql_type", CliStyle => "CliStyle", CollectCompileDiagnostics => "collect_compile_diagnostics", - CollectLintProjectFiles => "collect_lint_project_files", CommandWarehouseScopeContext => "_command_warehouse_scope_context", CompareContracts => "compare_contracts", CompileDbtInteropProject => "compile_dbt_interop_project", @@ -805,7 +796,6 @@ native_ids! { CompiledModelRecord => "compiled_model_record", CompiledProjectFacts => "compiled_project_facts", CompiledProjectRecord => "compiled_project_record", - CompiledRelationKeys => "compiled_relation_keys", CompleteSemanticProject => "complete_semantic_project", ComputeCursorBounds => "compute_cursor_bounds", ComputeModelCursorBounds => "compute_model_cursor_bounds", @@ -838,7 +828,6 @@ native_ids! { DecideModelMigration => "decide_model_migration", DeclarationRows => "declaration_rows", DeclaredColumnAliases => "declared_column_aliases", - DeclaredRelationKeys => "declared_relation_keys", DeclaredRelations => "declared_relations", DescribeConnectionWarehouse => "describe_connection_warehouse", DescribeFullRebuildCause => "describe_full_rebuild_cause", @@ -862,7 +851,6 @@ native_ids! { DiscoverSourceFiles => "discover_source_files", DiscoverTestFiles => "discover_test_files", DynamicContractProof => "dynamic_contract_proof", - EarlyLintExpansions => "early_lint_expansions", EditContractModel => "edit_contract_model", EditContractSource => "edit_contract_source", EffectiveConfigConnection => "effective_config_connection", @@ -872,7 +860,6 @@ native_ids! { EffectivelyFullRefreshedModelNames => "effectively_full_refreshed_model_names", EmptyCursorWindow => "empty_cursor_window", EmptyInputRebuildRefusal => "empty_input_rebuild_refusal", - EncodeSqlRuleFindings => "encode_sql_rule_findings", EnforceModelExecutionLimit => "enforce_model_execution_limit", EnforcePlanSafetyPolicies => "enforce_plan_safety_policies", EvaluateModelContracts => "evaluate_model_contracts", @@ -887,13 +874,10 @@ native_ids! { ExecuteDbtSqlbuildWork => "execute_dbt_sqlbuild_work", ExecuteDbtWithoutStateTracking => "execute_dbt_without_state_tracking", ExecutionEventWriter => "ExecutionEventWriter", - ExpandFileBodies => "expand_file_bodies", ExpandProjectUpstreamKeys => "expand_project_upstream_keys", ExpandSqlMacros => "expand_sql_macros", - ExpandSqlWithSpans => "expand_sql_with_spans", ExplicitReferenceReport => "explicit_reference_report", ExplicitReferencesHelp => "explicit_references_help", - ExternalIdentifiersForHeaders => "external_identifiers_for_headers", ExternallyDescribedModels => "externally_described_models", ExtractSeedColumns => "extract_seed_columns", FinalizeRuleFindingsRows => "finalize_rule_findings_rows", @@ -949,9 +933,6 @@ native_ids! { LaunchRecordOffered => "_launch_record_offered", LaunchRecordWithheld => "_launch_record_withheld", LineageGraphNames => "lineage_graph_names", - LintNativeHeaders => "lint_native_headers", - LintNativeSqlBodies => "lint_native_sql_bodies", - LintStopScopeFunction => "lint_stop_scope", ListCostRuns => "list_cost_runs", LoadCompiledDbtManifest => "load_compiled_dbt_manifest", LoadDbtManifestIndex => "load_dbt_manifest_index", @@ -960,10 +941,8 @@ native_ids! { LoadProgressReporter => "LoadProgressReporter", LoadProjectConfigFile => "load_project_config_file", LoadRulesConfig => "load_rules_config", - LoadSqlRuleCache => "load_sql_rule_cache", LoaderDestinationPartsFunction => "loader_destination_parts", LocateModelBindingDiagnostics => "locate_model_binding_diagnostics", - LookupSqlRuleCache => "lookup_sql_rule_cache", MachineOutputMode => "machine_output_mode", MacroStoreToken => "macro_store_token", MaintainSqlbuildSkills => "maintain_sqlbuild_skills", @@ -972,7 +951,6 @@ native_ids! { MatchProjectSelector => "match_project_selector", MatchingDynamicFamilies => "matching_dynamic_families", MaximumAllowedStart => "maximum_allowed_start", - MeasurementBodyRanges => "measurement_body_ranges", MergeRecomputedModelChanges => "merge_recomputed_model_changes", MicrobatchGuidance => "microbatch_guidance", MigrationDestinationFingerprints => "migration_destination_fingerprints", @@ -993,7 +971,6 @@ native_ids! { ModelPlaceholders => "model_placeholders", ModelPythonHookNames => "model_python_hook_names", ModelSyntaxChecks => "model_syntax_checks", - NamedDeclarationRoleKind => "named_declaration_role_kind", NativeCompiledModel => "NativeCompiledModel", NativeCompiledProject => "NativeCompiledProject", NativeCompiledSeed => "NativeCompiledSeed", @@ -1005,8 +982,6 @@ native_ids! { NativeProgressProjector => "NativeProgressProjector", NativeProgressProjectorContext => "_native_progress_projector_context", NestedCommandProgressCallbacks => "NestedCommandProgressCallbacks", - NeutralizeLintAuditParameters => "neutralize_lint_audit_parameters", - NeutralizeLintContextInterpolation => "neutralize_lint_context_interpolation", NormalizeAnalysisSql => "normalize_analysis_sql", NormalizeCursorSnapshotGrain => "normalize_cursor_snapshot_grain", NormalizeDbtProfileConnection => "normalize_dbt_profile_connection", @@ -1038,11 +1013,7 @@ native_ids! { PlanStaticSqlTests => "plan_static_sql_tests", PlanTableTypes => "plan_table_types", PlanningProgressReporter => "PlanningProgressReporter", - PrepareExpandedFiles => "prepare_expanded_files", - PrepareLintSql => "prepare_lint_sql", - PrepareLintSqlBatch => "prepare_lint_sql_batch", PrepareModelHeaderTokens => "prepare_model_header_tokens", - PrepareSqlRules => "prepare_sql_rules", PreparedCompileArtifacts => "PreparedCompileArtifacts", ProjectLineageViews => "project_lineage_views", ProjectRelativePath => "project_relative_path", @@ -1166,7 +1137,6 @@ native_ids! { RunInitCommand => "run_init_command", RunJanitorCommand => "run_janitor_command", RunLineageCommand => "run_lineage_command", - RunLint => "run_lint", RunLoadCommand => "run_load_command", RunPlanCommand => "run_plan_command", RunPlannedSqlRules => "run_planned_sql_rules", @@ -1183,14 +1153,12 @@ native_ids! { RunTestCommand => "run_test_command", RuntimeCursorProducerNames => "runtime_cursor_producer_names", SafeFormatFiles => "safe_format_files", - ScanHeaders => "scan_headers", SchemaValidations => "schema_validations", ScopeMetadataProjection => "scope_metadata_projection", SelectCatalogueRules => "select_catalogue_rules", SelectLatestFingerprints => "select_latest_fingerprints", SelectProjectGraph => "select_project_graph", SelectedRuleCodes => "selected_rule_codes", - SelectedSqlRuleCodes => "selected_sql_rule_codes", SessionOutcomeAnalysis => "session_outcome_analysis", SettingHelp => "setting_help", SharedTransientLineCoordinator => "shared_transient_line_coordinator", @@ -1200,13 +1168,8 @@ native_ids! { SplitProjectUnitTestSelectors => "split_project_unit_test_selectors", SplitSelectorExpansion => "split_selector_expansion", SplitUnitTestSelectors => "split_unit_test_selectors", - SqlBodyRanges => "sql_body_ranges", SqlCodeScanner => "SqlCodeScanner", - SqlFileRuleKey => "sql_file_rule_key", - SqlLintPreparation => "sql_lint_preparation", - SqlModelRuleKeys => "sql_model_rule_keys", SqlRefKey => "sql_ref_key", - SqlRuleLintKey => "sql_rule_lint_key", SqlTestModelChainNamesByKey => "sql_test_model_chain_names_by_key", SqlTestScanCache => "SqlTestScanCache", SqlValueValidationError => "SqlValueValidationError", @@ -1225,7 +1188,6 @@ native_ids! { UnreadableCursorInputsError => "unreadable_cursor_inputs_error", UpdateSqlbuildSkills => "update_sqlbuild_skills", UpstreamLoaderDependencyNames => "upstream_loader_dependency_names", - ValidRulesConfig => "valid_rules_config", ValidateAuthoredSqlTestCtes => "validate_authored_sql_test_ctes", ValidateCappedWatermarkInputs => "validate_capped_watermark_inputs", ValidateCliWarehouse => "validate_cli_warehouse", @@ -1240,7 +1202,6 @@ native_ids! { ValidateSqlSyntax => "validate_sql_syntax", WithBindingDiagnostics => "with_binding_diagnostics", WithDeclaredSourceReads => "with_declared_source_reads", - WithDuplicateLiteralHints => "with_duplicate_literal_hints", WithHonestModelWriteHashes => "with_honest_model_write_hashes", WithMigrationFingerprints => "with_migration_fingerprints", WithTypeRecovery => "with_type_recovery", @@ -1257,7 +1218,6 @@ native_ids! { WriteReusableCompile => "write_reusable_compile", WriteRuntimeTarget => "write_runtime_target", WriteSourceFreshnessRecords => "write_source_freshness_records", - WriteSqlRuleCache => "write_sql_rule_cache", WriteSqlbuildSkipNotice => "write_sqlbuild_skip_notice", YamlEntryLine => "yaml_entry_line", } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 3e35ac8ac3..f333f5920c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -3,11 +3,12 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::model_config::unicode_environment::unicode_environment; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; @@ -18,7 +19,8 @@ use crate::bindings::_helpers::render::declaration_expansion::{ }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; -use crate::bindings::_helpers::render::macro_expansion_result::expand_sql_macros; +use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::render::macro_expansion_result::{ExpansionRequest, macro_output}; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; @@ -137,53 +139,129 @@ fn declarations_expanded<'py>( Ok((expanded.spliced.sql, spans)) } -/// Expand authored SQL, returning each pass's substitution spans in order. -fn authored_sql_with_spans<'py>(request: &Request<'_, 'py>) -> PyResult> { - let py: Python<'py> = request.py(); - let none = || py.None().into_bound(py); - let file_path: Bound<'py, PyAny> = request.get("file_path")?; +/// Authored SQL expanded through vars, declarations and macros, with each pass's spans. +pub(crate) struct AuthoredExpansion { + pub(crate) sql: String, + /// Interpolation, declaration and macro spans, in pass order. + pub(crate) passes: [Vec; 3], +} + +/// What one authored SQL expansion reads. +struct AuthoredSources<'a, 'py> { + sql: &'a Bound<'py, PyAny>, + file_path: &'a Bound<'py, PyAny>, + effective_vars: &'a Bound<'py, PyAny>, + context_values: &'a Bound<'py, PyAny>, + declarations: &'a Bound<'py, PyAny>, + value_renderer: &'a Bound<'py, PyAny>, + collection_rendering: &'a Bound<'py, PyAny>, + loaded_macros: &'a Bound<'py, PyAny>, + macro_context: &'a Bound<'py, PyAny>, + declaration_resolver: Option>, +} + +fn authored_expansion(sources: AuthoredSources<'_, '_>) -> PyResult { + let py: Python<'_> = sources.sql.py(); let (interpolated, interpolation_spans) = substituted( - &request.get("sql")?, - &file_path, - ( - &request.get("effective_vars")?, - &request.optional("context_values")?.unwrap_or_else(none), - ), + sources.sql, + sources.file_path, + (sources.effective_vars, sources.context_values), )?; let (declared, declaration_spans) = declarations_expanded( &interpolated, - &file_path, + sources.file_path, ( - &request.optional("declarations")?.unwrap_or_else(none), - &request.get("value_renderer")?, - &request.get("collection_rendering")?, + sources.declarations, + sources.value_renderer, + sources.collection_rendering, ), )?; - let macro_options: Bound<'py, PyDict> = keywords( - py, - &[ - ("sql", PyString::new(py, &declared).into_any()), - ("file_path", file_path), - ("loaded_macros", request.get("loaded_macros")?), - ("macro_context", request.get("macro_context")?), - ( - "declaration_resolver", - request - .optional("declaration_resolver")? - .unwrap_or_else(none), - ), - ], - )?; - let expansion: Bound<'py, PyAny> = expand_sql_macros(py, Some(macro_options))?; - let passes: Bound<'py, PyTuple> = PyTuple::new( + let request: ExpansionRequest<'_> = ExpansionRequest { + sql: PyString::new(py, &declared), + file_path: sources.file_path.clone(), + loaded_macros: sources.loaded_macros.downcast()?.clone(), + macro_overrides: None, + macro_context: sources.macro_context.clone(), + declaration_resolver: sources.declaration_resolver, + declarations: None, + consumer: None, + }; + Ok(match macro_output(request)? { + Some(output) => AuthoredExpansion { + sql: output.sql, + passes: [ + interpolation_spans, + declaration_spans, + output.spans.into_iter().map(span_row).collect(), + ], + }, + None => AuthoredExpansion { + sql: declared, + passes: [interpolation_spans, declaration_spans, Vec::new()], + }, + }) +} + +/// `(sql, passes)` with each pass's `ExpansionSpan` objects. +fn expansion_tuple<'py>( + py: Python<'py>, + expansion: &AuthoredExpansion, +) -> PyResult> { + let passes: Vec> = expansion + .passes + .iter() + .map(|spans| span_objects(py, spans)) + .collect::>()?; + PyTuple::new( py, [ - span_objects(py, &interpolation_spans)?.into_any(), - span_objects(py, &declaration_spans)?.into_any(), - expansion.getattr("spans")?, + PyString::new(py, &expansion.sql).into_any(), + PyTuple::new(py, passes)?.into_any(), ], - )?; - PyTuple::new(py, [expansion.getattr("sql")?, passes.into_any()]) + ) +} + +/// Expand authored SQL, returning each pass's substitution spans in order. +fn authored_sql_with_spans<'py>(request: &Request<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.py(); + let none = || py.None().into_bound(py); + let expansion: AuthoredExpansion = authored_expansion(AuthoredSources { + sql: &request.get("sql")?, + file_path: &request.get("file_path")?, + effective_vars: &request.get("effective_vars")?, + context_values: &request.optional("context_values")?.unwrap_or_else(none), + declarations: &request.optional("declarations")?.unwrap_or_else(none), + value_renderer: &request.get("value_renderer")?, + collection_rendering: &request.get("collection_rendering")?, + loaded_macros: &request.get("loaded_macros")?, + macro_context: &request.get("macro_context")?, + declaration_resolver: request + .optional("declaration_resolver")? + .filter(|resolver| !resolver.is_none()), + })?; + expansion_tuple(py, &expansion) +} + +/// `sql` expanded with lint's expansion inputs, as `file_path`'s declarations see them. +pub(crate) fn expand_with_inputs( + inputs: &SqlExpansionInputs, + sql: &str, + file_path: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = file_path.py(); + let resolver: Bound<'_, PyAny> = inputs.declaration_resolver.bind(py).clone(); + authored_expansion(AuthoredSources { + sql: &PyString::new(py, sql).into_any(), + file_path, + effective_vars: inputs.effective_vars.bind(py), + context_values: &py.None().into_bound(py), + declarations: &input_declarations(inputs, file_path)?, + value_renderer: inputs.value_renderer.bind(py), + collection_rendering: inputs.collection_rendering.bind(py), + loaded_macros: inputs.loaded_macros.bind(py), + macro_context: inputs.macro_context.bind(py), + declaration_resolver: (!resolver.is_none()).then_some(resolver), + }) } /// Expand authored SQL, returning each pass's substitution spans in order. @@ -195,29 +273,29 @@ fn expand_authored_sql_with_spans<'py>( authored_sql_with_spans(&Request::new("expand_authored_sql_with_spans", request)?) } -/// The declarations visible to `file_path`: the resolver's, else the context's, plus model-local. -fn path_declarations<'py>( - context: &Bound<'py, PyAny>, +/// The declarations visible to `file_path`: the resolver's, else the inputs', plus model-local. +fn input_declarations<'py>( + inputs: &SqlExpansionInputs, file_path: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = context.py(); + let py: Python<'py> = file_path.py(); let resolution_class: Bound<'py, PyAny> = registry::class(py, RecordId::DeclarationResolutionContext)?; - let resolver: Bound<'py, PyAny> = context.getattr("declaration_resolver")?; + let resolver: &Bound<'py, PyAny> = inputs.declaration_resolver.bind(py); let declarations: Bound<'py, PyAny> = if resolver.is_none() { call_with( &resolution_class, &[ - ("enums", context.getattr("enums")?), - ("constants", context.getattr("constants")?), + ("enums", inputs.enums.bind(py).clone()), + ("constants", inputs.constants.bind(py).clone()), ], )? } else { - resolved_declaration_context(&resolver, file_path, None)? + resolved_declaration_context(resolver, file_path, None)? + }; + let Some(local) = inputs.local_declarations.bind(py).get_item(file_path)? else { + return Ok(declarations); }; - let local: Bound<'py, PyAny> = context - .getattr("local_declarations")? - .call_method1("get", (file_path,))?; if local.is_none() { return Ok(declarations); } @@ -245,32 +323,15 @@ fn path_declarations<'py>( #[pyfunction] #[pyo3(signature = (*, sql, file_path, context))] fn expand_sql_with_spans<'py>( - sql: &Bound<'py, PyAny>, + sql: &str, file_path: &Bound<'py, PyAny>, context: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = sql.py(); - let arguments: Bound<'py, PyDict> = keywords( - py, - &[ - ("sql", sql.clone()), - ("file_path", file_path.clone()), - ("effective_vars", context.getattr("effective_vars")?), - ("loaded_macros", context.getattr("loaded_macros")?), - ("macro_context", context.getattr("macro_context")?), - ("declarations", path_declarations(context, file_path)?), - ( - "declaration_resolver", - context.getattr("declaration_resolver")?, - ), - ("value_renderer", context.getattr("value_renderer")?), - ( - "collection_rendering", - context.getattr("collection_rendering")?, - ), - ], - )?; - authored_sql_with_spans(&Request::new("expand_sql_with_spans", Some(&arguments))?) + let inputs: SqlExpansionInputs = SqlExpansionInputs::from_record(context)?; + expansion_tuple( + file_path.py(), + &expand_with_inputs(&inputs, sql, file_path)?, + ) } /// Replace SQL interpolation tokens, returning the span of every substitution. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index ee59c533ca..4b14f89002 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -41,15 +41,15 @@ const GENERATED_ALLOWED: &str = "macro-generated references"; static EXPANSION_CLASSES: PyOnceLock> = PyOnceLock::new(); /// One expansion's inputs, as Python's `expand_sql_macros_result` takes them. -struct ExpansionRequest<'py> { - sql: Bound<'py, PyString>, - file_path: Bound<'py, PyAny>, - loaded_macros: Bound<'py, PyDict>, - macro_overrides: Option>, - macro_context: Bound<'py, PyAny>, - declaration_resolver: Option>, - declarations: Option>, - consumer: Option>, +pub(crate) struct ExpansionRequest<'py> { + pub(crate) sql: Bound<'py, PyString>, + pub(crate) file_path: Bound<'py, PyAny>, + pub(crate) loaded_macros: Bound<'py, PyDict>, + pub(crate) macro_overrides: Option>, + pub(crate) macro_context: Bound<'py, PyAny>, + pub(crate) declaration_resolver: Option>, + pub(crate) declarations: Option>, + pub(crate) consumer: Option>, } fn generated_classes(py: Python<'_>) -> PyResult> { @@ -190,12 +190,13 @@ fn result_model<'py>( call_with(®istry::class(py, RecordId::MacroExpansionResult)?, pairs) } -fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult> { - let py: Python<'py> = request.sql.py(); +/// The macro expansion of `request`'s SQL, or `None` when it calls no macro. +pub(crate) fn macro_output(request: ExpansionRequest<'_>) -> PyResult> { + let py: Python<'_> = request.sql.py(); if !request.sql.to_str()?.contains(MACRO_TOKEN) { - return result_model(py, &[("sql", request.sql.into_any())]); + return Ok(None); } - let declarations: Bound<'py, PyAny> = match (request.declarations, request.declaration_resolver) + let declarations: Bound<'_, PyAny> = match (request.declarations, request.declaration_resolver) { (Some(declarations), _) => declarations, (None, Some(resolver)) => { @@ -221,9 +222,17 @@ fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult = Bound::new(py, expansion)?; - let bridge: Bound<'py, PyAny> = host::entry(py, UserCodeHost::ActiveMacroBridge)?.call0()?; - let output: ExpansionOutput = expanded(&expansion, request.sql.to_str()?, &bridge)?; + let expansion: Bound<'_, NativeMacroExpansion> = Bound::new(py, expansion)?; + let bridge: Bound<'_, PyAny> = host::entry(py, UserCodeHost::ActiveMacroBridge)?.call0()?; + expanded(&expansion, request.sql.to_str()?, &bridge).map(Some) +} + +fn expansion_result<'py>(request: ExpansionRequest<'py>) -> PyResult> { + let py: Python<'py> = request.sql.py(); + let sql: Bound<'py, PyString> = request.sql.clone(); + let Some(output) = macro_output(request)? else { + return result_model(py, &[("sql", sql.into_any())]); + }; let span_class: Bound<'py, PyAny> = registry::class(py, RecordId::ExpansionSpan)?; let mut spans: Vec> = Vec::with_capacity(output.spans.len()); for span in output.spans { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs index 670af04e79..94767d13ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -67,31 +67,6 @@ const R_SQL_TEST_RULES_CONFIG: RecordSpec = RecordSpec { FieldDefault::Text("pipelines"), )], }; -const R_PREPARED_SQL_LINT_RESULT: RecordSpec = RecordSpec { - name: "PreparedSqlLintResult", - doc: "Reusable expansion context and completed preliminary SQL checks.", - fields: &[ - FieldSpec::required("result"), - FieldSpec::required("context"), - ], -}; -const R_SQL_EXPANSION_REUSE: RecordSpec = RecordSpec { - name: "SqlExpansionReuse", - doc: "Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.", - fields: &[ - FieldSpec::required("discovered_inputs"), - FieldSpec::required("declaration_scope"), - ], -}; -const R_PREPARED_SQL_LINT: RecordSpec = RecordSpec { - name: "PreparedSqlLint", - doc: "Process-local SQL checks started after compiler expansion has completed.", - fields: &[ - FieldSpec::required("codes"), - FieldSpec::required("future"), - FieldSpec::new("project_dir", FieldDefault::None), - ], -}; const R_SQL_RULES_EVALUATION: RecordSpec = RecordSpec { name: "SqlRulesEvaluation", doc: "SQL rule findings of one run, their cache accounting and elapsed time.", @@ -397,9 +372,6 @@ pub(crate) const RULE_RECORDS: &[RecordSpec] = &[ R_THRESHOLD_OVERRIDE, R_RULES_CACHE_CONFIG, R_SQL_TEST_RULES_CONFIG, - R_PREPARED_SQL_LINT_RESULT, - R_SQL_EXPANSION_REUSE, - R_PREPARED_SQL_LINT, R_SQL_RULES_EVALUATION, R_RULE_RUN_PLAN, R_RULES_RUN_RESULT, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs index 12528c17f5..674fc5886f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/dsl_headers.rs @@ -5,89 +5,39 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTupl use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::dsl_headers::constants::RULE_HEADER_PARSE; use sqlbuild_rules::dsl_headers::main::format_headers::format_headers; -use sqlbuild_rules::dsl_headers::main::inner_header_text::inner_header_text; use sqlbuild_rules::dsl_headers::main::lint_headers::lint_headers; use sqlbuild_rules::dsl_headers::main::measurement_body_ranges::measurement_body_ranges as measurement_ranges; use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers as scan; use sqlbuild_rules::dsl_headers::main::sql_body_ranges::sql_body_ranges as body_ranges; -use sqlbuild_rules::dsl_headers::models::{ - HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, ParseOutcome, -}; -use sqlbuild_rules::dsl_headers::types::HeaderParser; +use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderSpan, HeaderViolation}; +use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::rules::native_header_parser::{ - NativeHeaderParser, map_keys, map_value, -}; -use sqlbuild_sqltext::compiler::models::AuthoredValue; +use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; +use crate::bindings::_helpers::rules::native_header_parser::NativeHeaderParser; -const ENGINE: &str = "sqlbuild"; const LINE_FEED: &str = "\n"; const CARRIAGE_RETURN_LINE_FEED: &str = "\r\n"; -const FUNCTION_KIND: &str = "FUNCTION"; -const ARGUMENTS_KEY: &str = "arguments"; -/// The models and enums header results are built as. -struct LintClasses<'py> { - violation: Bound<'py, PyAny>, - severity: Bound<'py, PyAny>, -} - -impl<'py> LintClasses<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - violation: registry::class(py, RecordId::LintViolation)?, - severity: registry::class(py, RecordId::LintSeverity)?, - }) - } - - fn violation( - &self, - file_path: &Bound<'py, PyAny>, - violation: HeaderViolation, - ) -> PyResult> { - let py: Python<'py> = file_path.py(); - let severity: &str = match violation.severity { - HeaderSeverity::Fault => "FAULT", - HeaderSeverity::Warning => "WARNING", - }; - call_with( - &self.violation, - &[ - ("file_path", file_path.clone()), - ("line", violation.line.into_pyobject(py)?.into_any()), - ("column", violation.column.into_pyobject(py)?.into_any()), - ("code", PyString::new(py, violation.code).into_any()), - ("message", PyString::new(py, &violation.message).into_any()), - ("severity", self.severity.getattr(severity)?), - ("engine", PyString::new(py, ENGINE).into_any()), - ( - "remediation", - PyString::new(py, &violation.remediation).into_any(), - ), - ], - ) - } - - fn violations( - &self, - file_path: &Bound<'py, PyAny>, - violations: Vec, - ) -> PyResult>> { - violations - .into_iter() - .map(|violation| self.violation(file_path, violation)) - .collect() - } +/// Header violations against `file_path` as `LintViolation` records. +fn violation_objects<'py>( + file_path: &Bound<'py, PyAny>, + violations: Vec, +) -> PyResult>> { + let classes: ViolationClasses<'py> = ViolationClasses::new(file_path.py())?; + violations + .into_iter() + .map(|violation| classes.violation(file_path, &LintViolationRow::header(0, violation))) + .collect() } fn text(contents: &Bound<'_, PyAny>) -> PyResult> { Ok(contents.extract::()?.chars().collect()) } -fn rules_config(config: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn rules_config(config: &Bound<'_, PyAny>) -> PyResult { Ok(HeaderRulesConfig { line_width: config.getattr("line_width")?.extract()?, max_description_lines: config.getattr("max_description_lines")?.extract()?, @@ -157,13 +107,13 @@ fn measurement_body_ranges<'py>(contents: &Bound<'py, PyAny>) -> PyResult( +/// The header rule violations of `headers` in `contents`. +pub(crate) fn header_violations( contents: &[char], headers: &[HeaderSpan], - config: &Bound<'py, PyAny>, -) -> PyResult> { - let parser: NativeHeaderParser<'py> = NativeHeaderParser::new(config.py()); - lint_headers(contents, headers, rules_config(config)?, &parser) + config: HeaderRulesConfig, +) -> Result, String> { + lint_headers(contents, headers, config, &NativeHeaderParser) } /// Every header rule violation of `headers` in `contents`. @@ -176,9 +126,9 @@ fn lint_native_headers<'py>( config: &Bound<'py, PyAny>, ) -> PyResult> { let violations: Vec = - header_violations(&text(contents)?, &spans(headers)?, config)?; - let classes: LintClasses<'py> = LintClasses::new(contents.py())?; - PyTuple::new(contents.py(), classes.violations(file_path, violations)?) + header_violations(&text(contents)?, &spans(headers)?, rules_config(config)?) + .map_err(value_error)?; + PyTuple::new(contents.py(), violation_objects(file_path, violations)?) } /// `contents` with header formatting fixes applied, and the faults relocation reported. @@ -190,28 +140,28 @@ fn format_native_headers<'py>( config: &Bound<'py, PyAny>, ) -> PyResult<(String, Bound<'py, PyTuple>)> { let py: Python<'py> = contents.py(); - let parser: NativeHeaderParser<'py> = NativeHeaderParser::new(py); - let (formatted, faults) = format_headers(&text(contents)?, rules_config(config)?, &parser)?; - let classes: LintClasses<'py> = LintClasses::new(py)?; + let (formatted, faults) = + format_headers(&text(contents)?, rules_config(config)?, &NativeHeaderParser) + .map_err(value_error)?; Ok(( formatted.into_iter().collect(), - PyTuple::new(py, classes.violations(file_path, faults)?)?, + PyTuple::new(py, violation_objects(file_path, faults)?)?, )) } /// The header-parse faults of one file. fn parse_faults<'py>( - classes: &LintClasses<'py>, (file_path, contents): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), config: &Bound<'py, PyAny>, ) -> PyResult>> { let contents: Vec = text(contents)?; let violations: Vec = - header_violations(&contents, &scan(&contents, false), config)? + header_violations(&contents, &scan(&contents, false), rules_config(config)?) + .map_err(value_error)? .into_iter() .filter(|violation| violation.code == RULE_HEADER_PARSE) .collect(); - classes.violations(file_path, violations) + violation_objects(file_path, violations) } /// Parse-safe files, and every file's authored newline convention. @@ -222,11 +172,10 @@ fn safe_format_files<'py>( config: &Bound<'py, PyAny>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyDict>)> { let py: Python<'py> = files.py(); - let classes: LintClasses<'py> = LintClasses::new(py)?; let safe: Bound<'py, PyDict> = PyDict::new(py); let newlines: Bound<'py, PyDict> = PyDict::new(py); for (file_path, contents) in files.iter() { - if parse_faults(&classes, (&file_path, &contents), config)?.is_empty() { + if parse_faults((&file_path, &contents), config)?.is_empty() { safe.set_item(&file_path, &contents)?; } } @@ -254,11 +203,9 @@ fn reject_unparseable_header_rewrites<'py>( config: &Bound<'py, PyAny>, faults: &Bound<'py, PyList>, ) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyList>)> { - let classes: LintClasses<'py> = LintClasses::new(updated.py())?; let accepted: Bound<'py, PyDict> = updated.copy()?; for (file_path, contents) in updated.iter() { - let parse: Vec> = - parse_faults(&classes, (&file_path, &contents), config)?; + let parse: Vec> = parse_faults((&file_path, &contents), config)?; if parse.is_empty() { continue; } @@ -272,44 +219,6 @@ fn reject_unparseable_header_rewrites<'py>( Ok((accepted, faults.clone())) } -/// The SQL function argument names a FUNCTION header declares, lowercased and sorted. -#[pyfunction] -#[pyo3(signature = (*, contents, headers))] -fn external_identifiers_for_headers<'py>( - contents: &Bound<'py, PyAny>, - headers: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = contents.py(); - let text: Vec = text(contents)?; - let parser: NativeHeaderParser<'py> = NativeHeaderParser::new(py); - for header in spans(headers)? { - if header.kind != FUNCTION_KIND { - continue; - } - let inner: String = inner_header_text(&text[header.start..header.end]) - .iter() - .collect(); - let values: AuthoredValue = match parser.parse(&header.kind, &inner) { - ParseOutcome::Parsed(values) => values, - ParseOutcome::Rejected(_) => return Err(parser.rejected_error()), - ParseOutcome::Raised(error, _) => return Err(error), - }; - let Some(arguments) = map_value(&values, ARGUMENTS_KEY) else { - continue; - }; - if !matches!(arguments, AuthoredValue::Map(_)) { - continue; - } - let mut names: Vec = map_keys(arguments) - .into_iter() - .map(str::to_lowercase) - .collect(); - names.sort_unstable(); - return PyTuple::new(py, names); - } - Ok(PyTuple::empty(py)) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(scan_headers, module)?)?; module.add_function(wrap_pyfunction!(sql_body_ranges, module)?)?; @@ -321,6 +230,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { reject_unparseable_header_rewrites, module )?)?; - module.add_function(wrap_pyfunction!(external_identifiers_for_headers, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index a44077bd9b..0927a97207 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -3,187 +3,271 @@ use std::path::PathBuf; use std::time::Instant; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; +use std::thread::JoinHandle; + +use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, + Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRefMut, + PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use pyo3::{IntoPyObject, IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::types::PyTuple; +use pyo3::{pyclass, pymethods}; +use sqlbuild_rules::lint_text::models::LintSqlOptions; use sqlbuild_rules::rule_cache::main::rules_bulk_cache_path::rules_bulk_cache_path; +use sqlbuild_rules::rules::main::selected_sql_codes::selected_sql_codes; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ + SqlExpansionInputs, present, +}; +use crate::bindings::_helpers::rules::lint_expansion::{ + LintExpansionSources, lint_expansion_inputs, +}; +use crate::bindings::_helpers::rules::lint_inputs::{model_key_row, model_lint_expansion}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, +}; +use crate::bindings::_helpers::rules::relation_keys::{ + RelationKeyRow, RelationKeys, relation_keys, +}; +use crate::bindings::_helpers::rules::rule_cache::stored_entries; +use crate::bindings::_helpers::rules::rules_config::rules_config_of; /// The lint thresholds used when the project leaves them unset. const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; -const FUTURES_MODULE: &str = "concurrent.futures"; -const THREADING_MODULE: &str = "threading"; -const LINT_THREAD_PREFIX: &str = "sqlbuild-lint"; +const MAX_LITERAL_LENGTH: &str = "max_literal_length"; +const MAX_RANKING_ORDER_BY: &str = "max_ranking_order_by"; +const LINT_THREAD: &str = "sqlbuild-lint"; const SQL_RULES_CACHE_FILE: &str = "sql.json"; /// The SQL rule result cache file of `project_dir`. -pub(crate) fn sql_rule_cache_path<'py>( - project_dir: &Bound<'py, PyAny>, -) -> PyResult> { - let root: PathBuf = project_dir.extract()?; - Ok(rules_bulk_cache_path(&root, SQL_RULES_CACHE_FILE) - .into_pyobject(project_dir.py())? - .into_any()) +pub(crate) fn sql_rule_cache_path(project_dir: &Path) -> PathBuf { + rules_bulk_cache_path(project_dir, SQL_RULES_CACHE_FILE) } -/// The native SQL lint configuration that SQL rules run with. -pub(crate) fn sql_lint_config<'py>( - dialect: &Bound<'py, PyAny>, +/// The native SQL lint settings that SQL rules run with. +pub(crate) fn sql_lint_settings( + dialect: &str, codes: &[String], - thresholds: &Bound<'py, PyAny>, - (relation_keys, allow_suppressions): (Bound<'py, PyAny>, Bound<'py, PyAny>), -) -> PyResult> { - let py: Python<'py> = dialect.py(); - let threshold = |key: &str, default: i64| -> PyResult> { - thresholds.call_method1("get", (key, default)) - }; - call_with( - ®istry::class(py, RecordId::LintConfig)?, - &[ - ("dialect", dialect.clone()), - ("enabled_native_rules", PyTuple::new(py, codes)?.into_any()), - ("header_rules_enabled", false.into_bound_py_any(py)?), - ( - "max_literal_length", - threshold("max_literal_length", DEFAULT_MAX_LITERAL_LENGTH)?, - ), - ( - "max_ranking_order_by", - threshold("max_ranking_order_by", DEFAULT_MAX_RANKING_ORDER_BY)?, - ), - ("relation_keys", relation_keys), - ("allow_suppressions", allow_suppressions), - ], - ) + threshold: impl Fn(&str) -> PyResult>, + (relation_keys, allow_suppressions): (RelationKeys, bool), +) -> PyResult { + let defaults: LintSettings = LintSettings::default(); + Ok(LintSettings { + header_rules_enabled: false, + sql: LintSqlOptions { + dialect: dialect.to_owned(), + enabled_rules: Some(codes.to_vec()), + max_literal_length: threshold(MAX_LITERAL_LENGTH)? + .unwrap_or(DEFAULT_MAX_LITERAL_LENGTH), + max_ranking_order_by: threshold(MAX_RANKING_ORDER_BY)? + .unwrap_or(DEFAULT_MAX_RANKING_ORDER_BY), + ..defaults.sql + }, + allow_suppressions, + relation_keys, + ..defaults + }) +} + +/// What early SQL lint runs with: the project, its selected SQL rule codes and lint settings. +struct EarlyLintPlan { + project_dir: PathBuf, + codes: Vec, + settings: LintSettings, } -/// The project, SQL rule codes and lint settings early SQL lint runs with, if it should run. -#[pyfunction] -#[pyo3(signature = (*, inputs, dialect))] -fn sql_lint_preparation<'py>( - inputs: &Bound<'py, PyAny>, - dialect: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = inputs.py(); - let native = |name: NativeAttr| registry::native(py, name); - let project_dir: Bound<'py, PyAny> = inputs +/// The plan early SQL lint runs with, when the project selects SQL rules without a warm cache. +fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult> { + let py: Python<'_> = inputs.py(); + let project_dir: Bound<'_, PyAny> = inputs .getattr("discovered_inputs")? .getattr("project_dir")?; if project_dir.is_none() { return Ok(None); } + let project_dir: PathBuf = project_dir.extract()?; for diagnostic in inputs.getattr("diagnostics")?.try_iter()? { if diagnostic?.getattr("is_error")?.is_truthy()? { return Ok(None); } } - let config: Bound<'py, PyAny> = call_with( - &native(NativeAttr::ValidRulesConfig)?, - &[("project_dir", project_dir.clone())], - )?; - if config.is_none() || !config.getattr("select")?.is_truthy()? { + let Ok(config) = rules_config_of(&project_dir)? else { + return Ok(None); + }; + if config.select.is_empty() { return Ok(None); } - if config.getattr("cache")?.getattr("enabled")?.is_truthy()? - && call_with( - &native(NativeAttr::LoadSqlRuleCache)?, - &[("path", sql_rule_cache_path(&project_dir)?)], - )? - .is_truthy()? - { + if config.cache.enabled && !stored_entries(&sql_rule_cache_path(&project_dir))?.is_empty() { return Ok(None); } - let codes: Vec = call_with( - &native(NativeAttr::SelectedSqlRuleCodes)?, - &[ - ( - "select", - PyList::new( - py, - config - .getattr("select")? - .try_iter()? - .collect::>>()?, - )? - .into_any(), - ), - ( - "ignore", - PyList::new( - py, - config - .getattr("ignore")? - .try_iter()? - .collect::>>()?, - )? - .into_any(), - ), - ], - )? - .extract()?; + let codes: Vec = + compiler_guard(|| Ok(selected_sql_codes(&config.select, &config.ignore)))?; if codes.is_empty() { return Ok(None); } - let mut model_rows: Vec> = Vec::new(); + let mut rows: Vec = Vec::new(); for model in inputs.getattr("model_inputs")?.try_iter()? { - model_rows.push(model?.getattr("relation_key_row")?); + rows.push(model_key_row(&model?)?); } - let mut seeds: Vec> = Vec::new(); + let mut seeds: Vec> = Vec::new(); for seed in inputs.getattr("seed_inputs")?.try_iter()? { seeds.push(seed?.getattr("schema_entry")?); } - let mut sources: Vec> = Vec::new(); + let mut sources: Vec> = Vec::new(); for source in inputs.getattr("source_inputs")?.try_iter()? { sources.push(source?.getattr("source_entry")?); } - let relation_keys: Bound<'py, PyAny> = call_with( - &native(NativeAttr::DeclaredRelationKeys)?, - &[ - ("model_rows", PyList::new(py, model_rows)?.into_any()), - ("seeds", PyList::new(py, seeds)?.into_any()), - ("sources", PyList::new(py, sources)?.into_any()), - ], + let keys: RelationKeys = relation_keys( + py, + rows, + ( + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ), )?; - let lint_config: Bound<'py, PyAny> = sql_lint_config( + let settings: LintSettings = sql_lint_settings( dialect, &codes, - &config.getattr("thresholds")?, - (relation_keys, config.getattr("allow_exceptions")?), + |key| Ok(config.thresholds.get(key).map(|value| i64::from(*value))), + (keys, config.allow_exceptions), )?; - Ok(Some(PyTuple::new( - py, - [ - project_dir, - PyTuple::new(py, codes)?.into_any(), - lint_config, - ], - )?)) + Ok(Some(EarlyLintPlan { + project_dir, + codes, + settings, + })) } -/// Each model's compile expansion for early lint, whole when the compile handed one over. -#[pyfunction] -fn early_lint_expansions<'py>(model_inputs: &Bound<'py, PyAny>) -> PyResult> { - let expansions: Bound<'py, PyDict> = PyDict::new(model_inputs.py()); - for model in model_inputs.try_iter()? { - let model: Bound<'py, PyAny> = model?; - let mut expansion: Bound<'py, PyAny> = model.getattr("lint_expansion")?; - if !expansion.is_truthy()? { - expansion = model.getattr("sql_expansion")?; - } - if !expansion.is_none() { - expansions.set_item( - model.getattr("model_file")?.getattr("file_path")?, - expansion, - )?; +/// Early SQL lint's outcome and the expansion inputs it linted with. +pub(crate) struct EarlyLintResult { + pub(crate) outcome: LintOutcome, + pub(crate) context: SqlExpansionInputs, +} + +/// One early SQL lint running on a native thread, for one project and SQL rule codes. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct PreparedSqlLint { + codes: Vec, + project_dir: PathBuf, + thread: Mutex>>>, + result: OnceLock>, +} + +impl PreparedSqlLint { + /// Whether this lint ran for `project_dir` with exactly `codes`. + pub(crate) fn matches(&self, project_dir: &Path, codes: &[String]) -> bool { + self.project_dir == project_dir && same_codes(&self.codes, codes) + } + + /// Whether this lint ran with exactly `codes`. + pub(crate) fn has_codes(&self, codes: &[String]) -> bool { + same_codes(&self.codes, codes) + } + + /// The lint's result, waiting without the GIL until it finishes. + pub(crate) fn result(&self, py: Python<'_>) -> &Result { + self.result.get_or_init(|| { + let thread = self + .thread + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + match thread { + Some(thread) => py.detach(|| thread.join()).unwrap_or_else(|_| { + Err(PyRuntimeError::new_err("early SQL lint thread panicked")) + }), + None => Err(PyRuntimeError::new_err("early SQL lint was already joined")), + } + }) + } +} + +fn same_codes(left: &[String], right: &[String]) -> bool { + left.iter().collect::>() == right.iter().collect::>() +} + +/// Start early SQL lint on a native thread when the project selects SQL rules without a cache. +fn start_early_lint( + inputs: &Bound<'_, PyAny>, + dialect: &str, + stop: &Arc, +) -> PyResult> { + let Some(plan) = early_lint_plan(inputs, dialect)? else { + return Ok(None); + }; + let mut compiled: HashMap = HashMap::new(); + for model in inputs.getattr("model_inputs")?.try_iter()? { + if let Some((path, expansion)) = model_lint_expansion(&model?)? { + compiled.insert(path, expansion); } } - Ok(expansions) + let work: EarlyLintWork = EarlyLintWork { + project_dir: plan.project_dir.clone(), + settings: plan.settings, + discovered: inputs.getattr("discovered_inputs")?.unbind(), + static_scope: inputs.getattr("declaration_scope")?.unbind(), + compiled, + stop: Arc::clone(stop), + }; + let thread: JoinHandle> = std::thread::Builder::new() + .name(LINT_THREAD.to_owned()) + .spawn(move || Python::attach(|py| early_lint(py, &work))) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?; + Ok(Some(PreparedSqlLint { + codes: plan.codes, + project_dir: plan.project_dir, + thread: Mutex::new(Some(thread)), + result: OnceLock::new(), + })) +} + +/// What one early SQL lint thread lints: the project, its settings and the compile's expansions. +struct EarlyLintWork { + project_dir: PathBuf, + settings: LintSettings, + discovered: Py, + static_scope: Py, + compiled: HashMap, + stop: Arc, +} + +/// Expand and lint the project's SQL early, stopping between files once `stop` is set. +fn early_lint(py: Python<'_>, work: &EarlyLintWork) -> PyResult { + let project_dir: &Path = &work.project_dir; + let context: SqlExpansionInputs = lint_expansion_inputs(LintExpansionSources { + project_dir: project_dir.into_pyobject(py)?.into_any(), + value_renderer: None, + discovered_inputs: Some(work.discovered.bind(py).clone()), + declaration_scope: None, + static_declaration_scope: present(work.static_scope.bind(py).clone()), + })?; + let files: Vec = lint_sources(project_dir, None, None)?; + let outcome: LintOutcome = lint( + py, + &LintRequest { + project_dir, + settings: &work.settings, + dynamic_output_paths: &HashSet::new(), + compiled_expansions: &work.compiled, + context: Some(&context), + skip_unexpandable: false, + stop: Some(&work.stop), + }, + files, + )?; + Ok(EarlyLintResult { outcome, context }) +} + +/// The compiler's discovery and declaration scope, which lint expansion reuses for the same discovery. +pub(crate) struct ExpansionReuse { + pub(crate) discovered_inputs: Py, + pub(crate) declaration_scope: Py, } /// Own speculative SQL checks for one compilation invocation. @@ -191,34 +275,40 @@ fn early_lint_expansions<'py>(model_inputs: &Bound<'py, PyAny>) -> PyResult, - #[pyo3(get)] - expansion_reuse: Py, - stop_requested: Py, - executor: Py, + preparation: Option>, + expansion_reuse: Option, + stop_requested: Arc, } impl EarlySqlLint { /// A manager that starts speculative checks only when `enabled`. - pub(crate) fn new(py: Python<'_>, enabled: bool) -> PyResult { - let executor: Bound<'_, PyAny> = call_with( - &imported(py, FUTURES_MODULE, "ThreadPoolExecutor")?, - &[ - ("max_workers", 1.into_bound_py_any(py)?), - ( - "thread_name_prefix", - LINT_THREAD_PREFIX.into_bound_py_any(py)?, - ), - ], - )?; - Ok(Self { + pub(crate) fn new(enabled: bool) -> Self { + Self { enabled, - preparation: py.None(), - expansion_reuse: py.None(), - stop_requested: imported(py, THREADING_MODULE, "Event")?.call0()?.unbind(), - executor: executor.unbind(), - }) + preparation: None, + expansion_reuse: None, + stop_requested: Arc::new(AtomicBool::new(false)), + } + } + + /// The running early SQL lint, if one started. + pub(crate) fn preparation(&self) -> Option<&Py> { + self.preparation.as_ref() + } + + /// The compile's discovery and declaration scope, if it had one. + pub(crate) fn expansion_reuse(&self) -> Option<&ExpansionReuse> { + self.expansion_reuse.as_ref() + } + + /// Abandon unconsumed speculative work and return the milliseconds spent joining it. + pub(crate) fn stop(&self, py: Python<'_>) -> u64 { + let started: Instant = Instant::now(); + self.stop_requested.store(true, Ordering::Release); + if let Some(preparation) = &self.preparation { + let _ = preparation.get().result(py); + } + u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) } } @@ -226,8 +316,8 @@ impl EarlySqlLint { impl EarlySqlLint { #[new] #[pyo3(signature = (*, enabled))] - fn py_new(py: Python<'_>, enabled: bool) -> PyResult { - Self::new(py, enabled) + fn py_new(enabled: bool) -> Self { + Self::new(enabled) } fn __enter__(slf: PyRefMut<'_, Self>) -> PyRefMut<'_, Self> { @@ -235,9 +325,8 @@ impl EarlySqlLint { } #[pyo3(signature = (*_exception))] - fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult<()> { - let _ = self.stop(py)?; - Ok(()) + fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) { + let _ = self.stop(py); } /// Start early SQL lint for `inputs` when enabled and the project selects SQL rules. @@ -245,60 +334,37 @@ impl EarlySqlLint { fn start( mut slf: PyRefMut<'_, Self>, inputs: &Bound<'_, PyAny>, - dialect: &Bound<'_, PyAny>, + dialect: &str, ) -> PyResult<()> { if !slf.enabled { return Ok(()); } let py: Python<'_> = inputs.py(); - let preparation: Bound<'_, PyAny> = call_with( - ®istry::native(py, NativeAttr::PrepareSqlRules)?, - &[ - ("inputs", inputs.clone()), - ("executor", slf.executor.bind(py).clone()), - ("dialect", dialect.clone()), - ("stop", slf.stop_requested.bind(py).clone()), - ], - )?; + let stop: Arc = Arc::clone(&slf.stop_requested); + slf.preparation = start_early_lint(inputs, dialect, &stop)? + .map(|preparation| Py::new(py, preparation)) + .transpose()?; let declaration_scope: Bound<'_, PyAny> = inputs.getattr("declaration_scope")?; - let expansion_reuse: Bound<'_, PyAny> = if declaration_scope.is_none() { - declaration_scope + slf.expansion_reuse = if declaration_scope.is_none() { + None } else { - call_with( - ®istry::class(py, RecordId::SqlExpansionReuse)?, - &[ - ("discovered_inputs", inputs.getattr("discovered_inputs")?), - ("declaration_scope", declaration_scope), - ], - )? + Some(ExpansionReuse { + discovered_inputs: inputs.getattr("discovered_inputs")?.unbind(), + declaration_scope: declaration_scope.unbind(), + }) }; - slf.preparation = preparation.unbind(); - slf.expansion_reuse = expansion_reuse.unbind(); Ok(()) } /// Abandon unconsumed speculative work and return the milliseconds spent joining it. - fn stop(&self, py: Python<'_>) -> PyResult { - let started: Instant = Instant::now(); - let _ = self.stop_requested.bind(py).call_method0("set")?; - let _ = self.executor.bind(py).call_method( - "shutdown", - (), - Some(&keywords( - py, - &[ - ("wait", true.into_bound_py_any(py)?), - ("cancel_futures", true.into_bound_py_any(py)?), - ], - )?), - )?; - Ok(u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)) + #[pyo3(name = "stop")] + fn py_stop(&self, py: Python<'_>) -> u64 { + self.stop(py) } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - module.add_function(wrap_pyfunction!(sql_lint_preparation, module)?)?; - module.add_function(wrap_pyfunction!(early_lint_expansions, module)?)?; + module.add_class::()?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 0c5cd5437c..e8e1ecdba5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -1,6 +1,6 @@ //! Compiler-verified fixed-point repair planning, entirely in memory. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::PathBuf; use pyo3::exceptions::PyException; @@ -11,6 +11,7 @@ use sqlbuild_rules::fix_verification::main::refusal::refusal; use sqlbuild_rules::fix_verification::main::with_edits::with_edits; use sqlbuild_rules::fix_verification::models::{FixEdit, FixEligibility}; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -19,11 +20,20 @@ use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; use crate::bindings::_helpers::rules::fix_facts::compile_facts; use crate::bindings::_helpers::rules::fix_verdict::fix_verdict; +use crate::bindings::_helpers::rules::lint_expansion::{ + LintExpansionSources, lint_expansion_inputs, +}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + LintFile, LintOutcome, LintRequest, LintSettings, ViolationClasses, lint, lint_sources, +}; +use crate::bindings::_helpers::rules::relation_keys::{RelationKeys, project_relation_keys}; +use crate::bindings::_helpers::rules::run_lint::given_files; const VERIFICATION_FAILURE: &str = "Compilation could not prove unchanged output columns, types, dependencies and lineage"; const APPLIED_REASON: &str = "Meaning-preserving rewrite; compiler invariants verified"; const MAX_PASSES: usize = 128; +const SQL_RULE_PREFIX: &str = "SQBRSQL"; struct FixSession<'py> { project_dir: Bound<'py, PyAny>, @@ -141,42 +151,59 @@ impl<'py> FixSession<'py> { model_paths: &Bound<'py, PyAny>, ) -> PyResult> { let py = files.py(); - let relation_keys = self.relation_keys()?; - let config = replace( - &self.config, - &[ - ( - "enabled_native_rules", - PyTuple::new(py, ["SQBRSQL"])?.into_any(), - ), - ("relation_keys", relation_keys), - ], - )?; - let lint = call_with( - ®istry::native(py, NativeAttr::RunLint)?, - &[ - ("project_dir", self.project_dir.clone()), - ("config", config), - ("value_renderer", self.adapter.clone()), - ("discovered_inputs", updated_inputs(&self.inputs, files)?), - ("source_files", files.clone().into_any()), - ( - "selected_paths", - PyFrozenSet::new(py, files.keys())?.into_any(), - ), - ( - "skip_unexpandable", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], + let mut settings: LintSettings = LintSettings::from_record(&self.config)?; + settings.sql.enabled_rules = Some(vec![SQL_RULE_PREFIX.to_owned()]); + settings.relation_keys = self.relation_keys()?; + let project_dir: PathBuf = self.project_dir.extract()?; + let given: Option> = given_files(files.as_any())?; + let selected: BTreeSet = given + .iter() + .flatten() + .map(|file| file.path.clone()) + .collect(); + let sources: Vec = lint_sources(&project_dir, Some(&selected), given)?; + let context: Option = if settings.native_enabled { + Some(lint_expansion_inputs(LintExpansionSources { + project_dir: self.project_dir.clone(), + value_renderer: self.adapter.is_truthy()?.then(|| self.adapter.clone()), + discovered_inputs: Some(updated_inputs(&self.inputs, files)?), + declaration_scope: None, + static_declaration_scope: None, + })?) + } else { + None + }; + let lint: LintOutcome = lint( + py, + &LintRequest { + project_dir: &project_dir, + settings: &settings, + dynamic_output_paths: &HashSet::new(), + compiled_expansions: &HashMap::new(), + context: context.as_ref(), + skip_unexpandable: true, + stop: None, + }, + sources, )?; - self.skipped - .call_method1("update", (lint.getattr("unexpandable")?,))?; + let paths: Vec> = lint + .files + .iter() + .map(|file| Ok(file.path.clone().into_pyobject(py)?.into_any())) + .collect::>()?; + for (file, message) in &lint.unexpandable { + self.skipped.set_item(&paths[*file], message)?; + } + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; + let violations: Vec> = lint + .violations + .iter() + .map(|row| classes.violation(&paths[row.file], row)) + .collect::>()?; let edits = PyDict::new(py); let mut pending: Vec> = Vec::new(); let mut remaining: Vec> = Vec::new(); - for violation in lint.getattr("violations")?.try_iter()? { - let violation = violation?; + for violation in violations { if !violation .getattr("code")? .extract::()? @@ -353,7 +380,7 @@ impl<'py> FixSession<'py> { Ok(baseline) } - fn relation_keys(&mut self) -> PyResult> { + fn relation_keys(&mut self) -> PyResult { let py = self.inputs.py(); let baseline = match self.baseline() { Ok(value) => value, @@ -362,11 +389,11 @@ impl<'py> FixSession<'py> { .value(py) .is_instance(®istry::error(py, ErrorId::CompileInput)?)? => { - return Ok(PyDict::new(py).into_any()); + return Ok(RelationKeys::new()); } Err(error) => return Err(error), }; - registry::native(py, NativeAttr::CompiledRelationKeys)?.call1((baseline,)) + project_relation_keys(&baseline) } fn verdict(&mut self, files: &Bound<'py, PyDict>) -> PyResult>> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 6fdc2696b1..54e7413b52 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -39,31 +39,6 @@ fn prepare_lint_sql( .map_err(value_error) } -/// Prepare many bodies in one detached call; `false` marks a body to prepare again individually. -#[pyfunction] -fn prepare_lint_sql_batch( - py: Python<'_>, - requests: Vec, -) -> PyResult)>> { - py.compiler_detach(|| { - Ok(requests - .iter() - .map(|request| { - sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic(|| { - sqlbuild_rules::sql_lint::main::preparation::prepare( - &request.expanded, - &request.before_expansion, - &request.prior_sites, - &request.dialect, - ) - }) - .map_or((false, None), |prepared| (true, prepared)) - }) - .collect()) - }) - .map_err(value_error) -} - #[pyfunction] fn lint_backtick_identifiers(dialect: &str) -> PyResult { compiler_guard(|| { @@ -96,7 +71,6 @@ fn format_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; - module.add_function(wrap_pyfunction!(prepare_lint_sql_batch, module)?)?; module.add_function(wrap_pyfunction!(lint_backtick_identifiers, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs index 6ab29ec336..33db3aa639 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs @@ -6,10 +6,11 @@ use pyo3::prelude::{ use pyo3::types::{PyBool, PyDict}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_attachment::sql_expansion_context::build_sql_expansion_context; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ + ExpansionSources, SqlExpansionInputs, present, sql_expansion_inputs, }; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr}; @@ -39,59 +40,75 @@ pub(crate) fn build_lint_expansion_context<'py>( ], ), )?; - expansion_context(py, &arguments).or_else(|error| Err(compile_first_error(py, error)?)) + let renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; + lint_expansion_inputs(LintExpansionSources { + project_dir: arguments.value("project_dir")?, + value_renderer: renderer.is_truthy()?.then_some(renderer), + discovered_inputs: present(arguments.value("discovered_inputs")?), + declaration_scope: present(arguments.value("declaration_scope")?), + static_declaration_scope: present(arguments.value("static_declaration_scope")?), + })? + .record(py) } -fn expansion_context<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, -) -> PyResult> { - let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; - let mut discovered: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - if discovered.is_none() { - let no: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); - discovered = call_with( - ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, - &[ - ("project_dir", project_dir.clone()), - ("sql_analysis_enabled_override", no.clone()), - ("extract_output_column_locations", no), - ], - )?; - } - let mut renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; - if !renderer.is_truthy()? { - let adapter_name: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ResolveEffectiveAdapterName)?, - &[ - ("project_config", discovered.getattr("project_config")?), - ("local_config", discovered.getattr("local_config")?), - ], - )?; - renderer = call_with( - &host::entry(py, AdapterHost::ResolveAdapter)?, - &[ - ("adapter_name", adapter_name), - ("project_dir", project_dir.clone()), - ], - )?; - } - build_sql_expansion_context( - py, - Some(keywords( - py, - &[ - ("project_dir", project_dir), - ("discovered_inputs", discovered), - ("value_renderer", renderer), - ("declaration_scope", arguments.value("declaration_scope")?), - ( - "static_declaration_scope", - arguments.value("static_declaration_scope")?, - ), - ], - )?), - ) +/// Where lint's expansion inputs come from; a missing renderer is the project's adapter. +pub(crate) struct LintExpansionSources<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) value_renderer: Option>, + pub(crate) discovered_inputs: Option>, + pub(crate) declaration_scope: Option>, + pub(crate) static_declaration_scope: Option>, +} + +/// Lint's expansion inputs, reporting compile failures as `ProjectCompileError`. +pub(crate) fn lint_expansion_inputs( + sources: LintExpansionSources<'_>, +) -> PyResult { + let py: Python<'_> = sources.project_dir.py(); + expansion_inputs(sources).or_else(|error| Err(compile_first_error(py, error)?)) +} + +fn expansion_inputs(sources: LintExpansionSources<'_>) -> PyResult { + let py: Python<'_> = sources.project_dir.py(); + let project_dir: Bound<'_, PyAny> = sources.project_dir; + let discovered: Bound<'_, PyAny> = match sources.discovered_inputs { + Some(discovered) => discovered, + None => { + let no: Bound<'_, PyAny> = PyBool::new(py, false).to_owned().into_any(); + call_with( + ®istry::native(py, NativeAttr::DiscoverProjectInputs)?, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", no.clone()), + ("extract_output_column_locations", no), + ], + )? + } + }; + let renderer: Bound<'_, PyAny> = match sources.value_renderer { + Some(renderer) => renderer, + None => { + let adapter_name: Bound<'_, PyAny> = resolve_effective_adapter_name( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + )?; + call_with( + &host::entry(py, AdapterHost::ResolveAdapter)?, + &[ + ("adapter_name", adapter_name), + ("project_dir", project_dir.clone()), + ], + )? + } + }; + sql_expansion_inputs(ExpansionSources { + project_dir, + value_renderer: renderer, + cli_vars: None, + discovered_inputs: Some(discovered), + declaration_scope: sources.declaration_scope, + static_declaration_scope: sources.static_declaration_scope, + }) } /// Adapter, compile and discovery errors as `ProjectCompileError`; any other error unchanged. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs index 0d34892772..05251751e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs @@ -1,13 +1,16 @@ -//! Route lint bodies through the existing native declaration and header scanners. +//! The path-derived role of one lint input. -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; -use pyo3::types::PyDict; +use pyo3::types::PyBool; +use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +const HOOKS_FOLDER: &str = "hooks"; + /// Classify a lint path, retaining the caller's supplied relative path when available. #[pyfunction] #[pyo3(signature = (*, file_path, project_dir, relative_path=None))] @@ -17,73 +20,34 @@ pub(crate) fn lint_file_role<'py>( relative_path: Option>, ) -> PyResult> { let py = file_path.py(); - let relative = if let Some(relative) = relative_path { - relative - } else { - if !file_path - .call_method1("is_relative_to", (project_dir,))? - .is_truthy()? - { - return registry::class(py, RecordId::LintFileRole)?.call0(); + let relative: Bound<'py, PyAny> = match relative_path { + Some(relative) => relative, + None => { + if !file_path + .call_method1("is_relative_to", (project_dir,))? + .is_truthy()? + { + return registry::class(py, RecordId::LintFileRole)?.call0(); + } + file_path.call_method1("relative_to", (project_dir,))? } - file_path.call_method1("relative_to", (project_dir,))? + }; + let parts: Vec = relative.getattr("parts")?.extract()?; + let kind: Bound<'py, PyAny> = match named_declaration_role_kind(&parts) { + Some(kind) => registry::class(py, RecordId::DeclarationKind)?.call1((kind,))?, + None => py.None().into_bound(py), }; call_with( ®istry::class(py, RecordId::LintFileRole)?, &[ - ( - "declaration_kind", - call_with( - ®istry::native(py, NativeAttr::NamedDeclarationRoleKind)?, - &[("relative_path", relative.clone())], - )?, - ), - ("in_project", true.into_pyobject(py)?.to_owned().into_any()), + ("declaration_kind", kind), + ("in_project", PyBool::new(py, true).to_owned().into_any()), ( "in_hook_directory", - relative.call_method1("is_relative_to", ("hooks",))?, + PyBool::new(py, parts.first().is_some_and(|part| part == HOOKS_FOLDER)) + .to_owned() + .into_any(), ), ], ) } - -/// Use audit measurement ranges when present, otherwise the native SQL header boundaries. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn lint_body_ranges<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "lint_body_ranges", - options, - ( - &["contents", "headers", "file_path", "project_dir"], - &["role"], - ), - )?; - let mut role = args.value("role")?; - if role.is_none() { - role = lint_file_role(&args.value("file_path")?, &args.value("project_dir")?, None)?; - } - let kind = role.getattr("declaration_kind")?; - if role.getattr("in_project")?.is_truthy()? - && (kind.eq("audit")? || kind.eq("singular_audit")?) - { - let ranges = call_with( - ®istry::native(py, NativeAttr::MeasurementBodyRanges)?, - &[("contents", args.value("contents")?)], - )?; - if ranges.is_truthy()? { - return Ok(ranges); - } - } - call_with( - ®istry::native(py, NativeAttr::SqlBodyRanges)?, - &[ - ("contents", args.value("contents")?), - ("headers", args.value("headers")?), - ], - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs new file mode 100644 index 0000000000..70589cf566 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -0,0 +1,115 @@ +//! Key rows and compiled expansions lint reads natively from compile inputs and compiled projects. + +use std::collections::HashMap; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyStringMethods; + +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + ExpansionRecord, ModelInputRecord, NativeModelInput, model_relation_key_row, +}; +use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; +use crate::bindings::_helpers::rules::lint_pipeline::{CompiledExpansion, span_pass}; +use crate::bindings::_helpers::rules::relation_keys::RelationKeyRow; + +const FILE_PATH_ATTRIBUTE: &str = "file_path"; + +/// The declared key facts of a model input or compiled model. +pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + if let Ok(input) = model.downcast::() { + return input.get().read(py, model_relation_key_row); + } + if let Ok(compiled) = model.downcast::() { + return compiled.get().read(py, model_relation_key_row); + } + model.getattr("relation_key_row")?.extract() +} + +/// The model file path of a native input record. +fn record_path(record: &ModelInputRecord, py: Python<'_>) -> PyResult { + record + .model_file + .bind(py) + .getattr(FILE_PATH_ATTRIBUTE)? + .extract() +} + +/// An expansion record with the given passes. +fn expansion( + expansion: &ExpansionRecord, + passes: &[&Vec<(usize, usize, usize, usize)>], + py: Python<'_>, +) -> PyResult { + Ok(CompiledExpansion { + authored_sql: expansion.authored_sql.bind(py).to_str()?.to_owned(), + expanded_sql: expansion.expanded_sql.bind(py).to_str()?.to_owned(), + passes: passes.iter().map(|rows| span_pass(rows)).collect(), + }) +} + +/// A model input's whole compile expansion, variables included, keyed by its file path. +pub(crate) fn model_lint_expansion( + model: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = model.py(); + if let Ok(input) = model.downcast::() { + return input.get().read(py, |record, py| { + let whole: &ExpansionRecord = &record.expansion; + Ok(Some(( + record_path(record, py)?, + expansion( + whole, + &[ + &whole.interpolation_spans, + &whole.declaration_spans, + &whole.macro_spans, + ], + py, + )?, + ))) + }); + } + let mut record: Bound<'_, PyAny> = model.getattr("lint_expansion")?; + if !record.is_truthy()? { + record = model.getattr("sql_expansion")?; + } + if record.is_none() { + return Ok(None); + } + let path: PathBuf = model + .getattr("model_file")? + .getattr(FILE_PATH_ATTRIBUTE)? + .extract()?; + Ok(Some((path, CompiledExpansion::from_record(&record)?))) +} + +/// A compiled project's authored expansions: models whose interpolation left the query unchanged. +pub(crate) fn project_sql_expansions( + project: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = project.py(); + let mut expansions: HashMap = HashMap::new(); + if let Ok(view) = project.downcast::() { + let owner = view.get().owner().bind(py).borrow(); + for record in owner.inputs() { + let authored: &ExpansionRecord = &record.expansion; + if authored.authored { + expansions.insert( + record_path(record, py)?, + expansion( + authored, + &[&authored.declaration_spans, &authored.macro_spans], + py, + )?, + ); + } + } + return Ok(expansions); + } + crate::bindings::_helpers::rules::run_lint::compiled_expansion_records( + &project.getattr("sql_expansions")?, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs new file mode 100644 index 0000000000..a392e5efcf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -0,0 +1,1107 @@ +//! The lint pipeline over native data: header rules, expanded bodies, native SQL lint and suppressions. + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_rules::dsl_headers::main::measurement_body_ranges::measurement_body_ranges; +use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers; +use sqlbuild_rules::dsl_headers::main::sql_body_ranges::sql_body_ranges; +use sqlbuild_rules::dsl_headers::models::{ + HeaderRulesConfig, HeaderSeverity, HeaderSpan, HeaderViolation, +}; +use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; +use sqlbuild_rules::lint_text::main::apply_suppressions::apply_suppressions; +use sqlbuild_rules::lint_text::main::dependency_sites::dependency_sites; +use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; +use sqlbuild_rules::lint_text::main::lint_sql_bodies::lint_sql_bodies; +use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind; +use sqlbuild_rules::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; +use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; +use sqlbuild_rules::lint_text::models::{ + AuthoredDiagnostic, ExpansionSpan, InterpolationSite, LintSqlBody, LintSqlOptions, Neutralized, + RetainedViolation, SuppressibleViolation, SuppressionFinding, +}; +use sqlbuild_rules::sql_lint::main::preparation::prepare; +use sqlbuild_rules::sql_lint::types::PreparedSql; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; +use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::render::authored_sql::{AuthoredExpansion, expand_with_inputs}; +use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::rules::dsl_headers::header_violations; +use crate::bindings::_helpers::rules::native_header_parser::{ + HeaderFailure, NativeHeaderParser, external_identifiers, +}; +use crate::bindings::types::CompilerDetach; + +pub(crate) const NATIVE_ENGINE: &str = "native"; +const HEADER_ENGINE: &str = "sqlbuild"; +const HOOKS_FOLDER: &str = "hooks"; +const AUDIT_KIND: &str = "audit"; +const SINGULAR_AUDIT_KIND: &str = "singular_audit"; +const SQL_HOOK_KIND: &str = "sql_hook"; +const CEREMONIAL_KINDS: [&str; 2] = ["TEST", "SCENARIO"]; +const INTERPOLATION_MARKERS: [&str; 2] = ["@", "${"]; +const FAULT_SEVERITY: &str = "FAULT"; +const WARNING_SEVERITY: &str = "WARNING"; + +/// Resolved lint settings; the defaults are `LintConfig`'s. +#[derive(Clone, Debug)] +pub(crate) struct LintSettings { + pub(crate) native_enabled: bool, + pub(crate) header_rules_enabled: bool, + pub(crate) header_rules: HeaderRulesConfig, + pub(crate) sql: LintSqlOptions, + pub(crate) allow_suppressions: bool, + /// Declared keys of each relation, in declaration order. + pub(crate) relation_keys: Vec<(String, Vec>)>, +} + +impl Default for LintSettings { + fn default() -> Self { + Self { + native_enabled: true, + header_rules_enabled: true, + header_rules: HeaderRulesConfig { + line_width: 100, + max_description_lines: 10, + }, + sql: LintSqlOptions { + dialect: "generic".to_owned(), + enabled_rules: None, + ignored_rules: Vec::new(), + max_literal_length: 100, + max_ranking_order_by: 5, + }, + allow_suppressions: true, + relation_keys: Vec::new(), + } + } +} + +impl LintSettings { + /// The settings a `LintConfig` record holds. + pub(crate) fn from_record(config: &Bound<'_, PyAny>) -> PyResult { + let mut relation_keys: Vec<(String, Vec>)> = Vec::new(); + for item in config + .getattr("relation_keys")? + .call_method0("items")? + .try_iter()? + { + relation_keys.push(item?.extract()?); + } + Ok(Self { + native_enabled: config.getattr("native_enabled")?.is_truthy()?, + header_rules_enabled: config.getattr("header_rules_enabled")?.is_truthy()?, + header_rules: HeaderRulesConfig { + line_width: config.getattr("line_width")?.extract()?, + max_description_lines: config.getattr("max_description_lines")?.extract()?, + }, + sql: LintSqlOptions { + dialect: config.getattr("dialect")?.extract()?, + enabled_rules: config.getattr("enabled_native_rules")?.extract()?, + ignored_rules: config.getattr("ignored_native_rules")?.extract()?, + max_literal_length: config.getattr("max_literal_length")?.extract()?, + max_ranking_order_by: config.getattr("max_ranking_order_by")?.extract()?, + }, + allow_suppressions: config.getattr("allow_suppressions")?.is_truthy()?, + relation_keys, + }) + } + + /// The declared keys of `relation`, when it declares any. + fn keys_of(&self, relation: &str) -> Option<&Vec>> { + self.relation_keys + .iter() + .find(|(name, _)| name == relation) + .map(|(_, keys)| keys) + .filter(|keys| !keys.is_empty()) + } +} + +/// One authored SQL file and its text. +#[derive(Clone, Debug)] +pub(crate) struct LintFile { + pub(crate) path: PathBuf, + pub(crate) contents: String, +} + +/// A compile's authored-to-expanded SQL for one file, reused instead of expanding again. +#[derive(Clone, Debug)] +pub(crate) struct CompiledExpansion { + pub(crate) authored_sql: String, + pub(crate) expanded_sql: String, + pub(crate) passes: Vec>, +} + +impl CompiledExpansion { + /// The expansion a `CompiledSqlExpansion` record holds. + pub(crate) fn from_record(record: &Bound<'_, PyAny>) -> PyResult { + let mut passes: Vec> = Vec::new(); + for pass in record.getattr("passes")?.try_iter()? { + let mut spans: Vec = Vec::new(); + for span in pass?.try_iter()? { + let span: Bound<'_, PyAny> = span?; + spans.push(ExpansionSpan { + source_start: span.getattr("source_start")?.extract()?, + source_end: span.getattr("source_end")?.extract()?, + output_start: span.getattr("output_start")?.extract()?, + output_end: span.getattr("output_end")?.extract()?, + }); + } + passes.push(spans); + } + Ok(Self { + authored_sql: record.getattr("authored_sql")?.extract()?, + expanded_sql: record.getattr("expanded_sql")?.extract()?, + passes, + }) + } +} + +/// Expansion span rows as lint spans. +pub(crate) fn span_pass(rows: &[SpanRow]) -> Vec { + rows.iter() + .map( + |&(source_start, source_end, output_start, output_end)| ExpansionSpan { + source_start: offset(source_start), + source_end: offset(source_end), + output_start: offset(output_start), + output_end: offset(output_end), + }, + ) + .collect() +} + +fn offset(value: usize) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) +} + +/// A proposed authored-source replacement in the violation's file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct LintEditRow { + pub(crate) start: i64, + pub(crate) end: i64, + pub(crate) replacement: String, +} + +/// One lint violation against the file at index `file` of its run. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct LintViolationRow { + pub(crate) file: usize, + pub(crate) line: i64, + pub(crate) column: i64, + pub(crate) code: String, + pub(crate) message: String, + pub(crate) fault: bool, + pub(crate) engine: &'static str, + pub(crate) end: Option<(i64, i64)>, + pub(crate) remediation: Option, + pub(crate) fix: Option, + pub(crate) fix_unavailable_reason: Option, +} + +impl LintViolationRow { + pub(crate) fn header(file: usize, violation: HeaderViolation) -> Self { + Self { + file, + line: offset(violation.line), + column: offset(violation.column), + code: violation.code.to_owned(), + message: violation.message, + fault: violation.severity == HeaderSeverity::Fault, + engine: HEADER_ENGINE, + end: None, + remediation: Some(violation.remediation), + fix: None, + fix_unavailable_reason: None, + } + } + + pub(crate) fn diagnostic(diagnostic: AuthoredDiagnostic) -> Self { + Self { + file: diagnostic.file, + line: diagnostic.line, + column: diagnostic.column, + code: diagnostic.code, + message: diagnostic.message, + fault: diagnostic.fault, + engine: NATIVE_ENGINE, + end: diagnostic.end, + remediation: Some(diagnostic.remediation), + fix: diagnostic.fix.map(|edit| LintEditRow { + start: edit.start, + end: edit.end, + replacement: edit.replacement, + }), + fix_unavailable_reason: diagnostic.fix_unavailable_reason, + } + } + + pub(crate) fn suppression(finding: SuppressionFinding) -> Self { + Self { + file: finding.file, + line: offset(finding.line), + column: 1, + code: SUPPRESSION_CODE.to_owned(), + message: finding.message, + fault: finding.fault, + engine: NATIVE_ENGINE, + end: None, + remediation: Some(finding.remediation.to_owned()), + fix: finding.fix.map(|(start, end)| LintEditRow { + start: offset(start), + end: offset(end), + replacement: String::new(), + }), + fix_unavailable_reason: finding.fix_unavailable_reason.map(str::to_owned), + } + } +} + +/// The lint model classes violations are projected as. +pub(crate) struct ViolationClasses<'py> { + violation: Bound<'py, PyAny>, + edit: Bound<'py, PyAny>, + severity: Bound<'py, PyAny>, +} + +impl<'py> ViolationClasses<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + Ok(Self { + violation: registry::class(py, RecordId::LintViolation)?, + edit: registry::class(py, RecordId::LintEdit)?, + severity: registry::class(py, RecordId::LintSeverity)?, + }) + } + + /// `row` as a `LintViolation` against `file_path`. + pub(crate) fn violation( + &self, + file_path: &Bound<'py, PyAny>, + row: &LintViolationRow, + ) -> PyResult> { + let py: Python<'py> = file_path.py(); + let none = || py.None().into_bound(py); + let text = |value: &Option| { + value + .as_deref() + .map_or_else(none, |text| PyString::new(py, text).into_any()) + }; + let code: Bound<'py, PyAny> = PyString::new(py, &row.code).into_any(); + let fix: Bound<'py, PyAny> = match &row.fix { + Some(edit) => call_with( + &self.edit, + &[ + ("file_path", file_path.clone()), + ("code", code.clone()), + ("start", edit.start.into_pyobject(py)?.into_any()), + ("end", edit.end.into_pyobject(py)?.into_any()), + ( + "replacement", + PyString::new(py, &edit.replacement).into_any(), + ), + ], + )?, + None => none(), + }; + let (end_line, end_column) = match row.end { + Some((line, column)) => ( + line.into_pyobject(py)?.into_any(), + column.into_pyobject(py)?.into_any(), + ), + None => (none(), none()), + }; + let severity: &str = if row.fault { + FAULT_SEVERITY + } else { + WARNING_SEVERITY + }; + call_with( + &self.violation, + &[ + ("file_path", file_path.clone()), + ("line", row.line.into_pyobject(py)?.into_any()), + ("column", row.column.into_pyobject(py)?.into_any()), + ("code", code), + ("message", PyString::new(py, &row.message).into_any()), + ("severity", self.severity.getattr(severity)?), + ("engine", PyString::new(py, row.engine).into_any()), + ("end_line", end_line), + ("end_column", end_column), + ("remediation", text(&row.remediation)), + ("fix", fix), + ("fix_unavailable_reason", text(&row.fix_unavailable_reason)), + ], + ) + } +} + +/// Every linted file, its sorted violations and the files expansion skipped. +pub(crate) struct LintOutcome { + /// The linted files in the order they were given. + pub(crate) files: Vec, + pub(crate) violations: Vec, + /// Skipped files by index, with why they could not be expanded. + pub(crate) unexpandable: Vec<(usize, String)>, +} + +impl LintOutcome { + /// This outcome as a `LintRunResult`. + pub(crate) fn record<'py>(&self, py: Python<'py>) -> PyResult> { + let paths: Vec> = self + .files + .iter() + .map(|file| Ok(file.path.clone().into_pyobject(py)?.into_any())) + .collect::>()?; + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; + let violations: Vec> = self + .violations + .iter() + .map(|row| classes.violation(&paths[row.file], row)) + .collect::>()?; + let texts: Bound<'py, PyDict> = PyDict::new(py); + for (path, file) in paths.iter().zip(&self.files) { + texts.set_item(path, &file.contents)?; + } + let unexpandable: Bound<'py, PyDict> = PyDict::new(py); + for (file, message) in &self.unexpandable { + unexpandable.set_item(&paths[*file], message)?; + } + call_with( + ®istry::class(py, RecordId::LintRunResult)?, + &[ + ( + "files_checked", + self.files.len().into_pyobject(py)?.into_any(), + ), + ("violations", PyTuple::new(py, violations)?.into_any()), + ("formatted_files", PyTuple::empty(py).into_any()), + ("source_texts", texts.into_any()), + ("unexpandable", unexpandable.into_any()), + ], + ) + } +} + +/// What one lint run reads besides its files. +pub(crate) struct LintRequest<'a> { + pub(crate) project_dir: &'a Path, + pub(crate) settings: &'a LintSettings, + /// Resolved paths of models whose output columns a dynamic `*` proves. + pub(crate) dynamic_output_paths: &'a HashSet, + pub(crate) compiled_expansions: &'a HashMap, + /// The expansion inputs; without them bodies are not expanded or SQL-linted. + pub(crate) context: Option<&'a SqlExpansionInputs>, + pub(crate) skip_unexpandable: bool, + /// Set once the run's result is no longer needed. + pub(crate) stop: Option<&'a AtomicBool>, +} + +/// `path` resolved through the filesystem, or unchanged when it cannot be. +pub(crate) fn resolved(path: &Path) -> PathBuf { + fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +/// `given` filtered to the selection, or the project's lint files read from disk. +pub(crate) fn lint_sources( + project_dir: &Path, + selected: Option<&BTreeSet>, + given: Option>, +) -> PyResult> { + if let Some(given) = given { + return Ok(given + .into_iter() + .filter(|file| selected.is_none_or(|paths| paths.contains(&resolved(&file.path)))) + .collect()); + } + let collected: Vec<(PathBuf, String)> = + sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files( + project_dir, + selected, + ) + .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.to_string()))?; + Ok(collected + .into_iter() + .map(|(path, contents)| LintFile { path, contents }) + .collect()) +} + +/// Raise `LintStoppedError` once `stop` is set. +fn check_stop(py: Python<'_>, stop: Option<&AtomicBool>) -> PyResult<()> { + if stop.is_some_and(|stop| stop.load(Ordering::Acquire)) { + return Err(PyErr::from_value( + registry::error(py, ErrorId::LintStopped)?.call0()?, + )); + } + Ok(()) +} + +/// Where one body's expanded SQL comes from. +enum BodySource { + /// The compile's expansion of the same authored SQL. + Compiled(String, Vec>), + /// Text without interpolation, linted as written. + Plain, + /// Text that user macros expand. + Expand, +} + +/// One authored body, neutralized and ready to expand. +struct PlannedBody { + start: usize, + input: String, + sites: Vec, + source: BodySource, +} + +/// How a file's bodies expand, shared by each of its bodies. +struct BodyTraits { + external_identifiers: Vec, + ceremonial: bool, + dynamic_star: bool, +} + +/// One file's header violations and, when expanding, its planned bodies. +struct FilePlan { + file: usize, + header_violations: Vec, + expansion: Option), HeaderFailure>>, +} + +/// Why a file could not be planned. +enum PlanFailure { + OutsideProject { path: PathBuf }, + Native(String), +} + +/// One expanded body awaiting native preparation. +struct PendingBody { + start: usize, + input: String, + expanded: String, + passes: Vec>, + sites: Vec, +} + +/// One file's expanded bodies and the expansion failure that stopped it, if any. +struct ExpandedFile { + file: usize, + traits: BodyTraits, + bodies: Vec, + failure: Option, +} + +fn path_parts(relative: &Path) -> Vec { + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect() +} + +/// Lint `files`: header rules, then expanded bodies through the native SQL engine. +pub(crate) fn lint( + py: Python<'_>, + request: &LintRequest<'_>, + files: Vec, +) -> PyResult { + let mut order: Vec = (0..files.len()).collect(); + order.sort_by(|left, right| files[*left].path.cmp(&files[*right].path)); + let expanding: bool = request.context.is_some(); + let plans: Vec> = py + .compiler_detach(|| Ok(plan_files(request, &files, &order, expanding))) + .map_err(value_error)?; + let mut header_rows: Vec = Vec::new(); + let mut expanded_files: Vec = Vec::new(); + let mut scan_failure: Option = None; + for plan in plans { + let plan: FilePlan = match plan { + Ok(plan) => plan, + Err(failure) => { + scan_failure = Some(plan_error(request.project_dir, failure)); + break; + } + }; + header_rows.extend( + plan.header_violations + .into_iter() + .map(|violation| LintViolationRow::header(plan.file, violation)), + ); + let (Some(expansion), Some(context)) = (plan.expansion, request.context) else { + continue; + }; + check_stop(py, request.stop)?; + let (traits, bodies) = match expansion { + Ok(planned) => planned, + Err(failure) => { + scan_failure = Some(failure.into_pyerr(py)); + break; + } + }; + let file: &LintFile = &files[plan.file]; + let expanded: ExpandedFile = + match expand_bodies(py, context, file, (plan.file, traits, bodies)) { + Ok(expanded) => expanded, + Err(error) if error.is_instance_of::(py) => { + scan_failure = Some(error); + break; + } + Err(error) => return Err(error), + }; + let failed: bool = expanded.failure.is_some(); + expanded_files.push(expanded); + if failed && !request.skip_unexpandable { + break; + } + } + check_stop(py, request.stop)?; + let (bodies, unexpandable) = prepared_bodies(py, request, &files, expanded_files)?; + if let Some(error) = scan_failure { + return Err(error); + } + let mut rows: Vec = header_rows; + if !bodies.is_empty() && request.settings.native_enabled { + check_stop(py, request.stop)?; + let options: &LintSqlOptions = &request.settings.sql; + let diagnostics: Vec = py + .compiler_detach(|| { + let texts: Vec<&str> = files.iter().map(|file| file.contents.as_str()).collect(); + Ok(lint_sql_bodies(&bodies, &texts, options)) + }) + .map_err(value_error)?; + rows.extend(diagnostics.into_iter().map(LintViolationRow::diagnostic)); + check_stop(py, request.stop)?; + } + let violations: Vec = + suppressed(&files, rows, request.settings.allow_suppressions)?; + Ok(LintOutcome { + files, + violations, + unexpandable, + }) +} + +/// Each file's headers, header violations and bodies in path order, up to the first failure. +fn plan_files( + request: &LintRequest<'_>, + files: &[LintFile], + order: &[usize], + expanding: bool, +) -> Vec> { + let mut plans: Vec> = Vec::new(); + for &index in order { + let plan: Result = + plan_file(request, &files[index], index, expanding); + let failed: bool = plan.is_err(); + plans.push(plan); + if failed { + break; + } + } + plans +} + +fn plan_file( + request: &LintRequest<'_>, + file: &LintFile, + index: usize, + expanding: bool, +) -> Result { + let Ok(relative) = file.path.strip_prefix(request.project_dir) else { + return Err(PlanFailure::OutsideProject { + path: file.path.clone(), + }); + }; + let parts: Vec = path_parts(relative); + let text: Vec = file.contents.chars().collect(); + let headers: Vec = scan_headers(&text, first_header_only(&parts)); + let header_violations: Vec = if request.settings.header_rules_enabled { + header_violations(&text, &headers, request.settings.header_rules) + .map_err(PlanFailure::Native)? + } else { + Vec::new() + }; + let expansion = expanding.then(|| plan_bodies(request, file, (&text, &headers), &parts)); + Ok(FilePlan { + file: index, + header_violations, + expansion, + }) +} + +/// A file's bodies, neutralized and matched against the compile's expansion. +fn plan_bodies( + request: &LintRequest<'_>, + file: &LintFile, + (text, headers): (&[char], &[HeaderSpan]), + parts: &[String], +) -> Result<(BodyTraits, Vec), HeaderFailure> { + let kind: Option<&str> = named_declaration_role_kind(parts); + let in_hook_directory: bool = parts.first().is_some_and(|part| part == HOOKS_FOLDER); + let dynamic_star: bool = !request.dynamic_output_paths.is_empty() + && request.dynamic_output_paths.contains(&resolved(&file.path)); + let compiled: Option<&CompiledExpansion> = request.compiled_expansions.get(&file.path); + let external: Vec = external_identifiers(text, headers, &NativeHeaderParser)?; + let ceremonial: bool = headers + .iter() + .any(|header| CEREMONIAL_KINDS.contains(&header.kind.as_str())); + let mut ranges: Vec<(usize, usize)> = Vec::new(); + if matches!(kind, Some(AUDIT_KIND | SINGULAR_AUDIT_KIND)) { + ranges = measurement_body_ranges(text); + } + if ranges.is_empty() { + ranges = sql_body_ranges(text, headers); + } + let neutralize: Option Neutralized> = neutralizer(&BodyRole { + kind, + in_project: true, + in_hook_directory, + }); + let bodies: Vec = ranges + .into_iter() + .map(|range| plan_body(text, range, neutralize, compiled)) + .collect(); + Ok(( + BodyTraits { + external_identifiers: external, + ceremonial, + dynamic_star, + }, + bodies, + )) +} + +/// A lint file's path classification. +pub(crate) struct BodyRole<'a> { + pub(crate) kind: Option<&'a str>, + pub(crate) in_project: bool, + pub(crate) in_hook_directory: bool, +} + +/// How a body's interpolation is neutralized before expansion: audit parameters, or hook context. +fn neutralizer(role: &BodyRole<'_>) -> Option Neutralized> { + if role.in_project && role.kind == Some(AUDIT_KIND) { + Some(neutralize_audit_parameters) + } else if role.in_hook_directory || (role.in_project && role.kind == Some(SQL_HOOK_KIND)) { + Some(neutralize_context_interpolation) + } else { + None + } +} + +/// One authored body expanded and prepared for native SQL lint, outside a lint run. +pub(crate) struct SingleBody { + pub(crate) body: LintSqlBody, + pub(crate) body_end: usize, + pub(crate) dependency_relations: Vec<(String, String)>, +} + +/// One body to expand and prepare outside a lint run. +pub(crate) struct SingleBodyRequest<'a, 'py> { + pub(crate) context: &'a SqlExpansionInputs, + pub(crate) file_path: &'a Bound<'py, PyAny>, + pub(crate) contents: &'a str, + pub(crate) range: (usize, usize), + pub(crate) role: BodyRole<'a>, + pub(crate) dialect: &'a str, +} + +/// Expand and prepare the body at `range` of `contents` as `role` reads it. +pub(crate) fn prepare_single_body(request: &SingleBodyRequest<'_, '_>) -> PyResult { + let SingleBodyRequest { + context, + file_path, + contents, + range, + role, + dialect, + } = request; + let (context, file_path, range, dialect): ( + &SqlExpansionInputs, + &Bound<'_, PyAny>, + (usize, usize), + &str, + ) = (context, file_path, *range, dialect); + let py: Python<'_> = file_path.py(); + let text: Vec = contents.chars().collect(); + let planned: PlannedBody = plan_body(&text, range, neutralizer(role), None); + let (expanded, passes) = match planned.source { + BodySource::Expand => { + let expansion: AuthoredExpansion = + expand_with_inputs(context, &planned.input, file_path) + .map_err(|error| expansion_error(file_path, error))?; + ( + expansion.sql, + expansion + .passes + .iter() + .map(|rows| span_pass(rows)) + .collect(), + ) + } + BodySource::Compiled(expanded, passes) => (expanded, passes), + BodySource::Plain => (planned.input.clone(), Vec::new()), + }; + let pending: PendingBody = PendingBody { + start: planned.start, + input: planned.input, + expanded, + passes, + sites: planned.sites, + }; + let prior: Vec = prior_sites(&pending.sites); + let prepared: Option = py + .compiler_detach(|| prepare(&pending.expanded, &pending.input, &prior, dialect)) + .map_err(value_error)?; + let Some(prepared) = prepared else { + return Err(not_prepared(py, &file_path.extract::()?)?); + }; + let traits: BodyTraits = BodyTraits { + external_identifiers: Vec::new(), + ceremonial: false, + dynamic_star: false, + }; + let (body, dependency_relations) = lint_body((0, &traits), pending, prepared)?; + Ok(SingleBody { + body, + body_end: range.1, + dependency_relations, + }) +} + +/// The `NativeLintError` of a body native preparation declined. +fn not_prepared(py: Python<'_>, path: &Path) -> PyResult { + Ok(PyErr::from_value( + registry::error(py, ErrorId::NativeLint)?.call1((format!( + "{} could not be prepared for native SQL lint", + path.display() + ),))?, + )) +} + +fn plan_body( + text: &[char], + (mut start, end): (usize, usize), + neutralize: Option Neutralized>, + compiled: Option<&CompiledExpansion>, +) -> PlannedBody { + let body: String = text[start..end].iter().collect(); + let (mut input, sites): (String, Vec) = match neutralize { + Some(neutralize) => { + let neutralized: Neutralized = neutralize(&body); + (neutralized.text, neutralized.sites) + } + None => (body.clone(), Vec::new()), + }; + let source: BodySource = match compiled { + Some(compiled) if python_strip(&body) == compiled.authored_sql => { + let leading: usize = + body.chars().count() - body.trim_start_matches(python_isspace).chars().count(); + start += leading; + input.clone_from(&compiled.authored_sql); + BodySource::Compiled(compiled.expanded_sql.clone(), compiled.passes.clone()) + } + _ if INTERPOLATION_MARKERS + .iter() + .all(|marker| !input.contains(marker)) => + { + BodySource::Plain + } + _ => BodySource::Expand, + }; + PlannedBody { + start, + input, + sites, + source, + } +} + +/// The `ValueError` `PurePath.relative_to` raises for a path outside `project_dir`. +pub(crate) fn outside_project(path: &Path, project_dir: &Path) -> PyErr { + PyValueError::new_err(format!( + "{} is not in the subpath of {}", + python_str_repr(&path.to_string_lossy()), + python_str_repr(&project_dir.to_string_lossy()), + )) +} + +fn plan_error(project_dir: &Path, failure: PlanFailure) -> PyErr { + match failure { + PlanFailure::OutsideProject { path } => outside_project(&path, project_dir), + PlanFailure::Native(message) => value_error(message), + } +} + +/// Expand a file's bodies in order; a compile failure stops the file with its partial work. +fn expand_bodies( + py: Python<'_>, + context: &SqlExpansionInputs, + file: &LintFile, + (index, traits, planned): (usize, BodyTraits, Vec), +) -> PyResult { + let mut bodies: Vec = Vec::with_capacity(planned.len()); + let mut failure: Option = None; + let mut path: Option> = None; + for body in planned { + let (expanded, passes) = match body.source { + BodySource::Compiled(expanded, passes) => (expanded, passes), + BodySource::Plain => (body.input.clone(), Vec::new()), + BodySource::Expand => { + let file_path: &Bound<'_, PyAny> = match &path { + Some(path) => path, + None => path.insert(file.path.clone().into_pyobject(py)?.into_any()), + }; + match expand_with_inputs(context, &body.input, file_path) { + Ok(AuthoredExpansion { sql, passes }) => { + (sql, passes.iter().map(|rows| span_pass(rows)).collect()) + } + Err(error) => { + let error: PyErr = expansion_error(file_path, error); + if !error + .value(py) + .is_instance(®istry::error(py, ErrorId::ProjectCompile)?)? + { + return Err(error); + } + failure = Some(error); + break; + } + } + } + }; + bodies.push(PendingBody { + start: body.start, + input: body.input, + expanded, + passes, + sites: body.sites, + }); + } + Ok(ExpandedFile { + file: index, + traits, + bodies, + failure, + }) +} + +/// A compile-input error while expanding as `ProjectCompileError`; any other error unchanged. +fn expansion_error(file_path: &Bound<'_, PyAny>, error: PyErr) -> PyErr { + let py: Python<'_> = file_path.py(); + let wrapped = (|| -> PyResult> { + if !error + .value(py) + .is_instance(®istry::error(py, ErrorId::CompileInput)?)? + { + return Ok(None); + } + let message: String = format!( + "{} could not be expanded, so its SQL cannot be linted: {}", + file_path.str()?, + error.value(py).str()? + ); + Ok(Some(PyErr::from_value( + registry::error(py, ErrorId::ProjectCompile)?.call1((message,))?, + ))) + })(); + match wrapped { + Ok(Some(wrapped)) => { + wrapped.set_cause(py, Some(error)); + wrapped + } + Ok(None) => error, + Err(wrapping_error) => wrapping_error, + } +} + +/// Prepared bodies to lint, and skipped files by index with why they could not expand. +type PreparedBodies = (Vec, Vec<(usize, String)>); + +/// Prepare every expanded body natively, then publish bodies or failures in file order. +fn prepared_bodies( + py: Python<'_>, + request: &LintRequest<'_>, + files: &[LintFile], + expanded_files: Vec, +) -> PyResult { + let dialect: &str = &request.settings.sql.dialect; + let prepared: Vec, String>> = py + .compiler_detach(|| { + Ok(expanded_files + .iter() + .flat_map(|file| &file.bodies) + .map(|body| { + let prior: Vec = prior_sites(&body.sites); + catch_compiler_panic(|| prepare(&body.expanded, &body.input, &prior, dialect)) + }) + .collect()) + }) + .map_err(value_error)?; + let mut prepared: Vec> = prepared + .into_iter() + .collect::>() + .map_err(value_error)?; + let mut bodies: Vec = Vec::new(); + let mut unexpandable: Vec<(usize, String)> = Vec::new(); + let mut position: usize = 0; + for file in expanded_files { + let mut file_bodies: Vec = Vec::with_capacity(file.bodies.len()); + for body in file.bodies { + let Some(result) = prepared[position].take() else { + return Err(not_prepared(py, &files[file.file].path)?); + }; + position += 1; + let (mut lint_body, relations) = lint_body((file.file, &file.traits), body, result)?; + lint_body.relation_keys = relation_keys(request.settings, relations); + file_bodies.push(lint_body); + } + match file.failure { + None => bodies.extend(file_bodies), + Some(failure) if request.skip_unexpandable => { + let message: String = failure.value(py).str()?.extract()?; + let prefix: String = format!("{} ", files[file.file].path.display()); + unexpandable.push(( + file.file, + message.strip_prefix(&prefix).unwrap_or(&message).to_owned(), + )); + } + Some(failure) => return Err(failure), + } + } + Ok((bodies, unexpandable)) +} + +/// One prepared body as the native SQL engine lints it. +fn lint_body( + (file, traits): (usize, &BodyTraits), + body: PendingBody, + (lint_text, prepared_sites, externally_referenced_ctes): PreparedSql, +) -> PyResult<(LintSqlBody, Vec<(String, String)>)> { + let prepared_sites: Vec = prepared_sites + .into_iter() + .map( + |( + sentinel, + neutralized_start, + neutralized_end, + original_start, + original_end, + original_text, + )| { + InterpolationSite { + sentinel, + neutralized_start, + neutralized_end, + original_start, + original_end, + original_text, + } + }, + ) + .collect(); + let dependencies = dependency_sites(&prepared_sites).map_err(value_error)?; + let mut passes: Vec> = Vec::with_capacity(body.passes.len() + 2); + passes.push(sentinel_spans(&body.sites)); + passes.extend(body.passes); + passes.push(sentinel_spans(&prepared_sites)); + let body = LintSqlBody { + file, + body_start: offset(body.start), + lint_text, + passes, + external_identifiers: traits.external_identifiers.clone(), + dependency_identifiers: dependencies.identifiers, + externally_referenced_ctes, + allows_ceremonial_select: traits.ceremonial, + allows_dynamic_output_star: traits.dynamic_star, + allows_empty_fixture_star: traits.ceremonial, + relation_keys: Vec::new(), + }; + Ok((body, dependencies.relations)) +} + +/// The declared keys of each dependency relation that declares any, by its sentinel. +pub(crate) fn relation_keys( + settings: &LintSettings, + relations: Vec<(String, String)>, +) -> Vec<(String, Vec>)> { + let mut keyed: Vec<(String, Vec>)> = Vec::new(); + for (sentinel, name) in relations { + if let Some(keys) = settings.keys_of(&name) { + keyed.push((sentinel, keys.clone())); + } + } + keyed +} + +/// Where each pre-expansion site's sentinel starts in the neutralized body. +fn prior_sites(sites: &[InterpolationSite]) -> Vec { + sites.iter().map(|site| site.neutralized_start).collect() +} + +fn sentinel_spans(sites: &[InterpolationSite]) -> Vec { + sites + .iter() + .map(|site| ExpansionSpan { + source_start: offset(site.original_start), + source_end: offset(site.original_end), + output_start: offset(site.neutralized_start), + output_end: offset(site.neutralized_end), + }) + .collect() +} + +/// `rows` without suppressed violations, with suppression findings, sorted by path then place. +fn suppressed( + files: &[LintFile], + rows: Vec, + allow_suppressions: bool, +) -> PyResult> { + let keys: Vec = rows + .iter() + .map(|row| SuppressibleViolation { + file: row.file, + line: usize::try_from(row.line).unwrap_or(usize::MAX), + code: row.code.clone(), + native: row.engine == NATIVE_ENGINE, + }) + .collect(); + let texts: Vec<&str> = files.iter().map(|file| file.contents.as_str()).collect(); + let retained: Vec = catch_compiler_panic(|| { + apply_suppressions(&texts, &keys, allow_suppressions).map_err(|error| error.to_string()) + }) + .map_err(value_error)?; + let mut violations: Vec = retained + .into_iter() + .map(|item| match item { + RetainedViolation::Existing(index) => rows[index].clone(), + RetainedViolation::Added(finding) => LintViolationRow::suppression(finding), + }) + .collect(); + violations.sort_by(|left, right| { + ( + files[left.file].path.to_string_lossy(), + left.line, + left.column, + ) + .cmp(&( + files[right.file].path.to_string_lossy(), + right.line, + right.column, + )) + }); + Ok(violations) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs index d571ba353b..10597b7345 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs @@ -1,42 +1,58 @@ -//! Native lint expansion orchestration around the existing user-macro boundary. +//! One authored body expanded and prepared for native SQL lint, as a `LintBody`. -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; -use sqlbuild_rules::lint_text::main::dependency_sites::dependency_sites; -use sqlbuild_rules::lint_text::models::InterpolationSite; +use pyo3::types::{PyBool, PyDict, PyTuple}; +use sqlbuild_rules::lint_text::models::{ExpansionSpan, LintSqlBody}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::rules::lint_headers::{lint_body_ranges, lint_file_role}; -use crate::bindings::_helpers::rules::lint_stop::check_stop; +use crate::bindings::_helpers::rules::lint_pipeline::{ + BodyRole, SingleBody, SingleBodyRequest, prepare_single_body, +}; -/// Expand a single body and finish its native SQL preparation. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn prepare_lint_body<'py>( +/// `ExpansionSpan` objects of one pass. +fn span_objects<'py>( + class: &Bound<'py, PyAny>, + spans: &[ExpansionSpan], +) -> PyResult> { + let mut objects: Vec> = Vec::with_capacity(spans.len()); + for span in spans { + objects.push(class.call1(( + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ))?); + } + PyTuple::new(class.py(), objects) +} + +/// `ExpansionSpan` tuples for each pass. +fn pass_objects<'py>( py: Python<'py>, - options: Option>, -) -> PyResult> { - let pending = expand_lint_body(py, options)?; - let prepared = - registry::native(py, NativeAttr::PrepareLintSql)?.call1((pending.getattr("request")?,))?; - finish_body(&pending, &prepared) + passes: &[Vec], +) -> PyResult> { + let class: Bound<'py, PyAny> = registry::class(py, RecordId::ExpansionSpan)?; + let passes: Vec> = passes + .iter() + .map(|spans| span_objects(&class, spans)) + .collect::>()?; + PyTuple::new(py, passes) } -/// Expand authored SQL while retaining source offsets and pre-expansion substitutions. +/// Expand a single body and finish its native SQL preparation. #[pyfunction] #[pyo3(signature = (**options))] -pub(crate) fn expand_lint_body<'py>( +pub(crate) fn prepare_lint_body<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { - let args = KeywordArguments::read( + let arguments: KeywordArguments<'py> = KeywordArguments::read( py, - "expand_lint_body", + "prepare_lint_body", options, ( &[ @@ -47,451 +63,67 @@ pub(crate) fn expand_lint_body<'py>( "context", "dialect", ], - &[ - "external_identifiers", - "allows_ceremonial_select", - "allows_dynamic_output_star", - "compiled_expansion", - ], + &[], ), )?; - let (mut start, mut end): (usize, usize) = args.value("body_range")?.extract()?; - let body = args.value("contents")?.get_item(pyo3::types::PySlice::new( - py, - start as isize, - end as isize, - 1, - ))?; - let mut input = body.clone(); - let mut sites = PyTuple::empty(py).into_any(); - let role = args.value("role")?; - let kind = role.getattr("declaration_kind")?; - let in_project = role.getattr("in_project")?.is_truthy()?; - let neutralizer = if in_project && kind.eq("audit")? { - Some(NativeAttr::NeutralizeLintAuditParameters) - } else if role.getattr("in_hook_directory")?.is_truthy()? - || (in_project && kind.eq("sql_hook")?) - { - Some(NativeAttr::NeutralizeLintContextInterpolation) - } else { - None - }; - if let Some(neutralizer) = neutralizer { - let neutralized = registry::native(py, neutralizer)?.call1((&body,))?; - input = neutralized.get_item(0)?; - sites = site_objects(&neutralized.get_item(1)?)?; - } - let compiled = args.value("compiled_expansion")?; - let expanded; - let passes; - if !compiled.is_none() - && body - .call_method0("strip")? - .eq(compiled.getattr("authored_sql")?)? - { - start += body.len()? - body.call_method0("lstrip")?.len()?; - input = compiled.getattr("authored_sql")?; - end = start + input.len()?; - expanded = compiled.getattr("expanded_sql")?; - passes = compiled.getattr("passes")?; - } else if !input.extract::()?.contains('@') - && !input.extract::()?.contains("${") - { - expanded = input.clone(); - passes = PyTuple::empty(py).into_any(); - } else { - let value = call_with( - ®istry::native(py, NativeAttr::ExpandSqlWithSpans)?, - &[ - ("sql", input.clone()), - ("file_path", args.value("file_path")?), - ("context", args.value("context")?), - ], - ) - .map_err(|error| expansion_error(py, &args, error))?; - expanded = value.get_item(0)?; - passes = value.get_item(1)?; - } - let mut prior_sites: Vec> = Vec::new(); - for site in sites.try_iter()? { - prior_sites.push(site?.getattr("neutralized_start")?); - } - let request = keywords( - py, - &[ - ("expanded", expanded.clone()), - ("before_expansion", input.clone()), - ("prior_sites", PyList::new(py, prior_sites)?.into_any()), - ("dialect", args.value("dialect")?), - ], - )?; - let external = if args.passed("external_identifiers")? { - args.value("external_identifiers")? - } else { - PyTuple::empty(py).into_any() - }; + let role: Bound<'py, PyAny> = arguments.value("role")?; + let file_path: Bound<'py, PyAny> = arguments.value("file_path")?; + let contents: String = arguments.value("contents")?.extract()?; + let dialect: String = arguments.value("dialect")?.extract()?; + let kind: Option = role.getattr("declaration_kind")?.extract()?; + let prepared: SingleBody = prepare_single_body(&SingleBodyRequest { + context: &SqlExpansionInputs::from_record(&arguments.value("context")?)?, + file_path: &file_path, + contents: &contents, + range: arguments.value("body_range")?.extract()?, + role: BodyRole { + kind: kind.as_deref(), + in_project: role.getattr("in_project")?.is_truthy()?, + in_hook_directory: role.getattr("in_hook_directory")?.is_truthy()?, + }, + dialect: &dialect, + })?; + let body: &LintSqlBody = &prepared.body; + let flag = |value: bool| PyBool::new(py, value).to_owned().into_any(); call_with( - ®istry::class(py, RecordId::PendingLintBody)?, + ®istry::class(py, RecordId::LintBody)?, &[ - ("file_path", args.value("file_path")?), - ("body_start", start.into_pyobject(py)?.into_any()), - ("body_end", end.into_pyobject(py)?.into_any()), - ("expansion_input", input), - ("expanded", expanded), - ("expansion_passes", passes), - ("pre_expansion_sites", sites), - ("external_identifiers", external), + ("file_path", file_path.clone()), + ("body_start", body.body_start.into_pyobject(py)?.into_any()), + ("body_end", prepared.body_end.into_pyobject(py)?.into_any()), ( - "allows_ceremonial_select", - args.value_or_false("allows_ceremonial_select")?, + "lint_text", + body.lint_text.clone().into_pyobject(py)?.into_any(), ), + ("passes", pass_objects(py, &body.passes)?.into_any()), ( - "allows_dynamic_output_star", - args.value_or_false("allows_dynamic_output_star")?, - ), - ( - "allows_empty_fixture_star", - false.into_pyobject(py)?.to_owned().into_any(), + "external_identifiers", + PyTuple::new(py, &body.external_identifiers)?.into_any(), ), - ("request", request.into_any()), - ], - ) -} - -fn expansion_error(py: Python<'_>, args: &KeywordArguments<'_>, error: PyErr) -> PyErr { - let wrapped = (|| -> PyResult> { - if !error - .value(py) - .is_instance(®istry::error(py, ErrorId::CompileInput)?)? - { - return Ok(None); - } - let message = format!( - "{} could not be expanded, so its SQL cannot be linted: {}", - args.value("file_path")?.str()?, - error.value(py).str()? - ); - Ok(Some(PyErr::from_value( - registry::error(py, ErrorId::ProjectCompile)?.call1((message,))?, - ))) - })(); - match wrapped { - Ok(Some(wrapped)) => { - wrapped.set_cause(py, Some(error)); - wrapped - } - Ok(None) => error, - Err(wrapping_error) => wrapping_error, - } -} - -fn site_objects<'py>(sites: &Bound<'py, PyAny>) -> PyResult> { - let class = registry::class(sites.py(), RecordId::InterpolationSite)?; - let values: Vec> = sites - .try_iter()? - .map(|site| class.call1(site?.cast_into::()?)) - .collect::>()?; - Ok(PyTuple::new(sites.py(), values)?.into_any()) -} - -fn sentinel_spans<'py>(sites: &Bound<'py, PyAny>) -> PyResult> { - let py = sites.py(); - let mut spans: Vec> = Vec::new(); - for site in sites.try_iter()? { - let site = site?; - spans.push(call_with( - ®istry::class(py, RecordId::ExpansionSpan)?, - &[ - ("source_start", site.getattr("original_start")?), - ("source_end", site.getattr("original_end")?), - ("output_start", site.getattr("neutralized_start")?), - ("output_end", site.getattr("neutralized_end")?), - ], - )?); - } - Ok(PyTuple::new(py, spans)?.into_any()) -} - -fn finish_body<'py>( - pending: &Bound<'py, PyAny>, - prepared: &Bound<'py, PyAny>, -) -> PyResult> { - let py = pending.py(); - if prepared.is_none() { - return Err(PyErr::from_value( - registry::error(py, ErrorId::NativeLint)?.call1((format!( - "{} could not be prepared for native SQL lint", - pending.getattr("file_path")?.str()? - ),))?, - )); - } - let sites = site_objects(&prepared.get_item(1)?)?; - let native_sites: Vec = sites - .try_iter()? - .map(|site| { - let site = site?; - Ok(InterpolationSite { - sentinel: site.getattr("sentinel")?.extract()?, - original_text: site.getattr("original_text")?.extract()?, - original_start: site.getattr("original_start")?.extract()?, - original_end: site.getattr("original_end")?.extract()?, - neutralized_start: site.getattr("neutralized_start")?.extract()?, - neutralized_end: site.getattr("neutralized_end")?.extract()?, - }) - }) - .collect::>()?; - let dependencies = dependency_sites(&native_sites) - .map_err(crate::bindings::_helpers::boundary::panics::value_error)?; - let mut passes: Vec> = - vec![sentinel_spans(&pending.getattr("pre_expansion_sites")?)?]; - passes.extend( - pending - .getattr("expansion_passes")? - .try_iter()? - .collect::>>()?, - ); - passes.push(sentinel_spans(&sites)?); - let values = keywords( - py, - &[ - ("lint_text", prepared.get_item(0)?), - ("passes", PyTuple::new(py, passes)?.into_any()), ( "dependency_identifiers", - PyTuple::new(py, dependencies.identifiers)?.into_any(), + PyTuple::new(py, &body.dependency_identifiers)?.into_any(), ), ( "externally_referenced_ctes", - PyTuple::new(py, prepared.get_item(2)?.extract::>()?)?.into_any(), + PyTuple::new(py, &body.externally_referenced_ctes)?.into_any(), ), ( "dependency_relations", - PyTuple::new(py, dependencies.relations)?.into_any(), + PyTuple::new(py, &prepared.dependency_relations)?.into_any(), + ), + ( + "allows_ceremonial_select", + flag(body.allows_ceremonial_select), + ), + ( + "allows_dynamic_output_star", + flag(body.allows_dynamic_output_star), + ), + ( + "allows_empty_fixture_star", + flag(body.allows_empty_fixture_star), ), - ], - )?; - for name in [ - "file_path", - "body_start", - "body_end", - "external_identifiers", - "allows_ceremonial_select", - "allows_dynamic_output_star", - "allows_empty_fixture_star", - ] { - values.set_item(name, pending.getattr(name)?)?; - } - registry::class(py, RecordId::LintBody)?.call((), Some(&values)) -} - -fn prepare_batch<'py>( - py: Python<'py>, - pending: &[Bound<'py, PyAny>], -) -> PyResult>> { - if pending.is_empty() { - return Ok(Vec::new()); - } - let requests: Vec> = pending - .iter() - .map(|body| body.getattr("request")) - .collect::>()?; - let results = registry::native(py, NativeAttr::PrepareLintSqlBatch)? - .call1((PyList::new(py, requests)?,))?; - let mut prepared: Vec> = Vec::new(); - for (body, result) in pending.iter().zip(results.try_iter()?) { - let result = result?; - prepared.push(if result.get_item(0)?.is_truthy()? { - result.get_item(1)? - } else { - registry::native(py, NativeAttr::PrepareLintSql)?.call1((body.getattr("request")?,))? - }); - } - if prepared.len() != pending.len() || results.len()? != pending.len() { - return Err(pyo3::exceptions::PyValueError::new_err( - "native preparation batch length mismatch", - )); - } - Ok(prepared) -} - -/// Expand each file body in order, preserving the first expansion failure with its partial work. -#[pyfunction] -#[pyo3(signature = (**options))] -pub(crate) fn expand_file_bodies<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "expand_file_bodies", - options, - ( - &[ - "file_path", - "contents", - "headers", - "context", - "dialect", - "project_dir", - "relative_path", - "dynamic_output_paths", - "compiled_expansions", - ], - &[], - ), - )?; - check_stop(py)?; - let path = args.value("file_path")?; - let role = lint_file_role( - &path, - &args.value("project_dir")?, - Some(args.value("relative_path")?), - )?; - let dynamic = args.value("dynamic_output_paths")?; - let dynamic_star = dynamic.is_truthy()? && dynamic.contains(path.call_method0("resolve")?)?; - let expansions = args.value("compiled_expansions")?; - let compiled = if expansions.is_truthy()? { - expansions.call_method1("get", (&path,))? - } else { - py.None().into_bound(py) - }; - let external = call_with( - ®istry::native(py, NativeAttr::ExternalIdentifiersForHeaders)?, - &[ - ("contents", args.value("contents")?), - ("headers", args.value("headers")?), - ], - )?; - let mut ceremonial = false; - for header in args.value("headers")?.try_iter()? { - let kind = header?.getattr("kind")?; - if kind.eq("TEST")? || kind.eq("SCENARIO")? { - ceremonial = true; - break; - } - } - let mut bodies: Vec> = Vec::new(); - let mut failure = py.None().into_bound(py); - let expansion = (|| -> PyResult<()> { - let ranges = lint_body_ranges( - py, - Some(keywords( - py, - &[ - ("contents", args.value("contents")?), - ("headers", args.value("headers")?), - ("file_path", path.clone()), - ("project_dir", args.value("project_dir")?), - ("role", role.clone()), - ], - )?), - )?; - for range in ranges.try_iter()? { - let pending = expand_lint_body( - py, - Some(keywords( - py, - &[ - ("role", role.clone()), - ("file_path", path.clone()), - ("contents", args.value("contents")?), - ("body_range", range?), - ("context", args.value("context")?), - ("dialect", args.value("dialect")?), - ("external_identifiers", external.clone()), - ( - "allows_ceremonial_select", - ceremonial.into_pyobject(py)?.to_owned().into_any(), - ), - ( - "allows_dynamic_output_star", - dynamic_star.into_pyobject(py)?.to_owned().into_any(), - ), - ("compiled_expansion", compiled.clone()), - ], - )?), - )?; - let changes = keywords( - py, - &[( - "allows_empty_fixture_star", - ceremonial.into_pyobject(py)?.to_owned().into_any(), - )], - )?; - bodies.push( - py.import("dataclasses")? - .getattr("replace")? - .call((pending,), Some(&changes))?, - ); - } - Ok(()) - })(); - if let Err(error) = expansion { - if !error - .value(py) - .is_instance(®istry::error(py, ErrorId::ProjectCompile)?)? - { - return Err(error); - } - failure = error.value(py).clone().into_any(); - } - call_with( - ®istry::class(py, RecordId::ExpandedLintFile)?, - &[ - ("file_path", path), - ("bodies", PyTuple::new(py, bodies)?.into_any()), - ("failure", failure), - ], - ) -} - -/// Batch SQL preparation, then publish bodies or failures in the original file order. -#[pyfunction] -#[pyo3(signature = (*, expanded_files, dialect, skip_unexpandable))] -pub(crate) fn prepare_expanded_files<'py>( - expanded_files: &Bound<'py, PyAny>, - dialect: &str, - skip_unexpandable: bool, -) -> PyResult> { - let _ = dialect; - let py = expanded_files.py(); - check_stop(py)?; - let files: Vec> = expanded_files.try_iter()?.collect::>()?; - let mut pending: Vec> = Vec::new(); - for file in &files { - pending.extend( - file.getattr("bodies")? - .try_iter()? - .collect::>>()?, - ); - } - let prepared = prepare_batch(py, &pending)?; - let mut bodies: Vec> = Vec::new(); - let unexpandable = PyDict::new(py); - let mut position = 0; - for file in files { - let mut file_bodies: Vec> = Vec::new(); - for body in file.getattr("bodies")?.try_iter()? { - file_bodies.push(finish_body(&body?, &prepared[position])?); - position += 1; - } - let failure = file.getattr("failure")?; - if failure.is_none() { - bodies.extend(file_bodies); - } else if skip_unexpandable { - let path = file.getattr("file_path")?; - let message: String = failure.str()?.extract()?; - let prefix = format!("{} ", path.str()?); - unexpandable.set_item(path, message.strip_prefix(&prefix).unwrap_or(&message))?; - } else { - return Err(PyErr::from_value(failure)); - } - } - call_with( - ®istry::class(py, RecordId::PreparedLintFiles)?, - &[ - ("bodies", PyTuple::new(py, bodies)?.into_any()), - ("unexpandable", unexpandable.into_any()), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs index 3020f23ec0..5e8c656119 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs @@ -1,21 +1,17 @@ //! Native SQL lint of prepared lint bodies, as Python lint violations. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::main::lint_sql_bodies::lint_sql_bodies; use sqlbuild_rules::lint_text::models::{ AuthoredDiagnostic, ExpansionSpan, LintSqlBody, LintSqlOptions, }; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; -use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; use crate::bindings::types::CompilerDetach; -const NATIVE_ENGINE: &str = "native"; - fn spans(passes: &Bound<'_, PyAny>) -> PyResult>> { passes.try_iter()?.map(|pass| pass_spans(&pass?)).collect() } @@ -84,82 +80,6 @@ fn options(config: &Bound<'_, PyAny>) -> PyResult { }) } -/// The lint model classes native SQL diagnostics are built as. -struct ViolationClasses<'py> { - violation: Bound<'py, PyAny>, - edit: Bound<'py, PyAny>, - severity: Bound<'py, PyAny>, -} - -impl<'py> ViolationClasses<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - violation: registry::class(py, RecordId::LintViolation)?, - edit: registry::class(py, RecordId::LintEdit)?, - severity: registry::class(py, RecordId::LintSeverity)?, - }) - } - - fn violation( - &self, - file_path: &Bound<'py, PyAny>, - diagnostic: AuthoredDiagnostic, - ) -> PyResult> { - let py: Python<'py> = file_path.py(); - let none = || py.None().into_bound(py); - let code: Bound<'py, PyAny> = PyString::new(py, &diagnostic.code).into_any(); - let fix: Bound<'py, PyAny> = match diagnostic.fix { - Some(edit) => call_with( - &self.edit, - &[ - ("file_path", file_path.clone()), - ("code", code.clone()), - ("start", edit.start.into_pyobject(py)?.into_any()), - ("end", edit.end.into_pyobject(py)?.into_any()), - ( - "replacement", - PyString::new(py, &edit.replacement).into_any(), - ), - ], - )?, - None => none(), - }; - let (end_line, end_column) = match diagnostic.end { - Some((line, column)) => ( - line.into_pyobject(py)?.into_any(), - column.into_pyobject(py)?.into_any(), - ), - None => (none(), none()), - }; - let severity: &str = if diagnostic.fault { "FAULT" } else { "WARNING" }; - call_with( - &self.violation, - &[ - ("file_path", file_path.clone()), - ("line", diagnostic.line.into_pyobject(py)?.into_any()), - ("column", diagnostic.column.into_pyobject(py)?.into_any()), - ("code", code), - ("message", PyString::new(py, &diagnostic.message).into_any()), - ("severity", self.severity.getattr(severity)?), - ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), - ("end_line", end_line), - ("end_column", end_column), - ( - "remediation", - PyString::new(py, &diagnostic.remediation).into_any(), - ), - ("fix", fix), - ( - "fix_unavailable_reason", - diagnostic - .fix_unavailable_reason - .map_or_else(none, |reason| PyString::new(py, &reason).into_any()), - ), - ], - ) - } -} - /// Lint expanded bodies natively and map every diagnostic onto authored source. #[pyfunction] fn lint_native_sql_bodies<'py>( @@ -191,7 +111,10 @@ fn lint_native_sql_bodies<'py>( let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; diagnostics .into_iter() - .map(|diagnostic| classes.violation(&paths[diagnostic.file], diagnostic)) + .map(|diagnostic| { + let row: LintViolationRow = LintViolationRow::diagnostic(diagnostic); + classes.violation(&paths[row.file], &row) + }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs deleted file mode 100644 index 3f0801a4dd..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_stop.rs +++ /dev/null @@ -1,85 +0,0 @@ -//! The stop signal that lets a speculative lint run be abandoned between files. - -use std::sync::{Mutex, PoisonError}; - -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::sync::PyOnceLock; -use pyo3::types::PyTuple; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; - -use crate::bindings::_helpers::compile_driver::python_calls::context_variable; -use crate::bindings::_helpers::records::ids::ErrorId; -use crate::bindings::_helpers::records::registry; - -const STOP_VARIABLE: &str = "active_lint_stop"; - -static ACTIVE_STOP: PyOnceLock> = PyOnceLock::new(); - -/// The context variable holding the active lint run's stop event, if any. -fn active_stop_variable(py: Python<'_>) -> PyResult> { - context_variable(py, &ACTIVE_STOP, STOP_VARIABLE) -} - -/// Abandon a speculative lint run on this thread once its owner no longer needs it. -pub(crate) fn check_stop(py: Python<'_>) -> PyResult<()> { - let stop: Bound<'_, PyAny> = active_stop_variable(py)?.call_method0("get")?; - if !stop.is_none() && stop.call_method0("is_set")?.is_truthy()? { - return Err(PyErr::from_value( - registry::error(py, ErrorId::LintStopped)?.call0()?, - )); - } - Ok(()) -} - -/// Makes `stop` the active lint stop event while entered. -#[pyclass(frozen, module = "sqlbuild._native")] -struct LintStopScope { - stop: Py, - token: Mutex>>, -} - -#[pymethods] -impl LintStopScope { - fn __enter__(&self, py: Python<'_>) -> PyResult<()> { - let token: Bound<'_, PyAny> = - active_stop_variable(py)?.call_method1("set", (self.stop.bind(py),))?; - *self.token.lock().unwrap_or_else(PoisonError::into_inner) = Some(token.unbind()); - Ok(()) - } - - #[pyo3(signature = (*_exception))] - fn __exit__(&self, py: Python<'_>, _exception: &Bound<'_, PyTuple>) -> PyResult { - let token: Option> = self - .token - .lock() - .unwrap_or_else(PoisonError::into_inner) - .take(); - if let Some(token) = token { - active_stop_variable(py)?.call_method1("reset", (token,))?; - } - Ok(false) - } -} - -/// A scope that makes `stop` the active lint stop event while entered. -#[pyfunction] -fn lint_stop_scope(stop: &Bound<'_, PyAny>) -> LintStopScope { - LintStopScope { - stop: stop.clone().unbind(), - token: Mutex::new(None), - } -} - -/// The active lint run's stop event, or `None` outside a stop scope. -#[pyfunction] -fn active_lint_stop(py: Python<'_>) -> PyResult> { - active_stop_variable(py)?.call_method0("get") -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(lint_stop_scope, module)?)?; - module.add_function(wrap_pyfunction!(active_lint_stop, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index 6ae4847acc..9de19afb41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -6,7 +6,6 @@ use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; use sqlbuild_rules::lint_text::errors::LintTextError; use sqlbuild_rules::lint_text::main::apply_suppressions::apply_suppressions; use sqlbuild_rules::lint_text::main::format_sql_bodies::format_sql_bodies; @@ -19,16 +18,18 @@ use sqlbuild_rules::lint_text::main::restore_interpolation::restore_interpolatio use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use sqlbuild_rules::lint_text::models::{ FormatFile, FormattedBodies, InterpolationSite, Neutralized, RetainedViolation, - SuppressibleViolation, SuppressionFinding, + SuppressibleViolation, }; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::lint_pipeline::{ + LintViolationRow, NATIVE_ENGINE, ViolationClasses, +}; use crate::bindings::types::CompilerDetach; -const NATIVE_ENGINE: &str = "native"; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; @@ -137,7 +138,7 @@ fn named_declaration_role_kind<'py>( /// Remove violations matched by suppressions and add the suppression findings, as lint models. #[pyfunction] -fn apply_lint_suppressions<'py>( +pub(crate) fn apply_lint_suppressions<'py>( py: Python<'py>, violations: Vec>, contents_by_path: Bound<'py, PyDict>, @@ -159,12 +160,15 @@ fn apply_lint_suppressions<'py>( let retained: Vec = compiler_guard(|| { apply_suppressions(&texts, &keys, allow_suppressions).map_err(value_error) })?; - let classes: FindingClasses<'py> = FindingClasses::new(py)?; + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; retained .into_iter() .map(|item| match item { RetainedViolation::Existing(index) => Ok(violations[index].clone()), - RetainedViolation::Added(finding) => classes.violation(&paths[finding.file], finding), + RetainedViolation::Added(finding) => { + let row: LintViolationRow = LintViolationRow::suppression(finding); + classes.violation(&paths[row.file], &row) + } }) .collect() } @@ -186,70 +190,6 @@ fn suppressible( }) } -/// The lint model classes suppression findings are built as. -struct FindingClasses<'py> { - violation: Bound<'py, PyAny>, - edit: Bound<'py, PyAny>, - severity: Bound<'py, PyAny>, -} - -impl<'py> FindingClasses<'py> { - fn new(py: Python<'py>) -> PyResult { - Ok(Self { - violation: registry::class(py, RecordId::LintViolation)?, - edit: registry::class(py, RecordId::LintEdit)?, - severity: registry::class(py, RecordId::LintSeverity)?, - }) - } - - fn violation( - &self, - file_path: &Bound<'py, PyAny>, - finding: SuppressionFinding, - ) -> PyResult> { - let py: Python<'py> = file_path.py(); - let code: Bound<'py, PyAny> = PyString::new(py, SUPPRESSION_CODE).into_any(); - let fix: Bound<'py, PyAny> = match finding.fix { - Some((start, end)) => call_with( - &self.edit, - &[ - ("file_path", file_path.clone()), - ("code", code.clone()), - ("start", start.into_pyobject(py)?.into_any()), - ("end", end.into_pyobject(py)?.into_any()), - ("replacement", PyString::new(py, "").into_any()), - ], - )?, - None => py.None().into_bound(py), - }; - let severity: &str = if finding.fault { "FAULT" } else { "WARNING" }; - call_with( - &self.violation, - &[ - ("file_path", file_path.clone()), - ("line", finding.line.into_pyobject(py)?.into_any()), - ("column", 1_usize.into_pyobject(py)?.into_any()), - ("code", code), - ("message", PyString::new(py, &finding.message).into_any()), - ("severity", self.severity.getattr(severity)?), - ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), - ( - "remediation", - PyString::new(py, finding.remediation).into_any(), - ), - ("fix", fix), - ( - "fix_unavailable_reason", - finding.fix_unavailable_reason.map_or_else( - || py.None().into_bound(py), - |reason| PyString::new(py, reason).into_any(), - ), - ), - ], - ) - } -} - /// The parts of `path` below `project_dir`, or `None` outside the project. fn relative_parts(path: &std::path::Path, project_dir: &std::path::Path) -> Option> { match path.strip_prefix(project_dir) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index c2a3ac6ee2..9232d48281 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -19,9 +19,10 @@ pub(crate) mod guidance; pub(crate) mod lint; pub(crate) mod lint_expansion; pub(crate) mod lint_headers; +pub(crate) mod lint_inputs; +pub(crate) mod lint_pipeline; pub(crate) mod lint_preparation; pub(crate) mod lint_sql_bodies; -pub(crate) mod lint_stop; pub(crate) mod lint_text; pub(crate) mod native_header_parser; pub(crate) mod project_tree_queries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs index fc4c16a1f9..118e082fc6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs @@ -1,11 +1,10 @@ //! DSL header values parsed natively for header rules, compared as Python compares their objects. -use std::cell::RefCell; - -use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use sqlbuild_rules::dsl_headers::models::{ParseFailure, ParseOutcome}; +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; +use sqlbuild_rules::dsl_headers::main::inner_header_text::inner_header_text; +use sqlbuild_rules::dsl_headers::models::{HeaderSpan, ParseFailure, ParseOutcome}; use sqlbuild_rules::dsl_headers::types::HeaderParser; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; use sqlbuild_sqltext::compiler::main::model_header_parsing::parse_batch; @@ -15,55 +14,43 @@ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::sqltext::model_headers::header_failure; -use crate::bindings::types::CompilerDetach; /// The path header rules report header syntax errors at. const SENTINEL_PATH: &str = ""; const DESCRIPTION_KEY: &str = "description"; +const FUNCTION_KIND: &str = "FUNCTION"; +const ARGUMENTS_KEY: &str = "arguments"; const NEGATIVE_ZERO: &str = "-0"; const ZERO: &str = "0"; -/// Header values parsed natively; a rejected parse keeps its message for raising. -pub(crate) struct NativeHeaderParser<'py> { - py: Python<'py>, - rejected: RefCell>, -} - -impl<'py> NativeHeaderParser<'py> { - pub(crate) fn new(py: Python<'py>) -> Self { - Self { - py, - rejected: RefCell::new(None), - } - } +/// Header values parsed natively, compared as their Python projections compare. +#[derive(Default)] +pub(crate) struct NativeHeaderParser; - /// The `ModelSqlParseError` of the last rejected parse. - pub(crate) fn rejected_error(&self) -> PyErr { - let Some(failure) = self.rejected.take() else { - return PyValueError::new_err("no header parse was rejected"); - }; - let raised: PyResult = (|| { - let keywords: Bound<'py, PyDict> = PyDict::new(self.py); - keywords.set_item("help", failure.help)?; - let error: Bound<'py, PyAny> = registry::error(self.py, ErrorId::ModelSqlParse)? - .call((failure.message,), Some(&keywords))?; - Ok(PyErr::from_value(error)) - })(); - raised.unwrap_or_else(|error| error) - } +/// The `ModelSqlParseError` a rejected header parse raises. +pub(crate) fn rejection(py: Python<'_>, failure: ParseFailure) -> PyErr { + let raised: PyResult = (|| { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("help", failure.help)?; + let error: Bound<'_, PyAny> = registry::error(py, ErrorId::ModelSqlParse)? + .call((failure.message,), Some(&keywords))?; + Ok(PyErr::from_value(error)) + })(); + raised.unwrap_or_else(|error| error) } -impl HeaderParser for NativeHeaderParser<'_> { +impl HeaderParser for NativeHeaderParser { type Values = AuthoredValue; - type Error = PyErr; + /// A native failure's message. + type Error = String; - fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { + fn parse(&self, kind: &str, inner: &str) -> ParseOutcome { let headers: Vec = vec![inner.to_owned()]; - let parsed = match self.py.compiler_detach(|| parse_batch(&headers)) { + let parsed = match catch_compiler_panic(|| parse_batch(&headers)) { Ok(parsed) => parsed, Err(message) => { return ParseOutcome::Raised( - value_error(message.clone()), + message.clone(), ParseFailure { message, help: None, @@ -83,14 +70,10 @@ impl HeaderParser for NativeHeaderParser<'_> { ), help, }; - self.rejected.replace(Some(ParseFailure { - message: failure.message.clone(), - help: failure.help.clone(), - })); ParseOutcome::Rejected(failure) } - fn description(&self, values: &AuthoredValue) -> PyResult> { + fn description(&self, values: &AuthoredValue) -> Result, String> { Ok(match map_value(values, DESCRIPTION_KEY) { Some(AuthoredValue::String(text)) => Some(text.clone()), Some(AuthoredValue::BareWord(word)) if header_word_value(word) == HeaderWord::Text => { @@ -100,11 +83,60 @@ impl HeaderParser for NativeHeaderParser<'_> { }) } - fn same(&self, left: &AuthoredValue, right: &AuthoredValue) -> PyResult { + fn same(&self, left: &AuthoredValue, right: &AuthoredValue) -> Result { Ok(python_equal(left, right)) } } +/// Why a header could not be read: a header syntax error, or a native failure's message. +pub(crate) enum HeaderFailure { + Rejected(ParseFailure), + Native(String), +} + +impl HeaderFailure { + pub(crate) fn into_pyerr(self, py: Python<'_>) -> PyErr { + match self { + Self::Rejected(failure) => rejection(py, failure), + Self::Native(message) => value_error(message), + } + } +} + +/// The SQL function argument names the first FUNCTION header declares, lowercased and sorted. +pub(crate) fn external_identifiers( + text: &[char], + headers: &[HeaderSpan], + parser: &NativeHeaderParser, +) -> Result, HeaderFailure> { + for header in headers { + if header.kind != FUNCTION_KIND { + continue; + } + let inner: String = inner_header_text(&text[header.start..header.end]) + .iter() + .collect(); + let values: AuthoredValue = match parser.parse(&header.kind, &inner) { + ParseOutcome::Parsed(values) => values, + ParseOutcome::Rejected(failure) => return Err(HeaderFailure::Rejected(failure)), + ParseOutcome::Raised(message, _) => return Err(HeaderFailure::Native(message)), + }; + let Some(arguments) = map_value(&values, ARGUMENTS_KEY) else { + continue; + }; + if !matches!(arguments, AuthoredValue::Map(_)) { + continue; + } + let mut names: Vec = map_keys(arguments) + .into_iter() + .map(str::to_lowercase) + .collect(); + names.sort_unstable(); + return Ok(names); + } + Ok(Vec::new()) +} + /// The value a header map projects for `key`: its last entry, as a Python dict keeps it. pub(crate) fn map_value<'a>(values: &'a AuthoredValue, key: &str) -> Option<&'a AuthoredValue> { match values { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 802295076a..52ec1a67af 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -6,9 +6,9 @@ use pyo3::{types::PyModuleMethods, wrap_pyfunction}; use crate::bindings::_helpers::rules::{ builtin_catalogue, catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, - early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_stop, - lint_text, results, rule_api, rule_cache, rule_plan, rule_run, rules_config, run_lint, - selection, sql_rules, + early_sql_lint, evaluation, guidance, lint, lint_expansion, lint_sql_bodies, lint_text, + results, rule_api, rule_cache, rule_plan, rule_run, rules_config, run_lint, selection, + sql_rules, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -18,26 +18,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::rules::lint_headers::lint_file_role, module )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_headers::lint_body_ranges, - module - )?)?; module.add_function(wrap_pyfunction!( crate::bindings::_helpers::rules::lint_preparation::prepare_lint_body, module )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_preparation::expand_lint_body, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_preparation::expand_file_bodies, - module - )?)?; - module.add_function(wrap_pyfunction!( - crate::bindings::_helpers::rules::lint_preparation::prepare_expanded_files, - module - )?)?; module.add_function(wrap_pyfunction!( crate::bindings::_helpers::rules::fix_session::plan_rule_fixes, module @@ -62,7 +46,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { catalogue::register(module)?; rules_config::register(module)?; run_lint::register(module)?; - lint_stop::register(module)?; selection::register(module)?; results::register(module)?; guidance::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index dce80ccc23..1bacb85af8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -8,6 +8,8 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation_keys as derive_keys; use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; +use crate::bindings::_helpers::rules::lint_inputs::model_key_row; + const MATERIALIZED_CONFIG: &str = "materialized"; const INCREMENTAL_MODE_CONFIG: &str = "incremental_mode"; const INCREMENTAL_STRATEGY_CONFIG: &str = "incremental_strategy"; @@ -145,19 +147,15 @@ fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult>)>; + /// Keys that prove one row per key, by `kind:name` identity, from model rows, seeds and sources. -#[pyfunction] -#[pyo3(signature = (*, model_rows, seeds, sources))] -fn declared_relation_keys<'py>( +pub(crate) fn relation_keys<'py>( py: Python<'py>, - model_rows: &Bound<'py, PyAny>, - seeds: &Bound<'py, PyAny>, - sources: &Bound<'py, PyAny>, -) -> PyResult> { - let mut rows: Vec = Vec::new(); - for row in model_rows.try_iter()? { - rows.push(row?.extract()?); - } + mut rows: Vec, + (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult { for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { for entry in entries.try_iter()? { let entry: Bound<'py, PyAny> = entry?; @@ -172,6 +170,11 @@ fn declared_relation_keys<'py>( } let relations: Vec = rows.into_iter().map(declared_relation).collect(); let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); + Ok(keys.into_iter().collect()) +} + +/// `keys` as a `{identity: ((column, ...), ...)}` dict. +fn keys_dict<'py>(py: Python<'py>, keys: RelationKeys) -> PyResult> { let found: Bound<'py, PyDict> = PyDict::new(py); for (identity, unique_keys) in keys { let mut tuples: Vec> = Vec::with_capacity(unique_keys.len()); @@ -183,30 +186,49 @@ fn declared_relation_keys<'py>( Ok(found) } -/// Declared keys of a compiled project's models, seeds and sources. +/// Keys that prove one row per key, by `kind:name` identity, from model rows, seeds and sources. #[pyfunction] -fn compiled_relation_keys<'py>(project: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project.py(); - let mut model_rows: Vec> = Vec::new(); +#[pyo3(signature = (*, model_rows, seeds, sources))] +fn declared_relation_keys<'py>( + py: Python<'py>, + model_rows: Vec, + seeds: &Bound<'py, PyAny>, + sources: &Bound<'py, PyAny>, +) -> PyResult> { + keys_dict(py, relation_keys(py, model_rows, (seeds, sources))?) +} + +/// Declared keys of a compiled project's models, seeds and sources. +pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = project.py(); + let mut rows: Vec = Vec::new(); for model in project.getattr("models")?.try_iter()? { - model_rows.push(model?.getattr("relation_key_row")?); + rows.push(model_key_row(&model?)?); } - let mut seeds: Vec> = Vec::new(); + let mut seeds: Vec> = Vec::new(); for seed in project.getattr("seeds")?.try_iter()? { seeds.push(seed?.getattr("schema_entry")?); } - let mut sources: Vec> = Vec::new(); + let mut sources: Vec> = Vec::new(); for source in project.getattr("sources")?.try_iter()? { sources.push(source?.getattr("source_entry")?); } - declared_relation_keys( + relation_keys( py, - PyTuple::new(py, model_rows)?.as_any(), - PyTuple::new(py, seeds)?.as_any(), - PyTuple::new(py, sources)?.as_any(), + rows, + ( + PyTuple::new(py, seeds)?.as_any(), + PyTuple::new(py, sources)?.as_any(), + ), ) } +/// Declared keys of a compiled project's models, seeds and sources. +#[pyfunction] +fn compiled_relation_keys<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + keys_dict(project.py(), project_relation_keys(project)?) +} + fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits) = row; DeclaredRelation { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index 6600388df3..cfcc0e6bc1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -1,7 +1,7 @@ //! SQL rule result cache keys, stored-entry lookup and the cache file. use std::collections::BTreeMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{ @@ -22,33 +22,33 @@ use sqlbuild_rules::rule_cache::main::parse_rule_cache::parse_rule_cache; use sqlbuild_rules::rule_cache::main::rule_cache_text::rule_cache_text; use sqlbuild_rules::rule_cache::models::{CacheLookup, CachedEntry, LintKeyInput, ModelKeyInput}; use sqlbuild_rules::rule_cache::types::CachedEntries; +use sqlbuild_rules::rule_results::models::RuleFinding; + +use crate::bindings::_helpers::rules::lint_pipeline::LintSettings; /// Stored entries as `{path: (key, findings JSON)}`. -type StoredEntries = BTreeMap; +pub(crate) type StoredEntries = BTreeMap; fn invalid(error: serde_json::Error) -> PyErr { PyValueError::new_err(error.to_string()) } +/// The lint settings part of every SQL rule cache key. +pub(crate) fn settings_lint_key(settings: &LintSettings) -> String { + lint_key(&LintKeyInput { + dialect: settings.sql.dialect.clone(), + max_literal_length: settings.sql.max_literal_length, + max_ranking_order_by: settings.sql.max_ranking_order_by, + relation_keys: settings.relation_keys.clone(), + allow_suppressions: settings.allow_suppressions, + }) +} + /// The lint settings part of every SQL rule cache key. #[pyfunction] #[pyo3(signature = (*, config))] fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { - let mut relation_keys: Vec<(String, Vec>)> = Vec::new(); - for item in config - .getattr("relation_keys")? - .call_method0("items")? - .try_iter()? - { - relation_keys.push(item?.extract()?); - } - Ok(lint_key(&LintKeyInput { - dialect: config.getattr("dialect")?.extract()?, - max_literal_length: config.getattr("max_literal_length")?.extract()?, - max_ranking_order_by: config.getattr("max_ranking_order_by")?.extract()?, - relation_keys, - allow_suppressions: config.getattr("allow_suppressions")?.extract()?, - })) + Ok(settings_lint_key(&LintSettings::from_record(config)?)) } /// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. @@ -58,6 +58,15 @@ fn sql_model_rule_keys( lint_key: &str, codes: Vec, models: Vec<(String, String, String, Option)>, +) -> BTreeMap { + model_rule_keys(lint_key, &codes, &models) +} + +/// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. +pub(crate) fn model_rule_keys( + lint_key: &str, + codes: &[String], + models: &[(String, String, String, Option)], ) -> BTreeMap { models .iter() @@ -67,7 +76,7 @@ fn sql_model_rule_keys( query_sql, contract: contract.as_deref(), }; - (path.clone(), model_rule_key(lint_key, &codes, &model)) + (path.clone(), model_rule_key(lint_key, codes, &model)) }) .collect() } @@ -94,7 +103,12 @@ fn sql_file_rule_key(request: Option<&Bound<'_, PyDict>>) -> PyResult PyResult { - let Ok(text) = std::fs::read_to_string(&path) else { + stored_entries(&path) +} + +/// The usable stored entries of the cache file at `path`; unreadable files read as empty. +pub(crate) fn stored_entries(path: &Path) -> PyResult { + let Ok(text) = std::fs::read_to_string(path) else { return Ok(StoredEntries::new()); }; parse_rule_cache(&text) @@ -120,6 +134,69 @@ fn cached_entries(entries: &StoredEntries) -> PyResult { .collect() } +/// The stored findings of each path whose key matches, as rows; undecodable stored findings miss. +pub(crate) fn cached_rule_findings( + entries: &StoredEntries, + keys: &[(String, String)], +) -> PyResult>> { + let lookup: CacheLookup = lookup_rule_cache(&cached_entries(entries)?, keys); + let mut hits: BTreeMap> = BTreeMap::new(); + for (path, findings) in lookup.hits { + let decoded: Option> = findings.iter().map(payload_finding).collect(); + if let Some(decoded) = decoded { + hits.insert(path, decoded); + } + } + Ok(hits) +} + +/// One finding of a stored payload, or `None` when the payload is malformed. +fn payload_finding(value: &Value) -> Option { + let payload = value.as_object()?; + let text = |key: &str| payload.get(key).and_then(Value::as_str).map(str::to_owned); + let number = |key: &str| { + payload + .get(key) + .filter(|value| value.is_u64()) + .and_then(Value::as_i64) + }; + let flag = |key: &str| payload.get(key).and_then(Value::as_bool) == Some(true); + Some(RuleFinding { + code: text("code")?, + path: text("path")?, + line: number("line")?, + column: number("column")?, + message: text("message")?, + remediation: text("remediation")?, + unevaluated: flag("unevaluated"), + affected_rules: Vec::new(), + fixable: flag("fixable"), + }) +} + +/// The stored payload of one finding row. +pub(crate) fn finding_json(finding: &RuleFinding) -> Value { + json!({ + "unevaluated": finding.unevaluated, + "code": finding.code, + "path": finding.path, + "line": finding.line, + "column": finding.column, + "message": finding.message, + "remediation": finding.remediation, + "fixable": finding.fixable, + }) +} + +/// The stored JSON of one path's finding rows. +pub(crate) fn encoded_findings(findings: &[&RuleFinding]) -> PyResult { + let payloads: Vec = findings + .iter() + .map(|finding| finding_json(finding)) + .collect(); + serde_json::to_string(&payloads).map_err(invalid) +} + /// The stored findings of each path whose key matches; undecodable stored findings miss. #[pyfunction] #[pyo3(signature = (*, entries, keys))] @@ -179,13 +256,18 @@ fn write_sql_rule_cache(path: PathBuf, entries: &Bound<'_, PyDict>) -> PyResult< for (key, value) in entries.iter() { stored.insert(key.extract()?, value.extract()?); } - let text: String = rule_cache_text(&cached_entries(&stored)?); + write_rule_cache(&path, &stored) +} + +/// Replace the cache file at `path` atomically with `entries`. +pub(crate) fn write_rule_cache(path: &Path, stored: &StoredEntries) -> PyResult<()> { + let text: String = rule_cache_text(&cached_entries(stored)?); if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } let temporary: PathBuf = path.with_extension(format!("tmp-{}", std::process::id())); std::fs::write(&temporary, text)?; - std::fs::rename(&temporary, &path)?; + std::fs::rename(&temporary, path)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index 5146f908a7..37f2cbce91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -11,13 +11,11 @@ use pyo3::exceptions::{PyKeyboardInterrupt, PyRuntimeError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::types::{PyDict, PyFrozenSet, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::cache::native_store::current_code_identity; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, within, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -59,23 +57,7 @@ const EVALUATE_RULES_OPTIONS: (&[&str], &[&str]) = ( "project_dir", "dialect", ], - &[ - "selected_keys", - "prepared_sql", - "expansion_reuse", - "no_sql_analysis", - ], -); -const PREPARE_OPTIONS: (&[&str], &[&str]) = ( - &[ - "project_dir", - "config", - "discovered_inputs", - "static_declaration_scope", - "compiled_expansions", - "stop", - ], - &[], + &["selected_keys", "early_lint", "no_sql_analysis"], ); /// A running custom-rule evaluation, whose result is a `CustomRulesOutcome`. @@ -312,8 +294,7 @@ pub(crate) fn evaluate_rules<'py>( ("graph", graph), ("discovered_inputs", arguments.value("discovered_inputs")?), ("dialect", dialect), - ("prepared_sql", arguments.value("prepared_sql")?), - ("expansion_reuse", arguments.value("expansion_reuse")?), + ("early_lint", arguments.value("early_lint")?), ], ); cancel_on_interrupt(py, &sql_result, &run.cancel); @@ -330,97 +311,6 @@ pub(crate) fn evaluate_rules<'py>( ) } -/// Start early SQL lint on `executor` when the project selects SQL rules without a cache. -#[pyfunction] -#[pyo3(signature = (*, inputs, executor, dialect, stop))] -pub(crate) fn prepare_sql_rules<'py>( - inputs: &Bound<'py, PyAny>, - executor: &Bound<'py, PyAny>, - dialect: &Bound<'py, PyAny>, - stop: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = inputs.py(); - let preparation: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::SqlLintPreparation)?, - &[("inputs", inputs.clone()), ("dialect", dialect.clone())], - )?; - if preparation.is_none() { - return Ok(preparation); - } - let (project_dir, codes, config): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>) = - preparation.extract()?; - let expansions: Bound<'py, PyAny> = - native(py, NativeAttr::EarlyLintExpansions)?.call1((inputs.getattr("model_inputs")?,))?; - let work: Bound<'py, PyDict> = PyDict::new(py); - work.set_item("project_dir", &project_dir)?; - work.set_item("config", config)?; - work.set_item("discovered_inputs", inputs.getattr("discovered_inputs")?)?; - work.set_item( - "static_declaration_scope", - inputs.getattr("declaration_scope")?, - )?; - work.set_item("compiled_expansions", expansions)?; - work.set_item("stop", stop)?; - let future: Bound<'py, PyAny> = executor.call_method( - "submit", - (wrap_pyfunction!(prepare_early_sql_lint, py)?,), - Some(&work), - )?; - call_with( - ®istry::class(py, RecordId::PreparedSqlLint)?, - &[ - ("codes", codes), - ("project_dir", project_dir), - ("future", future), - ], - ) -} - -/// Expand and lint SQL early, stopping when `stop` is set; runs on the early-lint executor. -#[pyfunction] -#[pyo3(signature = (**options))] -fn prepare_early_sql_lint<'py>( - py: Python<'py>, - options: Option>, -) -> PyResult> { - let arguments: KeywordArguments<'py> = - KeywordArguments::read(py, "prepare_early_sql_lint", options, PREPARE_OPTIONS)?; - let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; - let discovered_inputs: Bound<'py, PyAny> = arguments.value("discovered_inputs")?; - let scope: Bound<'py, PyAny> = - native(py, NativeAttr::LintStopScopeFunction)?.call1((arguments.value("stop")?,))?; - within(&scope, |_| { - let context: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::BuildLintExpansionContext)?, - &[ - ("project_dir", project_dir.clone()), - ("discovered_inputs", discovered_inputs.clone()), - ( - "static_declaration_scope", - arguments.value("static_declaration_scope")?, - ), - ], - )?; - let result: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::RunLint)?, - &[ - ("project_dir", project_dir.clone()), - ("config", arguments.value("config")?), - ("discovered_inputs", discovered_inputs.clone()), - ( - "compiled_expansions", - arguments.value("compiled_expansions")?, - ), - ("expansion_context", context.clone()), - ], - )?; - call_with( - ®istry::class(py, RecordId::PreparedSqlLintResult)?, - &[("result", result), ("context", context)], - ) - }) -} - /// Evaluate built-in rules natively while custom rules run beside them. fn evaluate_native<'py>( run: &NativeRun<'py>, @@ -690,7 +580,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(evaluate_project_rules, module)?)?; module.add_function(wrap_pyfunction!(evaluate_native_rules, module)?)?; module.add_function(wrap_pyfunction!(evaluate_custom_rules, module)?)?; - module.add_function(wrap_pyfunction!(evaluate_rules, module)?)?; - module.add_function(wrap_pyfunction!(prepare_sql_rules, module)?)?; - module.add_function(wrap_pyfunction!(prepare_early_sql_lint, module)?) + module.add_function(wrap_pyfunction!(evaluate_rules, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index 87c7f7e583..3b8c057520 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -1,6 +1,6 @@ //! Rules configuration loaded natively and returned as the Python configuration models. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, @@ -241,20 +241,50 @@ fn rules_error(py: Python<'_>, class: ErrorId, message: String, help: Option, project_dir: PathBuf) -> PyResult> { + let config: RulesConfig = match rules_config_of(&project_dir)? { + Ok(config) => config, + Err(invalid) => return Err(invalid.into_pyerr(py)), + }; + Ok(config_object(py, &config)?.unbind()) +} + +/// Why a project's rules configuration is invalid. +pub(crate) enum InvalidRulesConfig { + /// The configuration could not be read: a `RulesError`. + Unreadable(String), + /// It allows exceptions it may not: a `RulesConfigError` with help. + Forbidden { message: String, help: String }, +} + +impl InvalidRulesConfig { + fn into_pyerr(self, py: Python<'_>) -> PyErr { + match self { + Self::Unreadable(message) => rules_error(py, ErrorId::Rules, message, None), + Self::Forbidden { message, help } => { + rules_error(py, ErrorId::RulesConfig, message, Some(help)) + } + } + } +} + +/// The project's validated rules configuration, or why it is invalid. +pub(crate) fn rules_config_of( + project_dir: &Path, +) -> PyResult> { let project_dir: PathBuf = project_dir .canonicalize() - .or_else(|_| std::path::absolute(&project_dir))?; - let config: RulesConfig = compiler_guard(|| Ok(load_config(&project_dir)))? - .map_err(|message| rules_error(py, ErrorId::Rules, message, None))?; + .or_else(|_| std::path::absolute(project_dir))?; + let config: RulesConfig = match compiler_guard(|| Ok(load_config(&project_dir)))? { + Ok(config) => config, + Err(message) => return Ok(Err(InvalidRulesConfig::Unreadable(message))), + }; if let Some(error) = forbidden_exceptions(&config) { - return Err(rules_error( - py, - ErrorId::RulesConfig, - error.message, - Some(error.help), - )); + return Ok(Err(InvalidRulesConfig::Forbidden { + message: error.message, + help: error.help, + })); } - Ok(config_object(py, &config)?.unbind()) + Ok(Ok(config)) } /// The project's rules configuration, or `None` when it is invalid and the rules phase reports it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index a041f3769e..2a46abc916 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -1,258 +1,86 @@ //! Lint authored SQL files: header rules, then expanded bodies through the native SQL engine. -use std::collections::BTreeSet; -use std::fs; -use std::path::{Path, PathBuf}; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::path::PathBuf; -use pyo3::exceptions::{PyException, PyOSError}; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; -use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, +use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ + SqlExpansionInputs, present, +}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::rules::lint_expansion::{ + LintExpansionSources, lint_expansion_inputs, +}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::rules::lint_expansion::build_lint_expansion_context; -use crate::bindings::_helpers::rules::lint_stop::check_stop; - -fn resolved(path: &Path) -> PathBuf { - fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) -} -fn selected_set(selected_paths: &Bound<'_, PyAny>) -> PyResult>> { - if selected_paths.is_none() { +/// The selected paths, or `None` for every file. +pub(crate) fn selected_paths(selected: &Bound<'_, PyAny>) -> PyResult>> { + if selected.is_none() { return Ok(None); } - selected_paths + selected .try_iter()? .map(|path| path?.extract()) .collect::>() .map(Some) } -/// Authored SQL files keyed by path: `source_files` filtered to the selection, or read from disk. -fn project_files<'py>( - project_dir: &Bound<'py, PyAny>, - selected_paths: &Bound<'py, PyAny>, - source_files: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let selected: Option> = selected_set(selected_paths)?; - let files: Bound<'py, PyDict> = PyDict::new(py); - if !source_files.is_none() { - for (path, contents) in source_files.downcast::()?.iter() { - let keep: bool = match &selected { - None => true, - Some(paths) => paths.contains(&resolved(&path.extract::()?)), - }; - if keep { - files.set_item(path, contents)?; - } - } - return Ok(files); +/// The files of a `{path: contents}` mapping, in its order. +pub(crate) fn given_files(files: &Bound<'_, PyAny>) -> PyResult>> { + if files.is_none() { + return Ok(None); + } + files + .downcast::()? + .iter() + .map(|(path, contents)| { + Ok(LintFile { + path: path.extract()?, + contents: contents.extract()?, + }) + }) + .collect::>() + .map(Some) +} + +/// The compile's expansions of a `{path: CompiledSqlExpansion}` mapping. +pub(crate) fn compiled_expansion_records( + expansions: &Bound<'_, PyAny>, +) -> PyResult> { + let mut compiled: HashMap = HashMap::new(); + if expansions.is_none() { + return Ok(compiled); } - let directory: PathBuf = project_dir.extract()?; - let collected: Vec<(PathBuf, String)> = collect_lint_files(&directory, selected.as_ref()) - .map_err(|error| PyOSError::new_err(error.to_string()))?; - for (path, contents) in collected { - files.set_item(path, contents)?; + for (path, expansion) in expansions.downcast::()?.iter() { + if !expansion.is_none() { + compiled.insert(path.extract()?, CompiledExpansion::from_record(&expansion)?); + } } - Ok(files) + Ok(compiled) } /// Collect supported SQL-analysis inputs of a project. #[pyfunction] #[pyo3(signature = (*, project_dir, selected_paths = None))] fn collect_lint_project_files<'py>( - project_dir: &Bound<'py, PyAny>, + py: Python<'py>, + project_dir: PathBuf, selected_paths: Option>, ) -> PyResult> { - let py: Python<'py> = project_dir.py(); - let none: Bound<'py, PyAny> = py.None().into_bound(py); - project_files( - project_dir, - &selected_paths.unwrap_or_else(|| none.clone()), - &none, - ) -} - -/// The keyword arguments of one lint run. -struct LintRun<'py> { - project_dir: Bound<'py, PyAny>, - config: Bound<'py, PyAny>, - dynamic_output_paths: Bound<'py, PyAny>, - compiled_expansions: Bound<'py, PyAny>, - skip_unexpandable: bool, -} - -/// One file's header violations and, when expanding, its expanded bodies. -fn scan_file<'py>( - run: &LintRun<'py>, - (file_path, contents): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - context: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Option>)> { - let py: Python<'py> = file_path.py(); - let relative_path: Bound<'py, PyAny> = - file_path.call_method1("relative_to", (&run.project_dir,))?; - let relative_parts: Vec = relative_path.getattr("parts")?.extract()?; - let headers: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ScanHeaders)?, - &[ - ("contents", contents.clone()), - ( - "first_only", - PyBool::new(py, first_header_only(&relative_parts)) - .to_owned() - .into_any(), - ), - ], - )?; - let header_violations: Bound<'py, PyAny> = - if run.config.getattr("header_rules_enabled")?.is_truthy()? { - call_with( - ®istry::native(py, NativeAttr::LintNativeHeaders)?, - &[ - ("contents", contents.clone()), - ("file_path", file_path.clone()), - ("headers", headers.clone()), - ("config", run.config.clone()), - ], - )? - } else { - PyTuple::empty(py).into_any() - }; - if context.is_none() { - return Ok((header_violations, None)); - } - let expanded: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::ExpandFileBodies)?, - &[ - ("file_path", file_path.clone()), - ("contents", contents.clone()), - ("headers", headers), - ("context", context.clone()), - ("dialect", run.config.getattr("dialect")?), - ("project_dir", run.project_dir.clone()), - ("relative_path", relative_path), - ("dynamic_output_paths", run.dynamic_output_paths.clone()), - ("compiled_expansions", run.compiled_expansions.clone()), - ], - )?; - Ok((header_violations, Some(expanded))) -} - -/// The expansion context: the caller's, or one built when native lint is enabled. -fn expansion_context<'py>( - run: &LintRun<'py>, - arguments: &KeywordArguments<'py>, -) -> PyResult> { - let py: Python<'py> = run.config.py(); - let given: Bound<'py, PyAny> = arguments.value("expansion_context")?; - if given.is_truthy()? { - return Ok(given); - } - if !run.config.getattr("native_enabled")?.is_truthy()? { - return Ok(py.None().into_bound(py)); - } - build_lint_expansion_context( - py, - Some(keywords( - py, - &[ - ("project_dir", run.project_dir.clone()), - ("value_renderer", arguments.value("value_renderer")?), - ("discovered_inputs", arguments.value("discovered_inputs")?), - ], - )?), - ) -} - -/// Violations sorted by file path text, then line and column. -fn sorted_violations<'py>(violations: Vec>) -> PyResult>> { - let mut keyed: Vec<((String, i64, i64), Bound<'py, PyAny>)> = violations - .into_iter() - .map(|violation| { - let key: (String, i64, i64) = ( - violation.getattr("file_path")?.str()?.extract()?, - violation.getattr("line")?.extract()?, - violation.getattr("column")?.extract()?, - ); - Ok((key, violation)) - }) - .collect::>()?; - keyed.sort_by(|left, right| left.0.cmp(&right.0)); - Ok(keyed.into_iter().map(|(_, violation)| violation).collect()) -} - -/// Header and expanded-body violations of every file, and the files expansion skipped. -fn lint_files<'py>( - run: &LintRun<'py>, - files: &Bound<'py, PyDict>, - context: &Bound<'py, PyAny>, -) -> PyResult<(Vec>, Bound<'py, PyAny>)> { - let py: Python<'py> = files.py(); - let mut items: Vec<(PathBuf, Bound<'py, PyAny>, Bound<'py, PyAny>)> = files - .iter() - .map(|(path, contents)| Ok((path.extract::()?, path, contents))) - .collect::>()?; - items.sort_by(|left, right| left.0.cmp(&right.0)); - let mut violations: Vec> = Vec::new(); - let mut expanded_files: Vec> = Vec::new(); - let mut failure: Option = None; - for (_, file_path, contents) in &items { - let (header_violations, expanded) = match scan_file(run, (file_path, contents), context) { - Ok(scanned) => scanned, - Err(error) if error.is_instance_of::(py) => { - failure = Some(error); - break; - } - Err(error) => return Err(error), - }; - for violation in header_violations.try_iter()? { - violations.push(violation?); - } - let Some(expanded) = expanded else { - continue; - }; - let failed: bool = !expanded.getattr("failure")?.is_none(); - expanded_files.push(expanded); - if failed && !run.skip_unexpandable { - break; - } - } - let skip: Bound<'py, PyAny> = PyBool::new(py, run.skip_unexpandable).to_owned().into_any(); - let prepared: Bound<'py, PyAny> = call_with( - ®istry::native(py, NativeAttr::PrepareExpandedFiles)?, - &[ - ( - "expanded_files", - PyTuple::new(py, expanded_files)?.into_any(), - ), - ("dialect", run.config.getattr("dialect")?), - ("skip_unexpandable", skip), - ], - )?; - if let Some(error) = failure { - return Err(error); - } - let bodies: Bound<'py, PyAny> = prepared.getattr("bodies")?; - if bodies.is_truthy()? && run.config.getattr("native_enabled")?.is_truthy()? { - check_stop(py)?; - let body_list: Vec> = bodies.extract()?; - let found: Bound<'py, PyAny> = registry::native(py, NativeAttr::LintNativeSqlBodies)? - .call1((body_list, files, &run.config))?; - for violation in found.try_iter()? { - violations.push(violation?); - } - check_stop(py)?; + let selected: Option> = match selected_paths { + Some(paths) => self::selected_paths(&paths)?, + None => None, + }; + let files: Bound<'py, PyDict> = PyDict::new(py); + for file in lint_sources(&project_dir, selected.as_ref(), None)? { + files.set_item(file.path, file.contents)?; } - Ok((violations, prepared.getattr("unexpandable")?)) + Ok(files) } /// Lint all DSL files without modifying anything; optionally skip unexpandable SQL. @@ -281,42 +109,51 @@ fn run_lint<'py>( ), )?; let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; - let dynamic_output_paths: Bound<'py, PyAny> = if arguments.passed("dynamic_output_paths")? { - arguments.value("dynamic_output_paths")? + let settings: LintSettings = LintSettings::from_record(&arguments.value("config")?)?; + let project_path: PathBuf = project_dir.extract()?; + let files: Vec = lint_sources( + &project_path, + selected_paths(&arguments.value("selected_paths")?)?.as_ref(), + given_files(&arguments.value("source_files")?)?, + )?; + let given: Bound<'py, PyAny> = arguments.value("expansion_context")?; + let context: Option = if given.is_truthy()? { + Some(SqlExpansionInputs::from_record(&given)?) + } else if settings.native_enabled { + let renderer: Bound<'py, PyAny> = arguments.value("value_renderer")?; + Some(lint_expansion_inputs(LintExpansionSources { + project_dir: project_dir.clone(), + value_renderer: renderer.is_truthy()?.then_some(renderer), + discovered_inputs: present(arguments.value("discovered_inputs")?), + declaration_scope: None, + static_declaration_scope: None, + })?) } else { - PyFrozenSet::empty(py)?.into_any() - }; - let run: LintRun<'py> = LintRun { - project_dir, - config: arguments.value("config")?, - dynamic_output_paths, - compiled_expansions: arguments.value("compiled_expansions")?, - skip_unexpandable: arguments.flag("skip_unexpandable")?, + None }; - let files: Bound<'py, PyDict> = project_files( - &run.project_dir, - &arguments.value("selected_paths")?, - &arguments.value("source_files")?, + let mut dynamic_output_paths: HashSet = HashSet::new(); + let dynamic: Bound<'py, PyAny> = arguments.value("dynamic_output_paths")?; + if !dynamic.is_none() { + for path in dynamic.try_iter()? { + dynamic_output_paths.insert(path?.extract()?); + } + } + let outcome: LintOutcome = lint( + py, + &LintRequest { + project_dir: &project_path, + settings: &settings, + dynamic_output_paths: &dynamic_output_paths, + compiled_expansions: &compiled_expansion_records( + &arguments.value("compiled_expansions")?, + )?, + context: context.as_ref(), + skip_unexpandable: arguments.flag("skip_unexpandable")?, + stop: None, + }, + files, )?; - let context: Bound<'py, PyAny> = expansion_context(&run, &arguments)?; - let (violations, unexpandable) = lint_files(&run, &files, &context)?; - let allow_suppressions: Bound<'py, PyAny> = run.config.getattr("allow_suppressions")?; - let retained: Vec> = registry::native(py, NativeAttr::ApplyLintSuppressions)? - .call1((violations, &files, allow_suppressions))? - .extract()?; - call_with( - ®istry::class(py, RecordId::LintRunResult)?, - &[ - ("files_checked", files.len().into_pyobject(py)?.into_any()), - ( - "violations", - PyTuple::new(py, sorted_violations(retained)?)?.into_any(), - ), - ("formatted_files", PyTuple::empty(py).into_any()), - ("source_texts", files.into_any()), - ("unexpandable", unexpandable), - ], - ) + outcome.record(py) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index c7de972fb2..038fbec56e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -1,34 +1,51 @@ //! Run the planned SQL rules through native lint, reusing cached and early-lint results. -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::path::{Path, PathBuf}; use std::time::Instant; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{ - PyDict, PyDictMethods, PyFrozenSet, PyList, PyListMethods, PySet, PySetMethods, PyString, - PyTuple, +use pyo3::prelude::{ + Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, + Python, }; +use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_rules::rule_cache::main::file_rule_key::file_rule_key; +use sqlbuild_rules::rule_results::main::duplicate_literal_hints::duplicate_literal_hints; +use sqlbuild_rules::rule_results::models::RuleFinding; use crate::bindings::_helpers::analysis::model_reads::output_proven; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::SAFE_RULE_FIX_CODES; -use crate::bindings::_helpers::rules::early_sql_lint::{sql_lint_config, sql_rule_cache_path}; +use crate::bindings::_helpers::rules::early_sql_lint::{ + EarlyLintResult, EarlySqlLint, PreparedSqlLint, sql_lint_settings, sql_rule_cache_path, +}; +use crate::bindings::_helpers::rules::lint_expansion::{ + LintExpansionSources, lint_expansion_inputs, +}; +use crate::bindings::_helpers::rules::lint_inputs::project_sql_expansions; +use crate::bindings::_helpers::rules::lint_pipeline::{ + CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, LintViolationRow, lint, + lint_sources, outside_project, resolved, +}; +use crate::bindings::_helpers::rules::relation_keys::project_relation_keys; +use crate::bindings::_helpers::rules::rule_cache::{ + StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, settings_lint_key, + stored_entries, write_rule_cache, +}; const DATACLASSES_MODULE: &str = "dataclasses"; const JSON_MODULE: &str = "json"; const BUILTINS_MODULE: &str = "builtins"; +const PATHLIB_MODULE: &str = "pathlib"; const SQL_RULE_PREFIX: &str = "SQBRSQL"; const SQL_RULE_SUPPRESSION_CODE: &str = "SQBRSQL000"; const DEFAULT_REMEDIATION: &str = "Update the authored SQL to satisfy this rule."; - -/// `native.name`, looked up at call time. -fn native<'py>(py: Python<'py>, name: NativeAttr) -> PyResult> { - registry::native(py, name) -} +const UTF8: &str = "utf-8"; /// The SQL rule codes among `rules`, in order. fn sql_rule_codes(rules: &[Bound<'_, PyAny>]) -> PyResult> { @@ -45,14 +62,30 @@ fn sql_rule_codes(rules: &[Bound<'_, PyAny>]) -> PyResult> { /// The settings, project and inputs one SQL rules evaluation reads. struct SqlRun<'a, 'py> { codes: &'a [String], - project_dir: Bound<'py, PyAny>, + project_dir: PathBuf, discovered_inputs: Bound<'py, PyAny>, project: Bound<'py, PyAny>, - lint_config: Bound<'py, PyAny>, - selected_model_paths: Bound<'py, PyAny>, + settings: LintSettings, + /// Whether the plan selected model paths, rather than the whole project. + model_selection: bool, cache_enabled: bool, - prepared_sql: Bound<'py, PyAny>, - expansion_reuse: Bound<'py, PyAny>, + early_lint: Option>, +} + +impl<'py> SqlRun<'_, 'py> { + /// The early SQL lint that started for this compile, if any. + fn prepared(&self) -> Option<&PreparedSqlLint> { + self.early_lint + .as_ref() + .and_then(|early| early.preparation()) + .map(|preparation| preparation.get()) + } +} + +/// `path` with forward slashes, as `PurePath.as_posix` writes it. +fn posix(path: &Path) -> String { + path.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") } /// Run the planned SQL rules on the caller while built-in rules evaluate natively. @@ -72,39 +105,43 @@ fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult< } } let codes: Vec = sql_rule_codes(&native_rules)?; - let lint_config: Bound<'py, PyAny> = sql_lint_config( - &options.get("dialect")?, + let thresholds: Bound<'py, PyAny> = effective_config.getattr("thresholds")?; + let dialect: String = options.get("dialect")?.extract()?; + let settings: LintSettings = sql_lint_settings( + &dialect, &codes, - &effective_config.getattr("thresholds")?, + |key| thresholds.call_method1("get", (key,))?.extract(), ( - native(py, NativeAttr::CompiledRelationKeys)?.call1((graph.getattr("project")?,))?, - effective_config.getattr("allow_exceptions")?, + project_relation_keys(&graph.getattr("project")?)?, + effective_config.getattr("allow_exceptions")?.is_truthy()?, ), )?; + let early_lint: Bound<'py, PyAny> = options.get("early_lint")?; let run = SqlRun { codes: &codes, - project_dir: plan.getattr("project_dir")?, + project_dir: plan.getattr("project_dir")?.extract()?, discovered_inputs: options.get("discovered_inputs")?, project: plan.getattr("project")?, - lint_config, - selected_model_paths: plan.getattr("model_paths")?, + settings, + model_selection: !plan.getattr("model_paths")?.is_none(), cache_enabled: effective_config .getattr("cache")? .getattr("enabled")? .is_truthy()?, - prepared_sql: options.get("prepared_sql")?, - expansion_reuse: options.get("expansion_reuse")?, + early_lint: if early_lint.is_none() { + None + } else { + Some(early_lint.downcast::()?.borrow()) + }, }; - let lint_key: String = lint_key(&run.lint_config)?; + let lint_key: String = settings_lint_key(&run.settings); let identities: Vec<(String, String)> = model_rule_identities(&run, &lint_key)?; - let project_files: Option> = if !identities.is_empty() - && run.prepared_sql.is_none() - && run.selected_model_paths.is_none() - { - Some(collected_project_files(&run.project_dir)?) - } else { - None - }; + let project_files: Option> = + if !identities.is_empty() && run.prepared().is_none() && !run.model_selection { + Some(lint_sources(&run.project_dir, None, None)?) + } else { + None + }; let started: Instant = Instant::now(); let (findings, hits, misses) = evaluated_sql_rules(&run, (&lint_key, identities), project_files)?; @@ -112,7 +149,7 @@ fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult< call_with( ®istry::class(py, RecordId::SqlRulesEvaluation)?, &[ - ("findings", findings), + ("findings", finding_objects(py, findings)?.into_any()), ("cache_hits", hits.into_bound_py_any(py)?), ("cache_misses", misses.into_bound_py_any(py)?), ("sql_ms", sql_ms.into_bound_py_any(py)?), @@ -120,23 +157,32 @@ fn run_planned_sql_rules<'py>(options: Option<&Bound<'py, PyDict>>) -> PyResult< ) } -fn lint_key(lint_config: &Bound<'_, PyAny>) -> PyResult { - call_with( - &native(lint_config.py(), NativeAttr::SqlRuleLintKey)?, - &[("config", lint_config.clone())], - )? - .extract() -} - -fn collected_project_files<'py>(project_dir: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = project_dir.py(); - call_with( - &native(py, NativeAttr::CollectLintProjectFiles)?, - &[ - ("project_dir", project_dir.clone()), - ("selected_paths", py.None().into_bound(py)), - ], - ) +/// Finding rows as `Finding` records. +fn finding_objects(py: Python<'_>, findings: Vec) -> PyResult> { + let class: Bound<'_, PyAny> = registry::class(py, RecordId::Finding)?; + let path_class: Bound<'_, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; + let objects: Vec> = findings + .into_iter() + .map(|finding| { + call_with( + &class, + &[ + ("code", PyString::new(py, &finding.code).into_any()), + ("path", path_class.call1((finding.path,))?), + ("line", finding.line.into_pyobject(py)?.into_any()), + ("column", finding.column.into_pyobject(py)?.into_any()), + ("message", PyString::new(py, &finding.message).into_any()), + ( + "remediation", + PyString::new(py, &finding.remediation).into_any(), + ), + ("unevaluated", finding.unevaluated.into_bound_py_any(py)?), + ("fixable", finding.fixable.into_bound_py_any(py)?), + ], + ) + }) + .collect::>()?; + PyTuple::new(py, objects) } /// The project's models SQL rules analyse, keyed by relative path in project order. @@ -174,192 +220,127 @@ fn sql_rule_models<'py>(project: &Bound<'py, PyAny>) -> PyResult, lint_key: &str) -> PyResult> { - let py: Python<'_> = run.project.py(); if run.codes.is_empty() { return Ok(Vec::new()); } - let mut models: Vec> = Vec::new(); - let mut paths: Vec = Vec::new(); + let mut models: Vec<(String, String, String, Option)> = Vec::new(); for (path, model) in sql_rule_models(&run.project)? { - models.push(PyTuple::new( - py, - [ - PyString::new(py, &path).into_any(), - model.getattr("authored_sql")?, - model.getattr("query_sql")?, - contract_text(&model)?, - ], - )?); - paths.push(path); + models.push(( + path, + model.getattr("authored_sql")?.extract()?, + model.getattr("query_sql")?.extract()?, + contract_text(&model)?, + )); } - let keys: Bound<'_, PyAny> = call_with( - &native(py, NativeAttr::SqlModelRuleKeys)?, - &[ - ("lint_key", PyString::new(py, lint_key).into_any()), - ("codes", PyList::new(py, run.codes)?.into_any()), - ("models", PyList::new(py, models)?.into_any()), - ], - )?; - paths + let keys: BTreeMap = model_rule_keys(lint_key, run.codes, &models); + Ok(models .into_iter() - .map(|path| { - let key: String = keys.get_item(&path)?.extract()?; - Ok((path, key)) + .map(|(path, ..)| { + let key: String = keys[&path].clone(); + (path, key) }) - .collect() + .collect()) } /// Canonical text of the dynamic column contract that SQL rule findings depend on. -fn contract_text<'py>(model: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = model.py(); - let contract: Bound<'py, PyAny> = model.getattr("dynamic_column_contract")?; +fn contract_text(model: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model.py(); + let contract: Bound<'_, PyAny> = model.getattr("dynamic_column_contract")?; if contract.is_none() { - return Ok(contract); - } - imported(py, JSON_MODULE, "dumps")?.call( - (imported(py, DATACLASSES_MODULE, "asdict")?.call1((contract,))?,), - Some(&keywords( - py, - &[ - ("sort_keys", true.into_bound_py_any(py)?), - ("separators", (",", ":").into_bound_py_any(py)?), - ("default", imported(py, BUILTINS_MODULE, "str")?), - ], - )?), - ) -} - -/// The stored findings of every path whose key still matches; undecodable ones miss. -fn cached_findings<'py>( - bucket: &Bound<'py, PyAny>, - keys: Vec<(String, String)>, -) -> PyResult> { - let py: Python<'py> = bucket.py(); - Ok(call_with( - &native(py, NativeAttr::LookupSqlRuleCache)?, - &[ - ("entries", bucket.clone()), - ("keys", keys.into_bound_py_any(py)?), - ], - )? - .downcast_into()?) -} - -/// The early SQL lint's full-project file read, when it ran for this project and these codes. -fn prepared_project_files<'py>(run: &SqlRun<'_, 'py>) -> PyResult>> { - let py: Python<'py> = run.project_dir.py(); - let prepared: &Bound<'py, PyAny> = &run.prepared_sql; - if prepared.is_none() - || prepared.getattr("project_dir")?.ne(&run.project_dir)? - || !same_codes(&prepared.getattr("codes")?, run.codes)? - || !prepared - .getattr("future")? - .call_method0("exception")? - .is_none() - { return Ok(None); } - let texts: Bound<'py, PyAny> = prepared - .getattr("future")? - .call_method0("result")? - .getattr("result")? - .getattr("source_texts")?; - Ok(Some( - imported(py, BUILTINS_MODULE, "dict")?.call1((texts,))?, - )) + imported(py, JSON_MODULE, "dumps")? + .call( + (imported(py, DATACLASSES_MODULE, "asdict")?.call1((contract,))?,), + Some(&keywords( + py, + &[ + ("sort_keys", true.into_bound_py_any(py)?), + ("separators", (",", ":").into_bound_py_any(py)?), + ("default", imported(py, BUILTINS_MODULE, "str")?), + ], + )?), + )? + .extract() + .map(Some) } -fn same_codes(codes: &Bound<'_, PyAny>, expected: &[String]) -> PyResult { - let mut known: BTreeSet = BTreeSet::new(); - if !codes.is_none() { - for code in codes.try_iter()? { - known.insert(code?.extract()?); - } +/// The early SQL lint's result, when it ran for this project and these codes. +fn prepared_result<'a>( + run: &'a SqlRun<'_, '_>, + py: Python<'_>, +) -> Option<&'a Result> { + let prepared: &PreparedSqlLint = run.prepared()?; + if !prepared.matches(&run.project_dir, run.codes) { + return None; } - Ok(known == expected.iter().cloned().collect()) + Some(prepared.result(py)) } /// Each analysed model's findings, cached or linted, then cache accounting. -fn evaluated_sql_rules<'py>( - run: &SqlRun<'_, 'py>, +fn evaluated_sql_rules( + run: &SqlRun<'_, '_>, (lint_key, identities): (&str, Vec<(String, String)>), - prepared_files: Option>, -) -> PyResult<(Bound<'py, PyAny>, usize, usize)> { - let py: Python<'py> = run.project_dir.py(); + prepared_files: Option>, +) -> PyResult<(Vec, usize, usize)> { + let py: Python<'_> = run.project.py(); if run.codes.is_empty() { - return Ok((PyTuple::empty(py).into_any(), 0, 0)); + return Ok((Vec::new(), 0, 0)); } - let project_dir: &Bound<'py, PyAny> = &run.project_dir; - let models: Vec<(String, Bound<'py, PyAny>)> = sql_rule_models(&run.project)?; - let cache_path: Bound<'py, PyAny> = sql_rule_cache_path(project_dir)?; - let bucket: Bound<'py, PyAny> = if run.cache_enabled { - call_with( - &native(py, NativeAttr::LoadSqlRuleCache)?, - &[("path", cache_path.clone())], - )? + let project_dir: &Path = &run.project_dir; + let models: Vec<(String, Bound<'_, PyAny>)> = sql_rule_models(&run.project)?; + let cache_path: PathBuf = sql_rule_cache_path(project_dir); + let mut bucket: StoredEntries = if run.cache_enabled { + stored_entries(&cache_path)? } else { - PyDict::new(py).into_any() + StoredEntries::new() }; - let findings: Bound<'py, PyList> = PyList::empty(py); - let model_hits: Bound<'py, PyDict> = cached_findings(&bucket, identities.clone())?; + let mut findings: Vec = Vec::new(); + let mut model_hits: BTreeMap> = + cached_rule_findings(&bucket, &identities)?; let mut misses: Vec<(String, String)> = Vec::new(); for (path, identity) in &identities { - match model_hits.get_item(path)? { - Some(hit) => { - for finding in hit.try_iter()? { - findings.append(finding?)?; - } - } + match model_hits.remove(path) { + Some(hit) => findings.extend(hit), None => misses.push((path.clone(), identity.clone())), } } - let selected_paths: Bound<'py, PySet> = PySet::empty(py)?; - for (path, _) in &misses { - selected_paths.add(project_dir.div(path)?)?; - } + let mut selected_paths: BTreeSet = misses + .iter() + .map(|(path, _)| project_dir.join(path)) + .collect(); let mut file_identities: Vec<(String, String)> = Vec::new(); - let mut project_files: Bound<'py, PyAny> = py.None().into_bound(py); - if run.selected_model_paths.is_none() { - let model_paths: Bound<'py, PySet> = PySet::empty(py)?; + let mut project_files: Option> = None; + let compiled: HashMap = project_sql_expansions(&run.project)?; + if !run.model_selection { + let mut model_paths: HashSet = HashSet::new(); for model in run.project.getattr("models")?.try_iter()? { - model_paths.add(project_dir.div(model?.getattr("relative_path")?)?)?; + model_paths + .insert(project_dir.join(model?.getattr("relative_path")?.extract::()?)); } - project_files = match prepared_project_files(run)? { - Some(files) => files, - None => match prepared_files { + let files: Vec = match prepared_result(run, py) { + Some(Ok(result)) => result.outcome.files.clone(), + Some(Err(_)) | None => match prepared_files { Some(files) => files, - None => collected_project_files(project_dir)?, + None => lint_sources(project_dir, None, None)?, }, }; - let expansions: Bound<'py, PyAny> = run.project.getattr("sql_expansions")?; - let mut file_keys: Vec<(Bound<'py, PyAny>, String, Option)> = Vec::new(); - for item in project_files.downcast::()?.iter() { - let (file_path, contents) = item; - if model_paths.contains(&file_path)? { + let mut file_keys: Vec<(PathBuf, String, Option)> = Vec::new(); + for file in &files { + if model_paths.contains(&file.path) { continue; } - let relative_path: String = file_path - .call_method1("relative_to", (project_dir,))? - .call_method0("as_posix")? - .extract()?; - let key: Option = call_with( - &native(py, NativeAttr::SqlFileRuleKey)?, - &[ - ("lint_key", PyString::new(py, lint_key).into_any()), - ("codes", PyList::new(py, run.codes)?.into_any()), - ( - "relative_path", - PyString::new(py, &relative_path).into_any(), - ), - ("contents", contents), - ( - "expanded", - expansions.contains(&file_path)?.into_bound_py_any(py)?, - ), - ], - )? - .extract()?; - file_keys.push((file_path, relative_path, key)); + let Ok(relative) = file.path.strip_prefix(project_dir) else { + return Err(outside_project(&file.path, project_dir)); + }; + let relative_path: String = posix(relative); + let key: Option = file_rule_key( + lint_key, + run.codes, + (&relative_path, &file.contents), + compiled.contains_key(&file.path), + ); + file_keys.push((file.path.clone(), relative_path, key)); } let mut keyed: Vec<(String, String)> = Vec::new(); for (_, path, key) in &file_keys { @@ -367,246 +348,230 @@ fn evaluated_sql_rules<'py>( keyed.push((path.clone(), key.clone())); } } - let file_hits: Bound<'py, PyDict> = cached_findings(&bucket, keyed)?; + let mut file_hits: BTreeMap> = + cached_rule_findings(&bucket, &keyed)?; for (file_path, relative_path, identity) in file_keys { - match file_hits.get_item(&relative_path)? { - Some(hit) => { - for finding in hit.try_iter()? { - findings.append(finding?)?; - } - } + match file_hits.remove(&relative_path) { + Some(hit) => findings.extend(hit), None => { - selected_paths.add(file_path)?; + selected_paths.insert(file_path); if let Some(identity) = identity { file_identities.push((relative_path, identity)); } } } } + project_files = Some(files); } - let violations: Vec> = if selected_paths.is_empty() { + let violations: Vec<(PathBuf, LintViolationRow)> = if selected_paths.is_empty() { Vec::new() } else { - let mut dynamic: Vec> = Vec::new(); + let mut dynamic: HashSet = HashSet::new(); for (_, model) in &models { if output_proven(model)? { - dynamic.push( - project_dir - .div(model.getattr("relative_path")?)? - .call_method0("resolve")?, - ); + dynamic.insert(resolved( + &project_dir.join(model.getattr("relative_path")?.extract::()?), + )); } } - prepared_lint_violations( - run, - &PyFrozenSet::new(py, selected_paths.iter())?, - &PyFrozenSet::new(py, dynamic)?, - &project_files, - )? + prepared_lint_violations(run, (&selected_paths, &dynamic), &compiled, project_files)? }; let model_paths: BTreeSet = models.into_iter().map(|(path, _)| path).collect(); - let evaluated: Vec> = violation_findings(run, &violations, &model_paths)?; - for finding in &evaluated { - findings.append(finding)?; - } + let evaluated: Vec = violation_findings(run, &violations, &model_paths)?; + findings.extend(evaluated.iter().cloned()); let mut written: Vec<(String, String)> = misses.clone(); written.extend(file_identities); if run.cache_enabled && !written.is_empty() { - write_cache(&bucket, (&cache_path, written), &evaluated)?; + let mut by_path: HashMap<&str, Vec<&RuleFinding>> = HashMap::new(); + for finding in &evaluated { + by_path + .entry(finding.path.as_str()) + .or_default() + .push(finding); + } + for (path, identity) in written { + let own: &[&RuleFinding] = by_path.get(path.as_str()).map_or(&[], Vec::as_slice); + let encoded: String = encoded_findings(own)?; + bucket.insert(path, (identity, encoded)); + } + write_rule_cache(&cache_path, &bucket)?; } - let hinted: Bound<'py, PyAny> = call_with( - &native(py, NativeAttr::WithDuplicateLiteralHints)?, - &[ - ("findings", PyTuple::new(py, findings.iter())?.into_any()), - ("project_dir", project_dir.clone()), - ], - )?; - Ok((hinted, identities.len() - misses.len(), misses.len())) + let hints: Vec> = + duplicate_literal_hints::(&findings, |path| file_lines(py, project_dir, path))?; + for (finding, hint) in findings.iter_mut().zip(hints) { + if let Some(remediation) = hint { + finding.remediation = remediation; + } + } + Ok((findings, identities.len() - misses.len(), misses.len())) } -/// Store each rewritten path's evaluated findings in `bucket` and replace the cache file. -fn write_cache<'py>( - bucket: &Bound<'py, PyAny>, - (cache_path, written): (&Bound<'py, PyAny>, Vec<(String, String)>), - evaluated: &[Bound<'py, PyAny>], -) -> PyResult<()> { - let py: Python<'py> = bucket.py(); - let encode: Bound<'py, PyAny> = native(py, NativeAttr::EncodeSqlRuleFindings)?; - let mut by_path: HashMap>> = HashMap::new(); - for finding in evaluated { - let path: String = finding - .getattr("path")? - .call_method0("as_posix")? - .extract()?; - by_path.entry(path).or_default().push(finding.clone()); - } - for (path, identity) in written { - let own: &[Bound<'py, PyAny>] = by_path.get(&path).map(Vec::as_slice).unwrap_or_default(); - let encoded: Bound<'py, PyAny> = - call_with(&encode, &[("findings", PyList::new(py, own)?.into_any())])?; - bucket.set_item(path, (identity, encoded))?; - } - let _ = call_with( - &native(py, NativeAttr::WriteSqlRuleCache)?, - &[("path", cache_path.clone()), ("entries", bucket.clone())], - )?; - Ok(()) +/// The authored lines of one finding's file; an unreadable file has none. +fn file_lines(py: Python<'_>, project_dir: &Path, path: &str) -> PyResult> { + let bytes: Vec = match std::fs::read(project_dir.join(path)) { + Ok(bytes) => bytes, + Err(_) => return Ok(Vec::new()), + }; + let text: String = match String::from_utf8(bytes) { + Ok(text) => text, + Err(error) => { + return PyBytes::new(py, error.as_bytes()) + .call_method1("decode", (UTF8,))? + .extract(); + } + }; + Ok(python_splitlines(&text) + .into_iter() + .map(str::to_owned) + .collect()) } -/// Lint violations of `selected_paths`, reusing a matching early SQL lint. -fn prepared_lint_violations<'py>( - run: &SqlRun<'_, 'py>, - selected_paths: &Bound<'py, PyFrozenSet>, - dynamic_output_paths: &Bound<'py, PyFrozenSet>, - source_files: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = run.project_dir.py(); - let run_lint: Bound<'py, PyAny> = native(py, NativeAttr::RunLint)?; - let compiled_expansions: Bound<'py, PyAny> = run.project.getattr("sql_expansions")?; - let enabled_codes: Bound<'py, PyAny> = run.lint_config.getattr("enabled_native_rules")?; - let enabled: Vec = if enabled_codes.is_none() { - Vec::new() - } else { - enabled_codes.extract()? +/// Lint violations of `selected_paths` by file path, reusing a matching early SQL lint. +fn prepared_lint_violations( + run: &SqlRun<'_, '_>, + (selected_paths, dynamic_output_paths): (&BTreeSet, &HashSet), + compiled: &HashMap, + source_files: Option>, +) -> PyResult> { + let py: Python<'_> = run.project.py(); + let enabled: &[String] = run + .settings + .sql + .enabled_rules + .as_deref() + .unwrap_or_default(); + let prepared: Option<&PreparedSqlLint> = run + .prepared() + .filter(|prepared| prepared.has_codes(enabled)); + let Some(prepared) = prepared else { + let context = if run.settings.native_enabled { + Some(lint_expansion_context(run)?) + } else { + None + }; + let files: Vec = + lint_sources(&run.project_dir, Some(selected_paths), source_files)?; + let outcome: LintOutcome = lint( + py, + &LintRequest { + project_dir: &run.project_dir, + settings: &run.settings, + dynamic_output_paths, + compiled_expansions: compiled, + context: context.as_ref(), + skip_unexpandable: false, + stop: None, + }, + files, + )?; + return Ok(path_rows(outcome)); }; - if run.prepared_sql.is_none() || !same_codes(&run.prepared_sql.getattr("codes")?, &enabled)? { - let expansion_context: Bound<'py, PyAny> = - if run.lint_config.getattr("native_enabled")?.is_truthy()? { - lint_expansion_context(run)? - } else { - py.None().into_bound(py) - }; - return call_with( - &run_lint, - &[ - ("project_dir", run.project_dir.clone()), - ("config", run.lint_config.clone()), - ("selected_paths", selected_paths.clone().into_any()), - ("discovered_inputs", run.discovered_inputs.clone()), - ("compiled_expansions", compiled_expansions), - ( - "dynamic_output_paths", - dynamic_output_paths.clone().into_any(), - ), - ("expansion_context", expansion_context), - ("source_files", source_files.clone()), - ], - )? - .getattr("violations")? - .try_iter()? + let result: &EarlyLintResult = match prepared.result(py) { + Ok(result) => result, + Err(error) => return Err(error.clone_ref(py)), + }; + let proof_paths: BTreeSet = selected_paths + .iter() + .filter(|path| dynamic_output_paths.contains(*path)) + .cloned() .collect(); - } - let completed: Bound<'py, PyAny> = - run.prepared_sql.getattr("future")?.call_method0("result")?; - let prepared: Bound<'py, PyAny> = completed.getattr("result")?; - let proof_paths: Bound<'py, PyAny> = - selected_paths.call_method1("__and__", (dynamic_output_paths,))?; - let mut violations: Vec> = Vec::new(); - for violation in prepared.getattr("violations")?.try_iter()? { - let violation: Bound<'py, PyAny> = violation?; - let path: Bound<'py, PyAny> = violation.getattr("file_path")?; - if selected_paths.contains(&path)? && !proof_paths.contains(&path)? { - violations.push(violation); - } - } - if proof_paths.is_truthy()? { - let proven: Bound<'py, PyAny> = call_with( - &run_lint, - &[ - ("project_dir", run.project_dir.clone()), - ("config", run.lint_config.clone()), - ("selected_paths", proof_paths.clone()), - ("discovered_inputs", run.discovered_inputs.clone()), - ("compiled_expansions", compiled_expansions), - ("dynamic_output_paths", proof_paths), - ("expansion_context", completed.getattr("context")?), - ( - "source_files", - imported(py, BUILTINS_MODULE, "dict")? - .call1((prepared.getattr("source_texts")?,))?, - ), - ], + let early: &LintOutcome = &result.outcome; + let mut violations: Vec<(PathBuf, LintViolationRow)> = early + .violations + .iter() + .filter_map(|row| { + let path: &PathBuf = &early.files[row.file].path; + (selected_paths.contains(path) && !proof_paths.contains(path)) + .then(|| (path.clone(), row.clone())) + }) + .collect(); + if !proof_paths.is_empty() { + let proof_set: HashSet = proof_paths.iter().cloned().collect(); + let files: Vec = lint_sources( + &run.project_dir, + Some(&proof_paths), + Some(early.files.clone()), )?; - for violation in proven.getattr("violations")?.try_iter()? { - violations.push(violation?); - } + let proven: LintOutcome = lint( + py, + &LintRequest { + project_dir: &run.project_dir, + settings: &run.settings, + dynamic_output_paths: &proof_set, + compiled_expansions: compiled, + context: Some(&result.context), + skip_unexpandable: false, + stop: None, + }, + files, + )?; + violations.extend(path_rows(proven)); } Ok(violations) } +/// An outcome's violations with their file paths. +fn path_rows(outcome: LintOutcome) -> Vec<(PathBuf, LintViolationRow)> { + let files: Vec = outcome.files; + outcome + .violations + .into_iter() + .map(|row| (files[row.file].path.clone(), row)) + .collect() +} + /// Build lint expansion, reusing the compiler's scope only for the same discovery. -fn lint_expansion_context<'py>(run: &SqlRun<'_, 'py>) -> PyResult> { - let py: Python<'py> = run.project_dir.py(); - let reuse: &Bound<'py, PyAny> = &run.expansion_reuse; - let declaration_scope: Bound<'py, PyAny> = if !reuse.is_none() - && reuse - .getattr("discovered_inputs")? - .is(&run.discovered_inputs) - { - reuse.getattr("declaration_scope")? - } else { - py.None().into_bound(py) - }; - call_with( - &native(py, NativeAttr::BuildLintExpansionContext)?, - &[ - ("project_dir", run.project_dir.clone()), - ("discovered_inputs", run.discovered_inputs.clone()), - ("declaration_scope", declaration_scope), - ], - ) +fn lint_expansion_context( + run: &SqlRun<'_, '_>, +) -> PyResult< + crate::bindings::_helpers::compile_attachment::sql_expansion_context::SqlExpansionInputs, +> { + let py: Python<'_> = run.project.py(); + let declaration_scope: Option> = run + .early_lint + .as_ref() + .and_then(|early| early.expansion_reuse()) + .filter(|reuse| reuse.discovered_inputs.bind(py).is(&run.discovered_inputs)) + .map(|reuse| reuse.declaration_scope.bind(py).clone()); + lint_expansion_inputs(LintExpansionSources { + project_dir: run.project_dir.clone().into_pyobject(py)?.into_any(), + value_renderer: None, + discovered_inputs: (!run.discovered_inputs.is_none()) + .then(|| run.discovered_inputs.clone()), + declaration_scope, + static_declaration_scope: None, + }) } /// The findings of the selected SQL rules' violations; an engine failure leaves each unevaluated. -fn violation_findings<'py>( - run: &SqlRun<'_, 'py>, - violations: &[Bound<'py, PyAny>], +fn violation_findings( + run: &SqlRun<'_, '_>, + violations: &[(PathBuf, LintViolationRow)], model_paths: &BTreeSet, -) -> PyResult>> { - let py: Python<'py> = run.project_dir.py(); +) -> PyResult> { + let py: Python<'_> = run.project.py(); let engine_error: String = registry::error(py, ErrorId::NativeLint)? .getattr("code")? .extract()?; - let safe_fixes: Bound<'py, PyAny> = PyFrozenSet::new(py, SAFE_RULE_FIX_CODES)?.into_any(); - let resolved: Bound<'py, PyAny> = run.project_dir.call_method0("resolve")?; - let replace: Bound<'py, PyAny> = imported(py, DATACLASSES_MODULE, "replace")?; + let resolved_dir: PathBuf = resolved(&run.project_dir); let no_paths: BTreeSet = BTreeSet::new(); - let mut evaluated: Vec> = Vec::new(); - for violation in violations { - let code: String = violation.getattr("code")?.extract()?; - if code == SQL_RULE_SUPPRESSION_CODE || run.codes.contains(&code) { - evaluated.push(lint_finding( - violation, - (&resolved, &safe_fixes), - model_paths, - )?); + let mut evaluated: Vec = Vec::new(); + for (path, violation) in violations { + let code: &str = &violation.code; + if code == SQL_RULE_SUPPRESSION_CODE || run.codes.iter().any(|known| known == code) { + evaluated.push(lint_finding((path, violation), &resolved_dir, model_paths)); } else if code == engine_error { - let finding: Bound<'py, PyAny> = - lint_finding(violation, (&resolved, &safe_fixes), &no_paths)?; - let message: String = violation.getattr("message")?.extract()?; + let finding: RuleFinding = lint_finding((path, violation), &resolved_dir, &no_paths); for rule_code in run.codes { - evaluated.push( - replace.call( - (&finding,), - Some(&keywords( - py, - &[ - ("code", PyString::new(py, rule_code).into_any()), - ( - "message", - PyString::new( - py, - &format!( - "Rule {rule_code} could not be evaluated: {message}" - ), - ) - .into_any(), - ), - ("unevaluated", true.into_bound_py_any(py)?), - ], - )?), - )?, - ); + evaluated.push(RuleFinding { + code: rule_code.clone(), + message: format!( + "Rule {rule_code} could not be evaluated: {}", + violation.message + ), + unevaluated: true, + ..finding.clone() + }); } } } @@ -614,44 +579,35 @@ fn violation_findings<'py>( } /// One lint violation as a finding; fixable when `format --fix` would attempt its edit. -fn lint_finding<'py>( - violation: &Bound<'py, PyAny>, - (resolved_dir, safe_fixes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +fn lint_finding( + (path, violation): (&Path, &LintViolationRow), + resolved_dir: &Path, fixable_paths: &BTreeSet, -) -> PyResult> { - let py: Python<'py> = violation.py(); - let mut path: Bound<'py, PyAny> = violation.getattr("file_path")?; - if path.call_method0("is_absolute")?.is_truthy()? - && path - .call_method1("is_relative_to", (resolved_dir,))? - .is_truthy()? - { - path = path.call_method1("relative_to", (resolved_dir,))?; +) -> RuleFinding { + let path: &Path = if path.is_absolute() { + path.strip_prefix(resolved_dir).unwrap_or(path) + } else { + path + }; + let path: String = posix(path); + let fixable: bool = violation.fix.is_some() + && SAFE_RULE_FIX_CODES.contains(&violation.code.as_str()) + && fixable_paths.contains(&path); + RuleFinding { + code: violation.code.clone(), + path, + line: violation.line, + column: violation.column, + message: violation.message.clone(), + remediation: violation + .remediation + .clone() + .filter(|remediation| !remediation.is_empty()) + .unwrap_or_else(|| DEFAULT_REMEDIATION.to_owned()), + unevaluated: false, + affected_rules: Vec::new(), + fixable, } - let code: Bound<'py, PyAny> = violation.getattr("code")?; - let remediation: Bound<'py, PyAny> = violation.getattr("remediation")?; - let fixable: bool = !violation.getattr("fix")?.is_none() - && safe_fixes.contains(&code)? - && fixable_paths.contains(&path.call_method0("as_posix")?.extract::()?); - call_with( - ®istry::class(py, RecordId::Finding)?, - &[ - ("code", code), - ("path", path), - ("line", violation.getattr("line")?), - ("column", violation.getattr("column")?), - ("message", violation.getattr("message")?), - ( - "remediation", - if remediation.is_truthy()? { - remediation - } else { - PyString::new(py, DEFAULT_REMEDIATION).into_any() - }, - ), - ("fixable", fixable.into_bound_py_any(py)?), - ], - ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 952a6352e5..0ed383f959 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4,7 +4,6 @@ import logging import threading from collections import Counter from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set -from concurrent.futures import Executor, Future from contextlib import AbstractContextManager from contextvars import Token from dataclasses import Field @@ -12,7 +11,6 @@ from datetime import date, datetime from decimal import Decimal from enum import StrEnum from pathlib import Path, PurePath -from threading import Event from types import TracebackType from typing import Any, ClassVar, Literal, Protocol, Self, TextIO, TypedDict, overload @@ -2396,14 +2394,6 @@ def compile_facts(*, project: CompiledProjectView, project_dir: Path) -> Compile def lint_file_role( *, file_path: Path, project_dir: Path, relative_path: PurePath | None = None ) -> LintFileRole: ... -def lint_body_ranges( - *, - contents: str, - headers: tuple[HeaderSpan, ...], - file_path: Path, - project_dir: Path, - role: LintFileRole | None = None, -) -> tuple[tuple[int, int], ...]: ... def prepare_lint_body( *, role: LintFileRole, @@ -2412,39 +2402,7 @@ def prepare_lint_body( body_range: tuple[int, int], context: SqlExpansionContext, dialect: str, - external_identifiers: tuple[str, ...] = (), - allows_ceremonial_select: bool = False, - allows_dynamic_output_star: bool = False, - compiled_expansion: CompiledSqlExpansion | None = None, ) -> LintBody: ... -def expand_lint_body( - *, - role: LintFileRole, - file_path: Path, - contents: str, - body_range: tuple[int, int], - context: SqlExpansionContext, - dialect: str, - external_identifiers: tuple[str, ...] = (), - allows_ceremonial_select: bool = False, - allows_dynamic_output_star: bool = False, - compiled_expansion: CompiledSqlExpansion | None = None, -) -> PendingLintBody: ... -def expand_file_bodies( - *, - file_path: Path, - contents: str, - headers: tuple[HeaderSpan, ...], - context: SqlExpansionContext, - dialect: str, - project_dir: Path, - relative_path: Path, - dynamic_output_paths: frozenset[Path], - compiled_expansions: dict[Path, CompiledSqlExpansion] | None, -) -> ExpandedLintFile: ... -def prepare_expanded_files( - *, expanded_files: tuple[ExpandedLintFile, ...], dialect: str, skip_unexpandable: bool -) -> PreparedLintFiles: ... def plan_rule_fixes( *, project_dir: Path, @@ -4140,9 +4098,6 @@ def lint_sql_batch_json(request_json: str) -> str: ... def prepare_lint_sql( request: LintPreparationRequest, ) -> tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None: ... -def prepare_lint_sql_batch( - requests: list[LintPreparationRequest], -) -> list[tuple[bool, tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None]]: ... def lint_backtick_identifiers(dialect: str) -> bool: ... def neutralize_lint_interpolation( body: str, dialect: str, for_formatting: bool = False @@ -4863,12 +4818,6 @@ class ProcessResourceTracker: ) -> None: ... def finish(self) -> ProcessResourceUsage: ... -class LintStopScope: - def __enter__(self) -> None: ... - def __exit__(self, *exception: object) -> bool: ... - -def lint_stop_scope(stop: Event, /) -> LintStopScope: ... -def active_lint_stop() -> Event | None: ... def error_code(*, error: BaseException, fallback_code: str) -> str: ... def error_help(error: BaseException, /) -> str | None: ... def error_message(error: BaseException, /) -> str: ... @@ -7064,22 +7013,9 @@ def evaluate_rules( project_dir: Path, dialect: str, selected_keys: frozenset[CompiledObjectKey] | None = None, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, + early_lint: EarlySqlLint | None = None, no_sql_analysis: bool = False, ) -> RulesRunResult: ... -def prepare_sql_rules( - *, inputs: CompileInputsView, executor: Executor, dialect: str, stop: Event -) -> PreparedSqlLint | None: ... -def prepare_early_sql_lint( - *, - project_dir: Path, - config: LintConfig, - discovered_inputs: DiscoveredProjectInputs, - static_declaration_scope: DeclarationScopeBuild | None, - compiled_expansions: dict[Path, CompiledSqlExpansion], - stop: Event, -) -> PreparedSqlLintResult: ... def custom_rule_test_evidence( *, rule: Rule, project_dir: Path ) -> list[tuple[str, int, int, str]]: ... @@ -7096,8 +7032,7 @@ def run_rules( project_dir: Path, dialect: str, selected_keys: frozenset[CompiledObjectKey] | None = None, - prepared_sql: PreparedSqlLint | None = None, - expansion_reuse: SqlExpansionReuse | None = None, + early_lint: EarlySqlLint | None = None, no_sql_analysis: bool = False, ) -> RulesRunResult: ... def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: ... @@ -7115,15 +7050,8 @@ def run_planned_sql_rules( graph: ProjectGraph, discovered_inputs: DiscoveredProjectInputs, dialect: str, - prepared_sql: PreparedSqlLint | None, - expansion_reuse: SqlExpansionReuse | None, + early_lint: EarlySqlLint | None, ) -> SqlRulesEvaluation: ... -def sql_lint_preparation( - *, inputs: CompileInputsView, dialect: str -) -> tuple[Path, tuple[str, ...], LintConfig] | None: ... -def early_lint_expansions( - model_inputs: tuple[ModelInput, ...], -) -> dict[Path, CompiledSqlExpansion]: ... class EarlySqlLint: """Own speculative SQL checks for one compilation invocation.""" @@ -7131,15 +7059,14 @@ class EarlySqlLint: def __init__(self, *, enabled: bool) -> None: ... @property def enabled(self) -> bool: ... - @property - def preparation(self) -> PreparedSqlLint | None: ... - @property - def expansion_reuse(self) -> SqlExpansionReuse | None: ... def __enter__(self) -> EarlySqlLint: ... def __exit__(self, *exception: object) -> None: ... def start(self, *, inputs: CompileInputsView, dialect: str) -> None: ... def stop(self) -> int: ... +class PreparedSqlLint: + """One early SQL lint running on a native thread, for one project and SQL rule codes.""" + def with_duplicate_literal_hints( *, findings: tuple[Finding, ...], project_dir: Path ) -> tuple[Finding, ...]: ... @@ -7176,9 +7103,6 @@ def safe_format_files( def reject_unparseable_header_rewrites( *, updated: dict[Path, str], config: LintConfig, faults: list[LintViolation] ) -> tuple[dict[Path, str], list[LintViolation]]: ... -def external_identifiers_for_headers( - *, contents: str, headers: tuple[HeaderSpan, ...] -) -> tuple[str, ...]: ... def prepare_model_header_tokens(headers: Sequence[str], /) -> None: ... def parse_header_values( *, @@ -10841,61 +10765,6 @@ class LintBody(NativeRecord): dependency_relations: tuple[tuple[str, str], ...] = ..., ) -> None: ... -class PendingLintBody(NativeRecord): - """One expanded authored body awaiting its native lexical preparation.""" - - file_path: Path - body_start: int - body_end: int - expansion_input: str - expanded: str - expansion_passes: tuple[tuple[ExpansionSpan, ...], ...] - pre_expansion_sites: tuple[InterpolationSite, ...] - external_identifiers: tuple[str, ...] - allows_ceremonial_select: bool - allows_dynamic_output_star: bool - allows_empty_fixture_star: bool - request: NativeLintPreparationRequest - def __init__( - self, - file_path: Path, - body_start: int, - body_end: int, - expansion_input: str, - expanded: str, - expansion_passes: tuple[tuple[ExpansionSpan, ...], ...], - pre_expansion_sites: tuple[InterpolationSite, ...], - external_identifiers: tuple[str, ...], - allows_ceremonial_select: bool, - allows_dynamic_output_star: bool, - allows_empty_fixture_star: bool, - request: NativeLintPreparationRequest, - ) -> None: ... - -class ExpandedLintFile(NativeRecord): - """The expanded bodies of one file, up to the first body that could not be expanded.""" - - file_path: Path - bodies: tuple[PendingLintBody, ...] - failure: ProjectCompileError | None - def __init__( - self, - file_path: Path, - bodies: tuple[PendingLintBody, ...], - failure: ProjectCompileError | None, - ) -> None: ... - -class PreparedLintFiles(NativeRecord): - """Lint bodies of every expandable file, and why the skipped files could not expand.""" - - bodies: tuple[LintBody, ...] - unexpandable: dict[Path, str] - def __init__( - self, - bodies: tuple[LintBody, ...], - unexpandable: dict[Path, str], - ) -> None: ... - class LintEdit(NativeRecord): """One deterministic authored-source edit proposed by a lint rule.""" @@ -13144,41 +13013,6 @@ class SqlTestRulesConfig(NativeRecord): pipeline_directory: str = ..., ) -> None: ... -class PreparedSqlLintResult(NativeRecord): - """Reusable expansion context and completed preliminary SQL checks.""" - - result: LintRunResult - context: SqlExpansionContext - def __init__( - self, - result: LintRunResult, - context: SqlExpansionContext, - ) -> None: ... - -class SqlExpansionReuse(NativeRecord): - """Compiler declaration scope for post-assembly SQL checks; assembly still runs its macros.""" - - discovered_inputs: DiscoveredProjectInputs - declaration_scope: DeclarationScopeBuild - def __init__( - self, - discovered_inputs: DiscoveredProjectInputs, - declaration_scope: DeclarationScopeBuild, - ) -> None: ... - -class PreparedSqlLint(NativeRecord): - """Process-local SQL checks started after compiler expansion has completed.""" - - codes: tuple[str, ...] - future: Future[PreparedSqlLintResult] - project_dir: Path | None - def __init__( - self, - codes: tuple[str, ...], - future: Future[PreparedSqlLintResult], - project_dir: Path | None = ..., - ) -> None: ... - class SqlRulesEvaluation(NativeRecord): """SQL rule findings of one run, their cache accounting and elapsed time.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 90b0042362..95eefdc761 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -3,10 +3,11 @@ from __future__ import annotations import json +import sys import time from collections.abc import Callable from pathlib import Path -from threading import Event +from types import SimpleNamespace from typing import Any, cast import pytest @@ -19,10 +20,8 @@ DeclarationScopeBuild, DiscoveredMacroFile, DiscoveredProjectInputs, - LintRunResult, LoadedMacro, SqlLexicalSyntax, - active_lint_stop, rules_bulk_cache_path, ) from sqlbuild._native import run_cli as main @@ -77,7 +76,9 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: _CUSTOM_CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT 1 AS order_id\n' _CUSTOM_CACHE_EDITED_SQL: str = _CUSTOM_CACHE_MODEL_SQL.replace("SELECT 1", "SELECT 2") -_EARLY_LINT_STOP_TIMEOUT_S: float = 30.0 +_EARLY_LINT_SLOW_HOOKS: int = 8 +_EARLY_LINT_SLOW_EXPANSION_S: float = 0.5 +_EARLY_LINT_EXPANSION_DELAYS_S: dict[str, float] = {"orders": 0.0} _EARLY_LINT_FINDINGS: tuple[tuple[str, str, int | None, int | None], ...] = ( ("SQBRSQL005", "models/order_cents.sql", 2, 116), ) @@ -916,14 +917,13 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("cold SQL rules reuse the early file read", 1, "SQBRSQL004")], + [RulesIntegrationTestCase("early file read gives the cached findings", 1, "SQBRSQL004")], ids=lambda case: case.description, ) -def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_file_read( +def test_given_cold_non_model_sql_finding_when_compiling_twice_then_warm_findings_match( test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], - monkeypatch: pytest.MonkeyPatch, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text( f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', @@ -939,25 +939,11 @@ def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_ b"__expected__orders AS (SELECT order_id FROM __ref__orders LIMIT 1)\r\n" b"SELECT 1\r\n" ) - collections: list[frozenset[Path] | None] = [] - original_collect: Callable[..., dict[Path, str]] = native_module.collect_lint_project_files - - def counting_collect( - *, project_dir: Path, selected_paths: frozenset[Path] | None - ) -> dict[Path, str]: - collections.append(selected_paths) - return original_collect(project_dir=project_dir, selected_paths=selected_paths) - - monkeypatch.setattr(native_module, "collect_lint_project_files", counting_collect) - cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) - cold_collections: int = len(collections) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) assert f"tests/unit/test_orders.sql:{test_case.expected_code}" in cold assert warm == cold - assert cold_collections == 0 - assert collections == [None] @pytest.mark.parametrize( @@ -1017,7 +1003,7 @@ def test_given_previous_build_rule_cache_when_compiling_then_lint_overlaps_with_ @pytest.mark.parametrize( "test_case", - [RulePassIntegrationTestCase("graph error stops the unused early lint", 1, ("B002",))], + [RulePassIntegrationTestCase("graph error stops the unused early lint", 1, ("B002", "S024"))], ids=lambda case: case.description, ) def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_awaited( @@ -1033,7 +1019,8 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ models_dir: Path = tmp_path / "models" models_dir.mkdir() (models_dir / "orders.sql").write_text( - 'MODEL (description "Orders");\nSELECT 1 AS order_id\n', encoding="utf-8" + 'MODEL (description "Orders");\nSELECT @slow_total("orders") AS order_id\n', + encoding="utf-8", ) (models_dir / "order_totals.sql").write_text( 'MODEL (description "Order totals");\n' @@ -1041,17 +1028,28 @@ def test_given_graph_error_when_compiling_then_early_lint_is_stopped_instead_of_ 'SELECT order_id, missing_column FROM __ref("orders")\n', encoding="utf-8", ) - lint_started: Event = Event() - stop_observed: list[bool] = [] - original_run_lint: Callable[..., LintRunResult] = native_module.run_lint + macros_dir: Path = tmp_path / "macros" + macros_dir.mkdir() + (tmp_path / "hooks").mkdir() + (macros_dir / "slow_total.py").write_text( + "import orders_lint_probe\n\n\n" + "def slow_total(name: str) -> str:\n" + ' """Record the expansion and take a while."""\n' + " return orders_lint_probe.expanded(name)\n", + encoding="utf-8", + ) + for index in range(_EARLY_LINT_SLOW_HOOKS): + (tmp_path / "hooks" / f"total_{index:02d}.sql").write_text( + f'SELECT @slow_total("total_{index:02d}") AS order_total\n', encoding="utf-8" + ) + expanded: list[str] = [] - def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: - lint_started.set() - stop: Event | None = active_lint_stop() - stop_observed.append(stop is not None and stop.wait(timeout=_EARLY_LINT_STOP_TIMEOUT_S)) - return original_run_lint(**kwargs) + def expand_slowly(name: str) -> str: + expanded.append(name) + time.sleep(_EARLY_LINT_EXPANSION_DELAYS_S.get(name, _EARLY_LINT_SLOW_EXPANSION_S)) + return "1" - monkeypatch.setattr(native_module, "run_lint", stop_awaiting_run_lint) + monkeypatch.setitem(sys.modules, "orders_lint_probe", SimpleNamespace(expanded=expand_slowly)) exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) @@ -1061,8 +1059,7 @@ def stop_awaiting_run_lint(**kwargs: Any) -> LintRunResult: assert tuple(diagnostic["code"] for diagnostic in diagnostics) == ( test_case.expected_diagnostics ) - assert lint_started.is_set() - assert stop_observed == [True] + assert len(set(expanded) - set(_EARLY_LINT_EXPANSION_DELAYS_S)) < _EARLY_LINT_SLOW_HOOKS @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py index 9be0333b4d..d32e7d0435 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/_test_types.py @@ -3,11 +3,11 @@ @dataclass(frozen=True) class SqlRulePublicationTestCase: - """Increasing path counts and the linear conversion budget.""" + """Paths whose lint findings are published to the SQL rule cache within a time limit.""" description: str sizes: tuple[int, ...] - expected_counts: list[tuple[int, int]] + expected_max_seconds: float @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py index 6fb27cb42a..00059e1f38 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/helpers.py @@ -1,7 +1,7 @@ import json import shutil from pathlib import Path -from typing import Any, ClassVar, cast +from typing import Any, cast import pytest @@ -14,16 +14,6 @@ type FindingKey = tuple[str, str, int | None, int | None, str] -class CountingRulePath(Path): - """Count finding path conversions across the real native cache publication.""" - - conversions: ClassVar[int] = 0 - - def as_posix(self) -> str: - CountingRulePath.conversions += 1 - return super().as_posix() - - def write_rules_fixture(*, project_dir: Path) -> None: """Copy the neutral fixture and select every built-in rule.""" diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index 91ab02ce99..033d1dc48d 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -1,7 +1,6 @@ """Bound native SQL rule cache publication work while preserving persisted findings.""" import json -from concurrent.futures import Future from pathlib import Path from time import monotonic from types import SimpleNamespace @@ -14,28 +13,34 @@ from tests.integration.src.sqlbuild.rule_engine.native_rows._test_types import ( SqlRulePublicationTestCase, ) -from tests.integration.src.sqlbuild.rule_engine.native_rows.helpers import CountingRulePath + +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n' @pytest.mark.parametrize( "test_case", - [ - SqlRulePublicationTestCase( - "doubling paths and findings", (4, 8, 16), [(4, 12), (8, 24), (16, 48)] - ) - ], + [SqlRulePublicationTestCase("doubling paths and findings", (4, 8, 16), 30.0)], ids=lambda case: case.description, ) -def test_given_increasing_findings_when_publishing_cache_then_work_is_linear_and_order_preserved( - test_case: SqlRulePublicationTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_given_increasing_findings_when_publishing_cache_then_findings_are_stored_by_path( + test_case: SqlRulePublicationTestCase, tmp_path: Path ) -> None: - """Exercise planning, finding conversion, encoding and disk publication through the binding.""" - monkeypatch.setattr(native, "compiled_relation_keys", lambda project: {}) - counts: list[tuple[int, int]] = [] + """Lint each model natively, then publish its findings to the cache file in finding order.""" started: float = monotonic() for size in test_case.sizes: project_dir: Path = tmp_path / str(size) + (project_dir / "models").mkdir(parents=True) + (project_dir / "sqlbuild_project.toml").write_text(_PROJECT_TOML, encoding="utf-8") paths: list[Path] = [Path(f"models/orders_{index}.sql") for index in range(size)] + for index, path in enumerate(paths[:-1]): + (project_dir / path).write_text( + f'MODEL (description "Orders {index}");\nSELECT 1 AS order_id LIMIT 1\n', + encoding="utf-8", + ) + (project_dir / paths[-1]).write_text( + 'MODEL (description "Orders without a limit");\nSELECT 1 AS order_id\n', + encoding="utf-8", + ) models: list[SimpleNamespace] = [ SimpleNamespace( relative_path=path, @@ -43,11 +48,16 @@ def test_given_increasing_findings_when_publishing_cache_then_work_is_linear_and query_sql="SELECT 1 AS order_id", config=SimpleNamespace(values={}), dynamic_column_contract=None, + relation_key_row=("model", path.stem, "", "", "", False, None, [], []), ) for path in paths ] project: SimpleNamespace = SimpleNamespace( - settings=SimpleNamespace(sql_analysis=True), models=models, sql_expansions={} + settings=SimpleNamespace(sql_analysis=True), + models=models, + seeds=(), + sources=(), + sql_expansions={}, ) plan: SimpleNamespace = SimpleNamespace( project_dir=project_dir, @@ -58,46 +68,26 @@ def test_given_increasing_findings_when_publishing_cache_then_work_is_linear_and thresholds={}, allow_exceptions=True, cache=SimpleNamespace(enabled=True) ), ) - violations: list[SimpleNamespace] = [ - SimpleNamespace( - file_path=CountingRulePath(project_dir / paths[index % (size - 1)]), - code="SQBRSQL004", - line=index + 1, - column=1, - message=f"Finding {index}", - remediation="Use an explicit projection.", - fix=None, - ) - for index in range(size) - ] - future: Future[Any] = Future() - future.set_result(SimpleNamespace(result=SimpleNamespace(violations=violations))) - CountingRulePath.conversions = 0 result: SqlRulesEvaluation = native.run_planned_sql_rules( plan=cast(Any, plan), graph=cast(Any, SimpleNamespace(project=project)), dialect="duckdb", discovered_inputs=cast(Any, None), - prepared_sql=cast(Any, SimpleNamespace(codes=("SQBRSQL004",), future=future)), - expansion_reuse=None, + early_lint=None, ) - conversions: int = CountingRulePath.conversions assert result.cache_misses == size - assert len(result.findings) == size + assert len(result.findings) == size - 1 stored: dict[str, tuple[str, str]] = native.load_sql_rule_cache( path=native.rules_bulk_cache_path(project_dir=project_dir, file_name="sql.json") ) assert set(stored) == {path.as_posix() for path in paths} - expected: list[list[Finding]] = [[] for _ in paths] - for position, finding in enumerate(result.findings): - expected[position % (size - 1)].append(finding) - for path, own in zip(paths, expected, strict=True): - assert stored[path.as_posix()][1] == native.encode_sql_rule_findings(findings=own) + by_path: dict[str, list[Finding]] = {path.as_posix(): [] for path in paths} + for finding in result.findings: + by_path[finding.path.as_posix()].append(finding) + for path, own in by_path.items(): + assert stored[path][1] == native.encode_sql_rule_findings(findings=own) assert json.loads(stored[paths[-1].as_posix()][1]) == [] - # One conversion each for grouping, encoding, and duplicate-literal hints. - counts.append((size, conversions)) - assert counts == test_case.expected_counts - assert monotonic() - started < 30 + assert monotonic() - started < test_case.expected_max_seconds if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index 43b1eff0ca..51fcfc51c6 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -219,7 +219,7 @@ class ManyViolationLocationTestCase: @dataclass(frozen=True) class BatchedPreparationTestCase: - """A project whose lint is compared between batched and per-body native preparation.""" + """A project whose bodies of every kind are prepared natively in one batch.""" description: str files: dict[str, str] @@ -227,13 +227,20 @@ class BatchedPreparationTestCase: @dataclass(frozen=True) -class BatchedPreparationFailureTestCase: - """A native preparation failure and an unexpandable file, in a given file order.""" +class UnexpandableFailureTestCase: + """Unexpandable files linted without skipping, and the file whose failure is raised.""" description: str - unexpandable_file: str - skip_unexpandable: bool - expected_error: type[Exception] | None + expected_error_file: str + + +@dataclass(frozen=True) +class UnexpandableSkipTestCase: + """Unexpandable files linted with skipping, and the files reported as skipped.""" + + description: str + expected_unexpandable: tuple[str, ...] + expected_codes: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index fedae56d34..f54c0e2632 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -4,7 +4,6 @@ from dataclasses import astuple from pathlib import Path -from typing import TYPE_CHECKING from sqlbuild._native import ( HeaderSpan, @@ -19,9 +18,6 @@ sql_body_ranges, ) -if TYPE_CHECKING: - from sqlbuild._native import NativeLintPreparationRequest, NativePreparedSql - def lint_bodies_for(*, file_path: Path, contents: str) -> tuple[LintBody, ...]: """Build unexpanded lint bodies for every SQL body in the contents.""" @@ -93,20 +89,6 @@ def write_lint_project(*, root: Path, files: dict[str, str]) -> None: path.write_text(contents, encoding="utf-8") -def prepare_nothing_in_batch( - requests: list[NativeLintPreparationRequest], -) -> list[tuple[bool, NativePreparedSql | None]]: - """Report every body as unprepared so each one takes the per-body native call.""" - - return [(False, None) for _request in requests] - - -def refuse_preparation(request: NativeLintPreparationRequest) -> NativePreparedSql | None: - """Fail the per-body native preparation of any body.""" - - raise ValueError(f"cannot prepare {request['dialect']} body") - - def write_orders_lint_project(*, project_dir: Path, model_bytes: bytes) -> Path: """Write a DuckDB project with one model and one SQL test; return the model path.""" diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py index 754ff94dc8..5912f01143 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py @@ -1,4 +1,4 @@ -"""Lint bodies are prepared natively in one batch with per-body results and failure order.""" +"""Lint bodies are prepared natively in one batch; unexpandable files fail or skip in file order.""" from __future__ import annotations @@ -6,17 +6,13 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import LintConfig, LintRunResult, ProjectCompileError, run_lint from tests.unit.src.sqlbuild.lint._helpers._test_types import ( - BatchedPreparationFailureTestCase, BatchedPreparationTestCase, + UnexpandableFailureTestCase, + UnexpandableSkipTestCase, ) -from tests.unit.src.sqlbuild.lint._helpers.helpers import ( - prepare_nothing_in_batch, - refuse_preparation, - write_lint_project, -) +from tests.unit.src.sqlbuild.lint._helpers.helpers import write_lint_project _ORDERS_MODEL: str = ( 'MODEL (description "Orders");\n' @@ -30,6 +26,13 @@ "__expected__orders AS (SELECT 1 AS customer_id)\nSELECT 1\n" ) _NOT_NULL_AUDIT: str = "AUDIT ();\nSELECT 1 FROM @relation WHERE @column IS NULL\n" +_BROKEN_MODEL: str = 'MODEL (description "Broken");\nSELECT @unknown()\n' +_UNEXPANDABLE_FILES: dict[str, str] = { + "models/customers.sql": _CUSTOMERS_MODEL, + "models/orders.sql": _ORDERS_MODEL, + "models/aa_broken.sql": _BROKEN_MODEL, + "models/zz_broken.sql": _BROKEN_MODEL, +} @pytest.mark.parametrize( @@ -48,66 +51,65 @@ ], ids=lambda case: case.description, ) -def test_given_project_when_linting_with_batched_preparation_then_result_matches_per_body( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: BatchedPreparationTestCase +def test_given_project_when_linting_then_bodies_of_every_kind_are_prepared_in_one_batch( + tmp_path: Path, test_case: BatchedPreparationTestCase ) -> None: write_lint_project(root=tmp_path, files=test_case.files) config: LintConfig = LintConfig(dialect="duckdb") - batched: LintRunResult = run_lint(project_dir=tmp_path, config=config) - monkeypatch.setattr(native_module, "prepare_lint_sql_batch", prepare_nothing_in_batch) - per_body: LintRunResult = run_lint(project_dir=tmp_path, config=config) + first: LintRunResult = run_lint(project_dir=tmp_path, config=config) + second: LintRunResult = run_lint(project_dir=tmp_path, config=config) - assert batched == per_body - assert tuple(violation.code for violation in batched.violations) == test_case.expected_codes + assert first == second + assert tuple(violation.code for violation in first.violations) == test_case.expected_codes @pytest.mark.parametrize( "test_case", [ - BatchedPreparationFailureTestCase( - description="native failure before a later unexpandable file", - unexpandable_file="models/zz_broken.sql", - skip_unexpandable=False, - expected_error=ValueError, - ), - BatchedPreparationFailureTestCase( - description="unexpandable file before a later native failure", - unexpandable_file="models/aa_broken.sql", - skip_unexpandable=False, - expected_error=ProjectCompileError, - ), - BatchedPreparationFailureTestCase( - description="skipped unexpandable file before a later native failure", - unexpandable_file="models/aa_broken.sql", - skip_unexpandable=True, - expected_error=ValueError, - ), + UnexpandableFailureTestCase( + description="first unexpandable file in path order is raised", + expected_error_file="models/aa_broken.sql", + ) ], ids=lambda case: case.description, ) -def test_given_native_and_expansion_failures_when_linting_then_first_in_file_order_is_raised( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: BatchedPreparationFailureTestCase +def test_given_unexpandable_files_when_linting_then_first_in_file_order_is_raised( + tmp_path: Path, test_case: UnexpandableFailureTestCase ) -> None: - write_lint_project( - root=tmp_path, - files={ - "models/customers.sql": _CUSTOMERS_MODEL, - "models/orders.sql": _ORDERS_MODEL, - test_case.unexpandable_file: 'MODEL (description "Broken");\nSELECT @unknown()\n', - }, - ) - monkeypatch.setattr(native_module, "prepare_lint_sql_batch", prepare_nothing_in_batch) - monkeypatch.setattr(native_module, "prepare_lint_sql", refuse_preparation) - - with pytest.raises(Exception) as raised: - _ = run_lint( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - skip_unexpandable=test_case.skip_unexpandable, + write_lint_project(root=tmp_path, files=_UNEXPANDABLE_FILES) + + with pytest.raises(ProjectCompileError) as raised: + _ = run_lint(project_dir=tmp_path, config=LintConfig(dialect="duckdb")) + + assert str(raised.value).startswith(str(tmp_path / test_case.expected_error_file)) + + +@pytest.mark.parametrize( + "test_case", + [ + UnexpandableSkipTestCase( + description="skipped unexpandable files are reported and the rest is linted", + expected_unexpandable=("models/aa_broken.sql", "models/zz_broken.sql"), + expected_codes=("SQBRSQL001",), ) + ], + ids=lambda case: case.description, +) +def test_given_unexpandable_files_when_linting_with_skipping_then_they_are_reported( + tmp_path: Path, test_case: UnexpandableSkipTestCase +) -> None: + write_lint_project(root=tmp_path, files=_UNEXPANDABLE_FILES) + + result: LintRunResult = run_lint( + project_dir=tmp_path, config=LintConfig(dialect="duckdb"), skip_unexpandable=True + ) - assert type(raised.value) is test_case.expected_error + assert ( + tuple(path.relative_to(tmp_path).as_posix() for path in result.unexpandable) + == test_case.expected_unexpandable + ) + assert tuple(violation.code for violation in result.violations) == test_case.expected_codes if __name__ == "__main__": From 331617ca649144a41aa95dc63e10459a324b8dcc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:15:00 +0100 Subject: [PATCH 07/12] refactor(planner): give directly called planner helpers typed requests Rust callers of model entry assembly, SQL resolution, cursor bounds and inputs, change detection, relation contexts, source freshness, plan entry results, migration planning, audit planning, SQL-test planning and scenario relation plans now pass typed request structs instead of keyword dicts. Python-facing wrappers parse keywords into the same structs; wrappers no Python code calls are removed. --- .../_helpers/planner_changes/detect.rs | 36 ++- .../_helpers/planner_entries/clone_plans.rs | 49 ++-- .../_helpers/planner_entries/command_plans.rs | 35 +-- .../planner_entries/execution_plan.rs | 91 ++---- .../bindings/_helpers/planner_entries/mod.rs | 3 +- .../_helpers/planner_entries/model_entries.rs | 276 +++++++----------- .../_helpers/planner_entries/models.rs | 31 ++ .../_helpers/planner_entries/plan_output.rs | 32 +- .../_helpers/planner_entries/relations.rs | 38 ++- .../planner_migrations/column_candidates.rs | 13 +- .../planner_migrations/column_overlay.rs | 21 +- .../planner_migrations/column_planning.rs | 45 ++- .../planner_migrations/compatibility.rs | 55 ++-- .../_helpers/planner_migrations/mod.rs | 2 +- .../planner_migrations/model_overlay.rs | 15 +- .../planner_migrations/model_planning.rs | 56 ++-- .../_helpers/planner_migrations/models.rs | 18 ++ .../planner_migrations/old_name_entries.rs | 10 +- .../planner_migrations/old_name_planning.rs | 58 ++-- .../planner_migrations/registration.rs | 1 - .../planner_phases/execution_changes.rs | 107 +++---- .../_helpers/planner_phases/identities.rs | 24 +- .../_helpers/planner_phases/registration.rs | 6 +- .../planner_phases/warehouse_state.rs | 109 +++---- .../planner_resolution/cursor_model.rs | 56 +--- .../planner_resolution/cursor_overrides.rs | 40 +-- .../_helpers/planner_resolution/mod.rs | 3 +- .../_helpers/planner_resolution/models.rs | 24 ++ .../planner_resolution/orchestration.rs | 190 ++++++------ .../_helpers/planner_resolution/refs.rs | 43 ++- .../planner_resolution/registration.rs | 1 - .../_helpers/planner_resolution/sources.rs | 57 +++- .../_helpers/planner_resolution/values.rs | 45 +-- .../bindings/_helpers/planner_scenario/cli.rs | 24 +- .../_helpers/planner_scenario/models.rs | 11 + .../_helpers/planner_scenario/registration.rs | 2 +- .../_helpers/planner_scenario/relations.rs | 42 ++- .../planner_strategy/audit_entries.rs | 79 +++-- .../planner_strategy/cursor_inputs.rs | 121 +++++--- .../_helpers/planner_strategy/cursor_types.rs | 43 ++- .../planner_strategy/model_entries.rs | 221 +++++++------- .../_helpers/planner_strategy/models.rs | 52 +++- .../_helpers/planner_strategy/registration.rs | 13 +- .../_helpers/planner_strategy/strategy.rs | 61 +++- .../planner_warehouse/registration.rs | 2 +- .../planner_warehouse/source_freshness.rs | 24 +- .../src/bindings/_helpers/records/ids.rs | 7 - .../bindings/_helpers/sql_test_glue/mod.rs | 1 + .../bindings/_helpers/sql_test_glue/models.rs | 20 ++ .../_helpers/sql_test_glue/plan_assembly.rs | 74 ++--- .../_helpers/sql_test_glue/plan_driver.rs | 74 +++-- .../_helpers/sql_test_glue/plan_fixtures.rs | 14 +- .../_helpers/sql_test_glue/registration.rs | 6 +- src/sqlbuild/_native.pyi | 85 ------ 54 files changed, 1292 insertions(+), 1274 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index db69992068..09b0ad43d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -60,9 +60,9 @@ pub(crate) struct ModelPass<'py> { } /// Selected model and function changes. -#[pyfunction] +#[pyfunction(name = "detect_changes")] #[pyo3(signature = (**options))] -pub(crate) fn detect_changes<'py>( +fn detect_changes_of<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -75,21 +75,29 @@ pub(crate) fn detect_changes<'py>( &["expected_version_hashes", "expected_metadata_jsons"], ), )?; - let (project, scope, snapshot) = ( - arguments.value("project")?, - arguments.value("scope")?, - arguments.value("snapshot")?, - ); - let (project, scope, snapshot) = (&project, &scope, &snapshot); - let full_refresh: bool = arguments.flag("full_refresh")?; - let models: Bound<'py, PyDict> = scope_model_changes( - (project, scope, snapshot), - None, + detect_changes( + ( + &arguments.value("project")?, + &arguments.value("scope")?, + &arguments.value("snapshot")?, + ), + arguments.flag("full_refresh")?, ( arguments.value("expected_version_hashes")?, arguments.value("expected_metadata_jsons")?, ), - )?; + ) +} + +/// Selected model and function changes against expected version hashes and metadata JSON. +pub(crate) fn detect_changes<'py>( + (project, scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + full_refresh: bool, + expected: (Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = project.py(); + let models: Bound<'py, PyDict> = + scope_model_changes((project, scope, snapshot), None, expected)?; let functions: Bound<'py, PyDict> = PyDict::new(py); let result_class: Bound<'py, PyAny> = registry::class(py, RecordId::FunctionChangeResult)?; let selected: Bound<'py, PyAny> = scope.getattr("selected_keys")?; @@ -784,7 +792,7 @@ impl<'py> ChangeResult<'py> { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(detect_changes, module)?)?; + module.add_function(wrap_pyfunction!(detect_changes_of, module)?)?; module.add_function(wrap_pyfunction!(detect_model_changes_in_scope, module)?)?; module.add_function(wrap_pyfunction!(detect_model_changes, module)?)?; module.add_function(wrap_pyfunction!(schema_changes, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs index d12fd6e241..99d1c06bd9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs @@ -5,6 +5,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::values::{ native, planner_model, planner_type, string, }; +use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; +use crate::bindings::_helpers::planner_resolution::orchestration::{Resolution, resolve_model_sql}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -208,32 +210,27 @@ fn clone_model_entries<'py>( ), ], )?; - native( - py, - NativeAttr::ResolveModelSql, - &[ - ("adapter", adapter.clone()), - ("model", model.clone()), - ("snapshot", native(py, RecordId::WarehouseSnapshot, &[])?), - ("context", context), - ( - "backfill", - native( - py, - RecordId::BackfillResult, - &[( - "action", - planner_type(py, RecordId::BackfillAction, "FORWARD_ONLY")?, - )], - )?, - ), - ("full_refresh", PyBool::new(py, false).to_owned().into_any()), - ( - "cursor_overrides", - native(py, RecordId::CursorOverridePair, &[])?, - ), - ], - )? + let sql = resolve_model_sql( + &Resolution::of_context(adapter.clone(), &context, py.None().into_bound(py))?, + &context, + &ModelSqlRequest { + model: model.clone(), + snapshot: native(py, RecordId::WarehouseSnapshot, &[])?, + backfill: native( + py, + RecordId::BackfillResult, + &[( + "action", + planner_type(py, RecordId::BackfillAction, "FORWARD_ONLY")?, + )], + )?, + full_refresh: false, + cursor_overrides: (py.None().into_bound(py), py.None().into_bound(py)), + replaces_relation: false, + full_rebuild_cause: None, + }, + )?; + string(py, &sql) } else { string(py, "") }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs index 3edf007fad..1930333df0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::plan_output::{ function_entry, selected_test_entries, test_selected, }; @@ -107,39 +108,31 @@ fn resolve_static_relation_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { - let options = kwargs.unwrap_or_else(|| PyDict::new(py)); - KeywordArguments::read( - py, - "resolve_static_relation_context", - Some(options.clone()), - ( - &["project", "adapter", "scope"], - &[ - "deferral", - "project_config", - "local_config", - "relation_keys", - ], - ), - )?; - options.set_item("connection", none(py))?; - options.set_item("known_source_columns", PyDict::new(py))?; let args = KeywordArguments::read( py, "resolve_static_relation_context", - Some(options), + kwargs, ( - &["project", "adapter", "scope", "connection"], + &["project", "adapter", "scope"], &[ "deferral", "project_config", "local_config", "relation_keys", - "known_source_columns", ], ), )?; - planner_relations_context(&args) + planner_relations_context(RelationsRequest { + project: args.value("project")?, + adapter: args.value("adapter")?, + connection: none(py), + scope: args.value("scope")?, + deferral: args.value("deferral")?, + project_config: args.value("project_config")?, + local_config: args.value("local_config")?, + known_source_columns: PyDict::new(py).into_any(), + relation_keys: args.value("relation_keys")?, + }) } /// Project selected audits from canonical static state. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index a47e075b37..945c6d688d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -1,7 +1,7 @@ //! Top-level planner orchestration producing an execution plan. use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, keywords, within, + KeywordArguments, call_with, imported, within, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -27,7 +27,7 @@ use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor use crate::bindings::_helpers::planner_warehouse::source_freshness::build_planner_source_freshness_result; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyTuple}; +use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use std::time::Instant; @@ -55,16 +55,7 @@ fn scopes_and_cursor_overrides<'py>( } } let overrides = args.value("overrides")?; - let typed = type_cursor_overrides( - py, - Some(keywords( - py, - &[ - ("cursor_overrides", overrides.getattr("cursor_overrides")?), - ("selected_models", PyTuple::new(py, models)?.into_any()), - ], - )?), - )?; + let typed = type_cursor_overrides(&overrides.getattr("cursor_overrides")?, &models)?; Ok(( scopes, replaced(&overrides, &[("cursor_overrides", typed)])?, @@ -81,24 +72,12 @@ fn change_results<'py>( let py = project.py(); let versions = identities.getattr("version_identities")?; let changes = detect_changes( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("scope", scopes.getattr("inspection_scope")?), - ("snapshot", snapshot.clone()), - ("full_refresh", overrides.getattr("full_refresh")?), - ( - "expected_version_hashes", - versions.getattr("model_version_hashes")?, - ), - ( - "expected_metadata_jsons", - versions.getattr("model_metadata_jsons")?, - ), - ], - )?), + (project, &scopes.getattr("inspection_scope")?, snapshot), + overrides.getattr("full_refresh")?.is_truthy()?, + ( + versions.getattr("model_version_hashes")?, + versions.getattr("model_metadata_jsons")?, + ), )?; if !policies.getattr("selection_diagnostics")?.is_truthy()? { let empty = native( @@ -111,19 +90,7 @@ fn change_results<'py>( )?; return Ok((changes, empty)); } - let stale = detect_stale_warning_changes( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("scopes", scopes.clone()), - ("snapshot", snapshot.clone()), - ("identities", identities.clone()), - ("execution_changes", changes.clone()), - ], - )?), - )?; + let stale = detect_stale_warning_changes((project, scopes), snapshot, (identities, &changes))?; Ok((changes, stale)) } @@ -156,40 +123,18 @@ fn planned_output<'py>( let inspection_scope = scopes.getattr("inspection_scope")?; let resolved_actions = resolve_model_actions(&inspection_scope, &changes)?; let source_freshness = build_planner_source_freshness_result( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("adapter", args.value("adapter")?), - ("connection", args.value("connection")?), - ("scope", inspection_scope), - ("relations", warehouse.getattr("inspection_relations")?), - ( - "freshness_state_schemas", - snapshot.getattr("source_freshness_state_schemas")?, - ), - ], - )?), + (&project, &inspection_scope), + (&args.value("adapter")?, &args.value("connection")?), + &warehouse.getattr("inspection_relations")?, + &snapshot.getattr("source_freshness_state_schemas")?, )?; let pruning = prune_planner_execution_scope(scopes, &resolved_actions)?; let reconciliation = reconcile_execution_changes(&warehouse, &identities, &pruning, &changes)?; let entries = build_planner_entry_results( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("warehouse", warehouse.clone()), - ("identities", identities.clone()), - ("overrides", overrides.clone()), - ("policies", policies.clone()), - ("deferral", deferral), - ("pruning", pruning.clone()), - ("reconciliation", reconciliation.clone()), - ("source_freshness", source_freshness.clone()), - ], - )?), + (runtime, &warehouse), + (overrides, &deferral), + (&pruning, &reconciliation), + &source_freshness, )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); let output = base_plan_output( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs index 8ae3efa1f4..519d1ac918 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/mod.rs @@ -4,7 +4,8 @@ mod clone_plans; mod command_plans; mod display_plan; mod execution_plan; -mod model_entries; +pub(crate) mod model_entries; +pub(crate) mod models; pub(crate) mod plan_output; pub(crate) mod registration; pub(crate) mod relations; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 2b7b859fa8..40bda792c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -1,13 +1,12 @@ //! Per-model plan entries from resolved change results and warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, imported, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::planner_changes::actions::replaces_incremental_relation; +use crate::bindings::_helpers::planner_entries::models::PlanEntriesRequest; use crate::bindings::_helpers::planner_entries::values::{ - COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, boolean, error_of, native, none, planner_type, - replaced, string, + COMPILE_TYPES, MICROBATCH_CONCURRENCY_MESSAGE, error_of, native, none, planner_type, replaced, + string, }; use crate::bindings::_helpers::planner_phases::full_refresh::{ check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, @@ -17,8 +16,9 @@ use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ model_cursor_overrides, plan_cursor_bounds, }; use crate::bindings::_helpers::planner_resolution::cursor_warnings::with_cursor_safety_warnings; +use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest as SqlRequest; use crate::bindings::_helpers::planner_resolution::orchestration::{ - resolve_model_hook_entries, resolve_model_sql, + Resolution, model_hook_entries, resolve_model_sql, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ build_cursor_input_relations, reject_unexposed_cursor_input_columns, @@ -26,6 +26,9 @@ use crate::bindings::_helpers::planner_strategy::cursor_inputs::{ use crate::bindings::_helpers::planner_strategy::lineage::reject_capped_watermark_inputs; use crate::bindings::_helpers::planner_strategy::microbatch_limits::apply_configured_microbatch_limits; use crate::bindings::_helpers::planner_strategy::model_entries::assemble_model_plan_entry; +use crate::bindings::_helpers::planner_strategy::models::{ + CursorInputLookups, ModelEntryDecisions, +}; use crate::bindings::_helpers::planner_strategy::strategy::{ materialization_of, resolve_model_plan_action, }; @@ -41,26 +44,28 @@ const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; /// The `ModelPlanContext` fields every model's planning reads, read once per plan. struct ContextReads<'py> { context: Bound<'py, PyAny>, - models_by_name: Bound<'py, PyAny>, - functions_by_name: Bound<'py, PyAny>, - source_map: Bound<'py, PyAny>, - source_warehouse_columns: Bound<'py, PyAny>, - model_locations: Bound<'py, PyAny>, - seed_locations: Bound<'py, PyAny>, - runtime_cursor_producer_names: Bound<'py, PyAny>, + lookups: CursorInputLookups<'py>, + resolution: Resolution<'py>, invocation_time: Bound<'py, PyAny>, } impl<'py> ContextReads<'py> { - fn of(context: Bound<'py, PyAny>) -> PyResult { + fn of( + context: Bound<'py, PyAny>, + adapter: Bound<'py, PyAny>, + external_resolver: Bound<'py, PyAny>, + ) -> PyResult { Ok(Self { - models_by_name: context.getattr("models_by_name")?, - functions_by_name: context.getattr("functions_by_name")?, - source_map: context.getattr("source_map")?, - source_warehouse_columns: context.getattr("source_warehouse_columns")?, - model_locations: context.getattr("model_locations")?, - seed_locations: context.getattr("seed_locations")?, - runtime_cursor_producer_names: context.getattr("runtime_cursor_producer_names")?, + lookups: CursorInputLookups { + models_by_name: context.getattr("models_by_name")?, + functions_by_name: context.getattr("functions_by_name")?, + source_map: context.getattr("source_map")?, + source_warehouse_columns: context.getattr("source_warehouse_columns")?, + model_locations: context.getattr("model_locations")?, + seed_locations: context.getattr("seed_locations")?, + runtime_cursor_producer_names: context.getattr("runtime_cursor_producer_names")?, + }, + resolution: Resolution::of_context(adapter, &context, external_resolver)?, invocation_time: context.getattr("invocation_time")?, context, }) @@ -71,13 +76,11 @@ impl<'py> ContextReads<'py> { struct ModelPlanRequest<'a, 'py> { model: Bound<'py, PyAny>, snapshot: Bound<'py, PyAny>, - adapter: Bound<'py, PyAny>, context: &'a ContextReads<'py>, - sql_analysis_enabled: Bound<'py, PyAny>, + sql_analysis_enabled: bool, full_refresh: bool, overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), change_result: Bound<'py, PyAny>, - external_resolver: Bound<'py, PyAny>, protect_full_refresh: bool, } @@ -88,8 +91,8 @@ fn plan_model_from_change<'py>( let model = &request.model; let reads = request.context; let context = &reads.context; + let adapter = &reads.resolution.adapter; let py = model.py(); - let full_refresh = boolean(py, request.full_refresh)?; let backfill = if request.full_refresh { native( py, @@ -103,20 +106,7 @@ fn plan_model_from_change<'py>( request.change_result.getattr("backfill")? }; let cursor_column = config_str(model, "cursor")?; - reject_unexposed_cursor_input_columns(Some(&keywords( - py, - &[ - ("model", model.clone()), - ("cursor_column", cursor_column.clone()), - ("models_by_name", reads.models_by_name.clone()), - ("functions_by_name", reads.functions_by_name.clone()), - ("source_map", reads.source_map.clone()), - ( - "source_warehouse_columns", - reads.source_warehouse_columns.clone(), - ), - ], - )?))?; + reject_unexposed_cursor_input_columns(model, &cursor_column, &reads.lookups)?; let resolved = resolve_model_plan_action(model, &request.change_result, request.full_refresh)?; let (action, reason) = (resolved.get_item(0)?, resolved.get_item(1)?); let materialization_type = materialization_of(model)?; @@ -128,69 +118,32 @@ fn plan_model_from_change<'py>( if request.protect_full_refresh && !request.full_refresh { check_full_rebuild_allowed(model, &request.change_result, &cause)?; } - string(py, &cause) + Some(cause) } else { - none(py) + None }; - let pair = native( - py, - RecordId::CursorOverridePair, - &[ - ("start_cursor_override", request.overrides.0.clone()), - ("end_cursor_override", request.overrides.1.clone()), - ], - )?; let resolved_sql = resolve_model_sql( - py, - Some(keywords( - py, - &[ - ("adapter", request.adapter.clone()), - ("model", model.clone()), - ("snapshot", request.snapshot.clone()), - ("context", context.clone()), - ("backfill", backfill.clone()), - ("full_refresh", full_refresh.clone()), - ("cursor_overrides", pair), - ( - "external_sql_reference_resolver", - request.external_resolver.clone(), - ), - ("replaces_relation", boolean(py, replaces_relation)?), - ("full_rebuild_cause", full_rebuild_cause), - ], - )?), - )?; - let hooks = resolve_model_hook_entries( - model, - request.adapter.clone(), - context.clone(), - request.external_resolver.clone(), + &reads.resolution, + context, + &SqlRequest { + model: model.clone(), + snapshot: request.snapshot.clone(), + backfill: backfill.clone(), + full_refresh: request.full_refresh, + cursor_overrides: request.overrides.clone(), + replaces_relation, + full_rebuild_cause, + }, )?; + let hooks = model_hook_entries(model, &reads.resolution)?; let (future_config, start_config) = cursor_safety_configs(model, context)?; reject_capped_watermark_inputs( model, - reads.models_by_name.clone(), - Some(reads.functions_by_name.clone()), + reads.lookups.models_by_name.clone(), + Some(reads.lookups.functions_by_name.clone()), )?; - let cursor_input_relations = build_cursor_input_relations(Some(&keywords( - py, - &[ - ("model", model.clone()), - ("adapter", request.adapter.clone()), - ("model_locations", reads.model_locations.clone()), - ("models_by_name", reads.models_by_name.clone()), - ("functions_by_name", reads.functions_by_name.clone()), - ("seed_locations", reads.seed_locations.clone()), - ("source_map", reads.source_map.clone()), - ("cursor_column", cursor_column.clone()), - ( - "runtime_cursor_producer_names", - reads.runtime_cursor_producer_names.clone(), - ), - ], - )?))? - .into_any(); + let cursor_input_relations = + build_cursor_input_relations((model, adapter), &cursor_column, &reads.lookups)?.into_any(); let mut any_runtime_owned = false; for relation in cursor_input_relations.try_iter()? { if relation?.getattr("is_runtime_owned")?.is_truthy()? { @@ -214,49 +167,40 @@ fn plan_model_from_change<'py>( let cursor_bounds = plan_cursor_bounds(model, &window)?; let (empty_inputs, waiting_on_empty_inputs) = empty_cursor_inputs(model, &window)?; let range = microbatch_range(model, &window)?; - let assembled = assemble_model_plan_entry(Some(&keywords( - py, - &[ - ("model", model.clone()), - ("adapter", request.adapter.clone()), - ("snapshot", request.snapshot.clone()), - ("change_result", request.change_result.clone()), - ("action", action), - ("reason", reason), - ("materialization_type", materialization_type), - ("resolved_sql", resolved_sql), - ("sql_analysis_enabled", request.sql_analysis_enabled.clone()), - ("pre_hooks", hooks.get_item(0)?), - ("post_hooks", hooks.get_item(1)?), - ("cursor_column", cursor_column), - ("cursor_input_relations", cursor_input_relations), - ("runtime_owned_cursor_bounds", boolean(py, runtime_owned)?), - ("cursor_bounds", cursor_bounds.clone()), - ("microbatch_range", range.clone()), - ("empty_cursor_inputs", empty_inputs), - ( - "waiting_on_empty_inputs", - boolean(py, waiting_on_empty_inputs)?, - ), - ("backfill", backfill), - ("start_cursor_override", request.overrides.0.clone()), - ("end_cursor_override", request.overrides.1.clone()), - ("future_cursor_config", future_config), - ("start_cursor_config", start_config), - ("invocation_time", reads.invocation_time.clone()), - ], - )?))?; + let (entry, entry_warnings) = assemble_model_plan_entry(&ModelEntryDecisions { + model: model.clone(), + adapter: adapter.clone(), + snapshot: request.snapshot.clone(), + change_result: request.change_result.clone(), + action, + reason, + materialization_type, + resolved_sql: string(py, &resolved_sql), + sql_analysis_enabled: request.sql_analysis_enabled, + hooks: (hooks.get_item(0)?, hooks.get_item(1)?), + cursor_column, + cursor_input_relations, + runtime_owned_cursor_bounds: runtime_owned, + cursor_bounds: cursor_bounds.clone(), + microbatch_range: range.clone(), + empty_cursor_inputs: empty_inputs, + waiting_on_empty_inputs, + backfill, + cursor_overrides: request.overrides.clone(), + cursor_configs: (future_config, start_config), + invocation_time: reads.invocation_time.clone(), + })?; let bounds = if range.is_truthy()? { range } else { cursor_bounds }; let warnings = with_cursor_safety_warnings( - &assembled.get_item(1)?, + &entry_warnings.into_any(), &model_reads::name_object(model)?, &bounds, )?; - Ok((assembled.get_item(0)?, warnings)) + Ok((entry, warnings)) } /// The binding-diagnostic failure that stops planning, if any diagnostic is an error. @@ -424,17 +368,19 @@ fn check_microbatch_concurrency<'py>( } /// Build model plan entries from snapshot and per-model resolved actions. -fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult> { - let project = args.value("project")?; +pub(crate) fn plan_entries<'py>(request: PlanEntriesRequest<'py>) -> PyResult> { + let PlanEntriesRequest { + project, + adapter, + scope, + snapshot, + relations, + resolved_models: models, + cursor_overrides, + full_refresh, + build_inputs: inputs, + } = request; let py = project.py(); - let adapter = args.value("adapter")?; - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; - let relations = args.value("relations")?; - let models = args.value("resolved_actions")?.getattr("models")?; - let cursor_overrides = args.value("cursor_overrides")?; - let full_refresh = args.value("full_refresh")?; - let inputs = args.value("build_inputs")?; let inputs = if inputs.is_none() { native(py, RecordId::PlanEntryBuildInputs, &[])? } else { @@ -494,9 +440,12 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult ("invocation_time", inputs.getattr("invocation_time")?), ], )?; - let reads = ContextReads::of(context)?; - let sql_analysis_enabled = settings.getattr("sql_analysis")?; - let external_resolver = project.getattr("external_sql_reference_resolver")?; + let reads = ContextReads::of( + context, + adapter.clone(), + project.getattr("external_sql_reference_resolver")?, + )?; + let sql_analysis_enabled = settings.getattr("sql_analysis")?.is_truthy()?; let skip = planner_type(py, RecordId::PlanAction, "SKIP")?; let microbatch = registry::class(py, RecordId::IncrementalMode)?.getattr("MICROBATCH")?; let mut entries: Vec> = Vec::new(); @@ -517,17 +466,15 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult &cursor_overrides, (start_override.clone(), end_override.clone()), )?; - let effective_full_refresh = resolve_model_full_refresh(&model, full_refresh.is_truthy()?)?; + let effective_full_refresh = resolve_model_full_refresh(&model, full_refresh)?; let (mut entry, entry_warnings) = plan_model_from_change(&ModelPlanRequest { model: model.clone(), snapshot: snapshot.clone(), - adapter: adapter.clone(), context: &reads, - sql_analysis_enabled: sql_analysis_enabled.clone(), + sql_analysis_enabled, full_refresh: effective_full_refresh, overrides, change_result: resolved.getattr("change")?, - external_resolver: external_resolver.clone(), protect_full_refresh: !freshness_blocked.contains(&name)? && !external_blocked.contains(&name)?, })?; @@ -590,34 +537,6 @@ fn plan_entries<'py>(args: &KeywordArguments<'py>) -> PyResult ) } -/// Build model plan entries from snapshot and per-model resolved actions. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -fn build_plan_entries<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "build_plan_entries", - kwargs, - ( - &[ - "project", - "adapter", - "scope", - "snapshot", - "relations", - "resolved_actions", - "cursor_overrides", - "full_refresh", - ], - &["build_inputs"], - ), - )?; - plan_entries(&args) -} - /// Detect changes and build a plan entry and warnings for a single model. #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -663,16 +582,18 @@ fn plan_model<'py>( plan_model_from_change(&ModelPlanRequest { model, snapshot, - adapter: args.value("adapter")?, - context: &ContextReads::of(args.value("context")?)?, - sql_analysis_enabled: args.value("sql_analysis_enabled")?, + context: &ContextReads::of( + args.value("context")?, + args.value("adapter")?, + args.value("external_sql_reference_resolver")?, + )?, + sql_analysis_enabled: args.flag("sql_analysis_enabled")?, full_refresh: args.flag("full_refresh")?, overrides: ( overrides.getattr("start_cursor_override")?, overrides.getattr("end_cursor_override")?, ), change_result, - external_resolver: args.value("external_sql_reference_resolver")?, protect_full_refresh: true, }) } @@ -710,7 +631,6 @@ pub(crate) fn scope_overlaps( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_plan_entries, module)?)?; module.add_function(wrap_pyfunction!(plan_model, module)?)?; module.add_function(wrap_pyfunction!(build_model_materializations, module)?)?; module.add_function(wrap_pyfunction!(scope_overlaps, module)?) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs new file mode 100644 index 0000000000..7614d1b835 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -0,0 +1,31 @@ +//! Typed requests of plan entry construction. + +use pyo3::prelude::{Bound, PyAny}; + +/// What resolving relation locations and source metadata for one scope reads. +#[derive(Debug)] +pub(crate) struct RelationsRequest<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) connection: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) deferral: Bound<'py, PyAny>, + pub(crate) project_config: Bound<'py, PyAny>, + pub(crate) local_config: Bound<'py, PyAny>, + pub(crate) known_source_columns: Bound<'py, PyAny>, + pub(crate) relation_keys: Bound<'py, PyAny>, +} + +/// What building model plan entries for one execution scope reads. +#[derive(Debug)] +pub(crate) struct PlanEntriesRequest<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) relations: Bound<'py, PyAny>, + pub(crate) resolved_models: Bound<'py, PyAny>, + pub(crate) cursor_overrides: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) build_inputs: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index 87d8d2f91b..ffee05797b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -1,6 +1,6 @@ //! `PlanOutput` assembly from planned model entries, seeds, sources, functions, audits and tests. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, @@ -9,7 +9,9 @@ use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::models::SqlTestAssembly; use crate::bindings::_helpers::sql_test_glue::plan_assembly::assemble_sql_test_plans; +use crate::bindings::_helpers::sql_test_glue::plan_driver::resolve_sql_test_model_chains; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -482,21 +484,13 @@ pub(super) fn selected_test_entries<'py>( none(py) }; let results = assemble_sql_test_plans( - py, - Some(keywords( - py, - &[ - ("tests", tuple_of(py, selected)?), - ("project", project.clone()), - ("adapter", adapter.clone()), - ("sql_analysis_enabled", analysis), - ( - "validate_fixtures", - pyo3::types::PyBool::new(py, true).to_owned().into_any(), - ), - ("fixture_planning_context", fixture_context), - ], - )?), + &selected, + (project, adapter), + &SqlTestAssembly { + sql_analysis_enabled: analysis, + validate_fixtures: true, + fixture_planning_context: fixture_context, + }, )?; let mut diagnostics: Vec = Vec::new(); for result in results.try_iter()? { @@ -755,11 +749,7 @@ fn sql_test_model_chain_names_by_key<'py>( tests: &Bound<'py, PyAny>, ) -> PyResult> { let py = project.py(); - let chains = native( - py, - NativeAttr::ResolveSqlTestModelChains, - &[("project", project.clone()), ("tests", tests.clone())], - )?; + let chains = resolve_sql_test_model_chains(project, tests)?; let by_key = PyDict::new(py); for (test, chain) in tests.try_iter()?.zip(chains.try_iter()?) { by_key.set_item(test?.getattr("key")?, chain?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs index 07db82f0c2..7a2e29c8e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/relations.rs @@ -1,6 +1,7 @@ //! Relation locations and source metadata for plan entry construction. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_entries::models::RelationsRequest; use crate::bindings::_helpers::planner_entries::values::{native, none, planner_model}; use crate::bindings::_helpers::planner_strategy::audit_entries::audit_is_selected; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -66,21 +67,25 @@ fn source_warehouse_columns<'py>( /// Resolve relation locations and source metadata for plan entry construction. pub(crate) fn planner_relations_context<'py>( - args: &KeywordArguments<'py>, + request: RelationsRequest<'py>, ) -> PyResult> { - let project = args.value("project")?; + let RelationsRequest { + project, + adapter, + connection, + scope, + deferral, + project_config, + local_config, + known_source_columns, + relation_keys, + } = request; let py = project.py(); - let adapter = args.value("adapter")?; - let connection = args.value("connection")?; - let scope = args.value("scope")?; - let deferral = args.value("deferral")?; let deferral = if deferral.is_none() { planner_model(py, RecordId::DeferralInputs, &[])? } else { deferral }; - let project_config = args.value("project_config")?; - let local_config = args.value("local_config")?; let selected_keys = scope.getattr("selected_keys")?; let mut model_locations = native( py, @@ -113,7 +118,6 @@ pub(crate) fn planner_relations_context<'py>( seed_locations = applied.get_item(1)?; } let audit_keys = selected_audit_relation_keys(&project, &selected_keys)?; - let relation_keys = args.value("relation_keys")?; let load_keys = if relation_keys.is_none() { selected_keys.call_method1("__or__", (&audit_keys,))? } else { @@ -174,7 +178,7 @@ pub(crate) fn planner_relations_context<'py>( let (columns, listed_source_names) = source_warehouse_columns( (&project, &adapter, &connection), &source_read_map, - &args.value("known_source_columns")?, + &known_source_columns, )?; let semantic_columns = native( py, @@ -211,7 +215,7 @@ pub(crate) fn planner_relations_context<'py>( /// Resolve relation locations and source metadata for plan entry construction. #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(crate) fn build_planner_relations_context<'py>( +fn build_planner_relations_context<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -230,7 +234,17 @@ pub(crate) fn build_planner_relations_context<'py>( ], ), )?; - planner_relations_context(&args) + planner_relations_context(RelationsRequest { + project: args.value("project")?, + adapter: args.value("adapter")?, + connection: args.value("connection")?, + scope: args.value("scope")?, + deferral: args.value("deferral")?, + project_config: args.value("project_config")?, + local_config: args.value("local_config")?, + known_source_columns: args.value("known_source_columns")?, + relation_keys: args.value("relation_keys")?, + }) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs index 4ba5097a0f..8d4f973716 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_candidates.rs @@ -1,21 +1,22 @@ //! Gather live candidate evidence without changing adapter inspection ownership. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, }; -use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; +use crate::bindings::_helpers::planner_migrations::models::{ + ColumnCandidate, ColumnMigrationInputs, MigrationPhase, +}; use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyTuple}; pub(super) fn candidates<'py>( - args: &KeywordArguments<'py>, + (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), dialect: &Bound<'py, PyAny>, ) -> PyResult>> { - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; + let scope = phase.scope.clone(); + let snapshot = phase.snapshot.clone(); let py = scope.py(); let mut result: Vec> = Vec::new(); for key in scope.getattr("execution_order")?.try_iter()? { @@ -36,7 +37,7 @@ pub(super) fn candidates<'py>( continue; } let name = model_reads::name_object(&model)?; - if args.value("full_refresh_model_names")?.contains(&name)? + if columns.full_refresh_model_names.contains(&name)? || !snapshot.getattr("existing_relations")?.contains(&name)? { continue; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs index 2aa146b874..1126347f21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_overlay.rs @@ -1,13 +1,14 @@ //! Project column renames onto the existing snapshot and redirect cursor reads. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_migrations::column_objects::{ config, definition_hash, lower, native, replace, }; use crate::bindings::_helpers::planner_migrations::models::ColumnCandidate; +use crate::bindings::_helpers::planner_migrations::models::{ + ColumnMigrationInputs, MigrationPhase, +}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -116,12 +117,12 @@ pub(super) fn overlay<'py>( } pub(super) fn cursors<'py>( - args: &KeywordArguments<'py>, + (phase, columns): (&MigrationPhase<'py>, &ColumnMigrationInputs<'py>), snapshot: Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], ) -> PyResult> { let py = snapshot.py(); - let scope = args.value("scope")?; + let scope = phase.scope.clone(); let models = scope.getattr("models_by_name")?; let origins = PyDict::new(py); for entry in entries { @@ -144,9 +145,7 @@ pub(super) fn cursors<'py>( } let targets = PyDict::new(py); for (name, _) in origins.iter() { - let physical = args - .value("physical_relations")? - .call_method1("get", (&name,))?; + let physical = columns.physical_relations.call_method1("get", (&name,))?; let target = if physical.is_truthy()? { physical } else { @@ -165,15 +164,15 @@ pub(super) fn cursors<'py>( let redirected = call_with( ®istry::native(py, NativeAttr::MigrationRedirectedCursorSnapshots)?, &[ - ("runtime", args.value("runtime")?), + ("runtime", phase.runtime.clone()), ("scope", scope), ( "existing_relations", snapshot.getattr("existing_relations")?, ), ("target_relations", targets.into_any()), - ("overrides", args.value("overrides")?), - ("deferral", args.value("deferral")?), + ("overrides", phase.overrides.clone()), + ("deferral", phase.deferral.clone()), ("origin_cursor_columns", origins.into_any()), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 33c059bba3..d9c61e3e77 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -5,6 +5,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::replace; use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; +use crate::bindings::_helpers::planner_migrations::models::{ + ColumnMigrationInputs, MigrationPhase, +}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -12,9 +15,9 @@ use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; /// Decide column renames and return the existing planning result model. -#[pyfunction] +#[pyfunction(name = "plan_column_migrations")] #[pyo3(signature = (**options))] -pub(crate) fn plan_column_migrations<'py>( +fn plan_column_migrations_of<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -36,11 +39,33 @@ pub(crate) fn plan_column_migrations<'py>( &[], ), )?; - let runtime = args.value("runtime")?; - let snapshot = args.value("snapshot")?; + plan_column_migrations( + &MigrationPhase { + runtime: args.value("runtime")?, + scope: args.value("scope")?, + snapshot: args.value("snapshot")?, + overrides: args.value("overrides")?, + deferral: args.value("deferral")?, + }, + &ColumnMigrationInputs { + full_refresh_model_names: args.value("full_refresh_model_names")?, + physical_relations: args.value("physical_relations")?, + source_columns: args.value("source_columns")?, + }, + ) +} + +/// Decide column renames for the selected incremental and snapshot models. +pub(crate) fn plan_column_migrations<'py>( + phase: &MigrationPhase<'py>, + columns: &ColumnMigrationInputs<'py>, +) -> PyResult> { + let py = phase.runtime.py(); + let runtime = phase.runtime.clone(); + let snapshot = phase.snapshot.clone(); let adapter = runtime.getattr("adapter")?; let candidates = super::column_candidates::candidates( - &args, + (phase, columns), &adapter.call_method0("sql_analysis_dialect")?, )?; if candidates.is_empty() { @@ -77,7 +102,7 @@ pub(crate) fn plan_column_migrations<'py>( py, KnownInputColumns::new( &snapshot.getattr("existing_columns")?, - &args.value("source_columns")?, + &columns.source_columns, ), )? .into_any(); @@ -91,8 +116,8 @@ pub(crate) fn plan_column_migrations<'py>( &state.getattr("column_events")?, &inputs, )?; - let physical = args - .value("physical_relations")? + let physical = columns + .physical_relations .call_method1("get", (model_reads::name_object(&candidate.model)?,))?; let checked = super::column_entries::checked(&runtime, candidate, decided, &physical)?; for hint in super::column_entries::hints(candidate, &checked.entries)?.try_iter()? { @@ -112,7 +137,7 @@ pub(crate) fn plan_column_migrations<'py>( } } let overlaid = super::column_overlay::overlay(&snapshot, &candidates, &entries, &inputs)?; - let snapshot = super::column_overlay::cursors(&args, overlaid, &entries)?; + let snapshot = super::column_overlay::cursors((phase, columns), overlaid, &entries)?; native( py, RecordId::ColumnMigrationPlanning, @@ -126,5 +151,5 @@ pub(crate) fn plan_column_migrations<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_column_migrations, module)?) + module.add_function(wrap_pyfunction!(plan_column_migrations_of, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs index da8768b2ea..00f3fa21a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/compatibility.rs @@ -20,23 +20,20 @@ fn config(model: &Bound<'_, PyAny>, key: &str) -> PyResult> { .extract() } +/// The schema findings that block migrating one model's history; `snapshots_config` is given for snapshots. fn blocking<'py>( - py: Python<'py>, - arguments: &KeywordArguments<'py>, - snapshot: bool, + (model, findings_in): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + dialect: &Bound<'py, PyAny>, + snapshots_config: Option<&Bound<'py, PyAny>>, ) -> PyResult> { - let model = arguments.value("model")?; - let policy = if snapshot { - config(&model, "snapshot_schema_change")? + let py = model.py(); + let snapshot = snapshots_config.is_some(); + let policy = if let Some(snapshots_config) = snapshots_config { + config(model, "snapshot_schema_change")? .filter(|value| !value.is_empty()) - .unwrap_or( - arguments - .value("snapshots_config")? - .getattr("schema_change")? - .extract()?, - ) + .unwrap_or(snapshots_config.getattr("schema_change")?.extract()?) } else { - config(&model, "on_schema_change")? + config(model, "on_schema_change")? .filter(|value| !value.is_empty()) .unwrap_or_else(|| "append_new_columns".to_owned()) }; @@ -50,7 +47,7 @@ fn blocking<'py>( )? .call1((&policy,))?; let mut findings: Vec = Vec::new(); - for finding in arguments.value("findings")?.try_iter()? { + for finding in findings_in.try_iter()? { let finding = finding?; findings.push(CompatibilityFinding { kind: finding.getattr("kind")?.extract()?, @@ -59,7 +56,7 @@ fn blocking<'py>( actual_type: finding.getattr("actual_type")?.extract()?, }); } - let dialect: Option = arguments.value("dialect")?.extract()?; + let dialect: Option = dialect.extract()?; PyTuple::new( py, compatibility_findings( @@ -83,7 +80,11 @@ fn migration_incremental_blocking_findings<'py>( options, (&["model", "findings", "dialect"], &[]), )?; - blocking(py, &arguments, false) + blocking( + (&arguments.value("model")?, &arguments.value("findings")?), + &arguments.value("dialect")?, + None, + ) } #[pyfunction] @@ -98,7 +99,11 @@ fn migration_snapshot_blocking_findings<'py>( options, (&["model", "findings", "dialect", "snapshots_config"], &[]), )?; - blocking(py, &arguments, true) + blocking( + (&arguments.value("model")?, &arguments.value("findings")?), + &arguments.value("dialect")?, + Some(&arguments.value("snapshots_config")?), + ) } #[pyfunction] @@ -181,16 +186,12 @@ fn check_migration_compatibility<'py>( return result(py, "not_checked", PyTuple::empty(py)); } let snapshot = config(&model, "materialized")?.as_deref() == Some("snapshot"); - let options = PyDict::new(py); - options.set_item("model", model)?; - options.set_item("findings", change.getattr("schema_findings")?)?; - options.set_item("dialect", arguments.value("column_dialect")?)?; - let findings = if snapshot { - options.set_item("snapshots_config", arguments.value("snapshots_config")?)?; - migration_snapshot_blocking_findings(py, Some(options))? - } else { - migration_incremental_blocking_findings(py, Some(options))? - }; + let snapshots_config = arguments.value("snapshots_config")?; + let findings = blocking( + (&model, &change.getattr("schema_findings")?), + &arguments.value("column_dialect")?, + snapshot.then_some(&snapshots_config), + )?; result( py, if findings.is_empty() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs index 44516c9d52..b59d161658 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/mod.rs @@ -20,7 +20,7 @@ mod model_handover; mod model_overlay; pub(crate) mod model_planning; mod model_warnings; -mod models; +pub(crate) mod models; mod old_name_entries; mod old_name_objects; pub(crate) mod old_name_planning; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs index a224f6ec86..51d7ca507d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs @@ -1,24 +1,25 @@ //! Project inherited fingerprints and live relation facts before normal planning. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; +use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; pub(super) fn overlay<'py>( - args: &KeywordArguments<'py>, + phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], handovers: &Bound<'py, PyDict>, ) -> PyResult> { - let snapshot = args.value("snapshot")?; + let snapshot = phase.snapshot.clone(); if handovers.is_empty() { return Ok(snapshot); } - let runtime = args.value("runtime")?; + let runtime = phase.runtime.clone(); let py = runtime.py(); let relations = snapshot .getattr("existing_relations")? @@ -91,11 +92,11 @@ pub(super) fn overlay<'py>( ®istry::native(py, NativeAttr::MigrationRedirectedCursorSnapshots)?, &[ ("runtime", runtime), - ("scope", args.value("scope")?), + ("scope", phase.scope.clone()), ("existing_relations", relations.clone().into_any()), ("target_relations", redirected.into_any()), - ("overrides", args.value("overrides")?), - ("deferral", args.value("deferral")?), + ("overrides", phase.overrides.clone()), + ("deferral", phase.deferral.clone()), ], )?; cursors.call_method1("update", (gathered,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index f16dd1a153..f1586bd36c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_migrations::column_objects::{config, native, replace}; +use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -100,30 +101,14 @@ fn decide_model_migration<'py>( )?) } -#[pyfunction] -#[pyo3(signature = (**options))] +/// Model migrations of the selected scope: discovered and declared moves, decided against live state. pub(crate) fn plan_model_migrations<'py>( - py: Python<'py>, - options: Option>, + phase: &MigrationPhase<'py>, + fingerprints: &Bound<'py, PyAny>, ) -> PyResult> { - let args = KeywordArguments::read( - py, - "plan_model_migrations", - options, - ( - &[ - "runtime", - "scope", - "snapshot", - "overrides", - "deferral", - "fingerprints", - ], - &[], - ), - )?; - let runtime = args.value("runtime")?; - let snapshot = args.value("snapshot")?; + let py = phase.runtime.py(); + let runtime = phase.runtime.clone(); + let snapshot = phase.snapshot.clone(); let schemas = call_with( ®istry::native(py, NativeAttr::MigrationProjectSchemas)?, &[("runtime", runtime.clone())], @@ -143,19 +128,19 @@ pub(crate) fn plan_model_migrations<'py>( ("known_fingerprints", snapshot.getattr("fingerprints")?), ], )?; - let discovery = discover(&args, &state, &schemas)?; - finish(&args, &state, &schemas, &discovery) + let discovery = discover((phase, fingerprints), &state, &schemas)?; + finish(phase, &state, &schemas, &discovery) } fn discover<'py>( - args: &KeywordArguments<'py>, + (phase, fingerprints): (&MigrationPhase<'py>, &Bound<'py, PyAny>), state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, ) -> PyResult> { - let runtime = args.value("runtime")?; + let runtime = phase.runtime.clone(); let py = runtime.py(); - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; + let scope = phase.scope.clone(); + let snapshot = phase.snapshot.clone(); let mut declared_requests: Vec> = Vec::new(); for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { if let Some(request) = declared(&item?)? { @@ -198,21 +183,21 @@ fn discover<'py>( ("declarations", PyTuple::new(py, declarations)?.into_any()), ("state", state.clone()), ("project_schemas", schemas.clone()), - ("fingerprints", args.value("fingerprints")?), + ("fingerprints", fingerprints.clone()), ], ) } fn finish<'py>( - args: &KeywordArguments<'py>, + phase: &MigrationPhase<'py>, state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, discovery: &Bound<'py, PyAny>, ) -> PyResult> { - let runtime = args.value("runtime")?; + let runtime = phase.runtime.clone(); let py = runtime.py(); - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; + let scope = phase.scope.clone(); + let snapshot = phase.snapshot.clone(); let mut resolved: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); for request in discovery.getattr("requests")?.try_iter()? { let request = request?; @@ -320,7 +305,7 @@ fn finish<'py>( let overlaid = if entries.is_empty() { snapshot.clone() } else { - super::model_overlay::overlay(args, state, &entries, &handovers)? + super::model_overlay::overlay(phase, state, &entries, &handovers)? }; let reference_renames = native( py, @@ -351,6 +336,5 @@ fn finish<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(manual_migration_requests, module)?)?; - module.add_function(wrap_pyfunction!(decide_model_migration, module)?)?; - module.add_function(wrap_pyfunction!(plan_model_migrations, module)?) + module.add_function(wrap_pyfunction!(decide_model_migration, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs index 1c11e9b2fd..9536eb9803 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs @@ -26,3 +26,21 @@ pub(crate) struct CheckedColumns<'py> { pub entries: Vec>, pub warnings: Vec>, } + +/// The planning-phase inputs every migration planner reads. +#[derive(Debug)] +pub(crate) struct MigrationPhase<'py> { + pub(crate) runtime: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) overrides: Bound<'py, PyAny>, + pub(crate) deferral: Bound<'py, PyAny>, +} + +/// The column migration planning inputs beyond the phase. +#[derive(Debug)] +pub(crate) struct ColumnMigrationInputs<'py> { + pub(crate) full_refresh_model_names: Bound<'py, PyAny>, + pub(crate) physical_relations: Bound<'py, PyAny>, + pub(crate) source_columns: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index 8265c05d0f..a398c75a9e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -1,8 +1,6 @@ //! Assemble new and resumed old-name view steps from recorded facts. -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::planner_migrations::column_objects::{model, native, text}; use crate::bindings::_helpers::planner_migrations::constants::{PENDING_ARCHIVE, PENDING_VIEW}; use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; @@ -16,12 +14,12 @@ use pyo3::types::{PyBool, PyList, PyTuple}; impl<'py> OldNamePlanner<'py> { pub(super) fn new( - args: &KeywordArguments<'py>, + (runtime, columns): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), models: Vec>, histories: Vec>, views: Bound<'py, PyAny>, ) -> PyResult { - let runtime = args.value("runtime")?; + let runtime = runtime.clone(); let py = runtime.py(); let project_config = runtime.getattr("project_config")?; let config = if project_config.is_none() { @@ -33,7 +31,7 @@ impl<'py> OldNamePlanner<'py> { models, histories, views, - columns: args.value("column_entries")?, + columns: columns.clone(), retention: config.getattr("old_name_views")?, now: runtime.getattr("invocation_time")?, runtime, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 2b506c7163..f771290e8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -1,23 +1,23 @@ //! Old-name planning orchestration over existing live state inspection. -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; fn schemas<'py>( - args: &KeywordArguments<'py>, + (phase, migration_entries): (&MigrationPhase<'py>, &Bound<'py, PyAny>), selected: &[Bound<'py, PyAny>], ) -> PyResult> { - let runtime = args.value("runtime")?; + let runtime = phase.runtime.clone(); let py = runtime.py(); - let snapshot = args.value("snapshot")?; + let snapshot = phase.snapshot.clone(); let needed = PySet::empty(py)?; for item in selected { let schema = item.getattr("destination")?.getattr("schema")?; @@ -25,7 +25,7 @@ fn schemas<'py>( needed.add(schema)?; } } - for entry in args.value("migration_entries")?.try_iter()? { + for entry in migration_entries.try_iter()? { let schema = entry?.getattr("origin")?.getattr("schema")?; if !schema.is_none() { needed.add(schema)?; @@ -67,29 +67,14 @@ fn schemas<'py>( ) } -#[pyfunction] -#[pyo3(signature = (**options))] +/// Old-name compatibility views to create, resume or retire for the planned model and column migrations. pub(crate) fn plan_old_name_views<'py>( - py: Python<'py>, - options: Option>, + phase: &MigrationPhase<'py>, + moves: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - let args = KeywordArguments::read( - py, - "plan_old_name_views", - options, - ( - &[ - "runtime", - "scope", - "snapshot", - "migration_entries", - "column_entries", - ], - &[], - ), - )?; - let runtime = args.value("runtime")?; - let scope = args.value("scope")?; + let py = phase.runtime.py(); + let runtime = phase.runtime.clone(); + let scope = phase.scope.clone(); let project = runtime.getattr("project")?; let models: Vec> = project .getattr("models")? @@ -124,7 +109,7 @@ pub(crate) fn plan_old_name_views<'py>( ), ], )?; - let needed = schemas(&args, &selected)?; + let needed = schemas((phase, moves.0), &selected)?; if needed.is_truthy()? { call_with( &state.getattr("inspect_old_name_views")?, @@ -146,10 +131,15 @@ pub(crate) fn plan_old_name_views<'py>( histories.push(history); } let views = PyTuple::new(py, views)?.into_any(); - let planner = super::models::OldNamePlanner::new(&args, models, histories, views.clone())?; + let planner = super::models::OldNamePlanner::new( + (&phase.runtime, moves.1), + models, + histories, + views.clone(), + )?; let mut entries: Vec> = Vec::new(); let covered = PySet::empty(py)?; - for entry in args.value("migration_entries")?.try_iter()? { + for entry in moves.0.try_iter()? { let entry = entry?; let name = entry.getattr("model_name")?; if let Some(item) = by_name.get_item(&name)? @@ -172,7 +162,7 @@ pub(crate) fn plan_old_name_views<'py>( NativeAttr::OldNameClaimErrors, &[ ("selected", PyTuple::new(py, selected)?.into_any()), - ("snapshot", args.value("snapshot")?), + ("snapshot", phase.snapshot.clone()), ("views", views.clone()), ("now", runtime.getattr("invocation_time")?), ], @@ -191,7 +181,3 @@ pub(crate) fn plan_old_name_views<'py>( ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(plan_old_name_views, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs index 2718ea8c91..576f52c48e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/registration.rs @@ -7,7 +7,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::compatibility::register(module)?; super::matching::register(module)?; super::old_names::register(module)?; - super::old_name_planning::register(module)?; super::columns::register(module)?; super::column_planning::register(module)?; super::model_planning::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 0b13e121f6..762a1811b4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,14 +1,13 @@ //! Execution-scope pruning, write-hash reconciliation and plan entry construction. -use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; +use crate::bindings::_helpers::planner_entries::models::{PlanEntriesRequest, RelationsRequest}; +use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; -use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::pyfunction; -use pyo3::types::PyDict as KeywordDict; use pyo3::types::{PyFrozenSet, PyTuple}; #[pyfunction] @@ -87,64 +86,30 @@ fn config_or<'py>( Ok(value) } -#[pyfunction] -#[pyo3(signature = (**options))] +/// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. pub(crate) fn build_planner_entry_results<'py>( - py: pyo3::Python<'py>, - options: Option>, + (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (overrides, deferral): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (pruning, reconciliation): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + source_freshness: &Bound<'py, PyAny>, ) -> PyResult> { - let args = KeywordArguments::read( - py, - "build_planner_entry_results", - options, - ( - &[ - "runtime", - "warehouse", - "identities", - "overrides", - "policies", - "deferral", - "pruning", - "reconciliation", - "source_freshness", - ], - &[], - ), - )?; - let runtime = &args.value("runtime")?; - let warehouse = &args.value("warehouse")?; - let identities = &args.value("identities")?; - let overrides = &args.value("overrides")?; - let policies = &args.value("policies")?; - let deferral = &args.value("deferral")?; - let pruning = &args.value("pruning")?; - let reconciliation = &args.value("reconciliation")?; - let source_freshness = &args.value("source_freshness")?; - - let _ = (identities, policies); let py = runtime.py(); let project = runtime.getattr("project")?; let config = runtime.getattr("project_config")?; let execution_scope = pruning.getattr("execution_scope")?; - let relations = call_with( - ®istry::native(py, NativeAttr::BuildPlannerRelationsContext)?, - &[ - ("project", project.clone()), - ("adapter", runtime.getattr("adapter")?), - ("connection", runtime.getattr("connection")?), - ("scope", execution_scope.clone()), - ("deferral", deferral.clone()), - ("project_config", config.clone()), - ("local_config", runtime.getattr("local_config")?), - ( - "known_source_columns", - warehouse - .getattr("inspection_relations")? - .getattr("source_warehouse_columns")?, - ), - ], - )?; + let relations = planner_relations_context(RelationsRequest { + project: project.clone(), + adapter: runtime.getattr("adapter")?, + connection: runtime.getattr("connection")?, + scope: execution_scope.clone(), + deferral: deferral.clone(), + project_config: config.clone(), + local_config: runtime.getattr("local_config")?, + known_source_columns: warehouse + .getattr("inspection_relations")? + .getattr("source_warehouse_columns")?, + relation_keys: py.None().into_bound(py), + })?; let propagation = source_freshness.getattr("propagation")?; let blocked = if propagation.is_none() { PyFrozenSet::empty(py)?.into_any() @@ -202,23 +167,19 @@ pub(crate) fn build_planner_entry_results<'py>( ("microbatch_limit_action", limit_action), ], )?; - let entries = call_with( - ®istry::native(py, NativeAttr::BuildPlanEntries)?, - &[ - ("project", project), - ("adapter", runtime.getattr("adapter")?), - ("scope", execution_scope), - ("snapshot", warehouse.getattr("snapshot")?), - ("relations", relations), - ( - "resolved_actions", - reconciliation.getattr("resolved_actions")?, - ), - ("cursor_overrides", overrides.getattr("cursor_overrides")?), - ("full_refresh", overrides.getattr("full_refresh")?), - ("build_inputs", build_inputs), - ], - )?; + let entries = plan_entries(PlanEntriesRequest { + project, + adapter: runtime.getattr("adapter")?, + scope: execution_scope, + snapshot: warehouse.getattr("snapshot")?, + relations, + resolved_models: reconciliation + .getattr("resolved_actions")? + .getattr("models")?, + cursor_overrides: overrides.getattr("cursor_overrides")?, + full_refresh: overrides.getattr("full_refresh")?.is_truthy()?, + build_inputs, + })?; native( py, RecordId::PlannerEntryResults, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs index be031788b3..8d08fb586b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/identities.rs @@ -76,9 +76,9 @@ fn reusable_model_names( Ok(names) } -#[pyfunction] +#[pyfunction(name = "detect_stale_warning_changes")] #[pyo3(signature = (**options))] -pub(crate) fn detect_stale_warning_changes<'py>( +pub(crate) fn detect_stale_warning_changes_of<'py>( py: pyo3::Python<'py>, options: Option>, ) -> PyResult> { @@ -97,12 +97,22 @@ pub(crate) fn detect_stale_warning_changes<'py>( &[], ), )?; - let project = &args.value("project")?; - let scopes = &args.value("scopes")?; - let snapshot = &args.value("snapshot")?; - let identities = &args.value("identities")?; - let execution_changes = &args.value("execution_changes")?; + detect_stale_warning_changes( + (&args.value("project")?, &args.value("scopes")?), + &args.value("snapshot")?, + ( + &args.value("identities")?, + &args.value("execution_changes")?, + ), + ) +} +/// Changes of stale-warning models whose identities differ from the execution identities. +pub(crate) fn detect_stale_warning_changes<'py>( + (project, scopes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + snapshot: &Bound<'py, PyAny>, + (identities, execution_changes): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { let py = project.py(); let reusable = reusable_model_names( identities, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 66fc997a07..85b98dfdee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -13,7 +13,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!( - super::identities::detect_stale_warning_changes, + super::identities::detect_stale_warning_changes_of, module )?)?; module.add_function(wrap_pyfunction!( @@ -28,10 +28,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::execution_changes::reconcile_execution_changes, module )?)?; - module.add_function(wrap_pyfunction!( - super::execution_changes::build_planner_entry_results, - module - )?)?; module.add_function(wrap_pyfunction!( super::full_refresh::resolve_model_full_refresh, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index e06e19c04c..62ecf3ee5e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -1,9 +1,12 @@ //! The warehouse inspection phase: snapshot, migrations and inspection relations in one pass. -use crate::bindings::_helpers::compile_driver::python_calls::keywords; -use crate::bindings::_helpers::planner_entries::relations::build_planner_relations_context; +use crate::bindings::_helpers::planner_entries::models::RelationsRequest; +use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_migrations::column_planning::plan_column_migrations; use crate::bindings::_helpers::planner_migrations::model_planning::plan_model_migrations; +use crate::bindings::_helpers::planner_migrations::models::{ + ColumnMigrationInputs, MigrationPhase, +}; use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; use crate::bindings::_helpers::planner_phases::objects::native; @@ -88,36 +91,26 @@ pub(crate) fn gather_planner_warehouse_state<'py>( deferred_locations: deferral.getattr("deferred_locations")?, cursor_scope, })?; - let migrations = plan_model_migrations( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope.clone()), - ("snapshot", snapshot.clone()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ("fingerprints", fingerprints.clone()), - ], - )?), - )?; + let phase = MigrationPhase { + runtime: runtime.clone(), + scope: selected_scope.clone(), + snapshot: snapshot.clone(), + overrides: overrides.clone(), + deferral: deferral.clone(), + }; + let migrations = plan_model_migrations(&phase, &fingerprints)?; fingerprints.call_method0("persist")?; - let inspection_relations = build_planner_relations_context( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", runtime.getattr("connection")?), - ("scope", scopes.getattr("inspection_scope")?), - ("deferral", deferral.clone()), - ("project_config", config), - ("local_config", runtime.getattr("local_config")?), - ], - )?), - )?; + let inspection_relations = planner_relations_context(RelationsRequest { + project: project.clone(), + adapter: adapter.clone(), + connection: runtime.getattr("connection")?, + scope: scopes.getattr("inspection_scope")?, + deferral: deferral.clone(), + project_config: config, + local_config: runtime.getattr("local_config")?, + known_source_columns: py.None().into_bound(py), + relation_keys: py.None().into_bound(py), + })?; let physical = PyDict::new(py); for entry in migrations.getattr("entries")?.try_iter()? { let entry = entry?; @@ -139,39 +132,31 @@ pub(crate) fn gather_planner_warehouse_state<'py>( } } let columns = plan_column_migrations( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope.clone()), - ("snapshot", migrations.getattr("snapshot")?), - ( - "full_refresh_model_names", - effectively_full_refreshed_model_names(&project, full_refresh)?, - ), - ("physical_relations", physical.into_any()), - ("overrides", overrides.clone()), - ("deferral", deferral.clone()), - ( - "source_columns", - inspection_relations.getattr("source_warehouse_columns")?, - ), - ], - )?), + &MigrationPhase { + snapshot: migrations.getattr("snapshot")?, + ..phase + }, + &ColumnMigrationInputs { + full_refresh_model_names: effectively_full_refreshed_model_names( + &project, + full_refresh, + )?, + physical_relations: physical.into_any(), + source_columns: inspection_relations.getattr("source_warehouse_columns")?, + }, )?; let old_names = plan_old_name_migration_views( - py, - Some(keywords( - py, - &[ - ("runtime", runtime.clone()), - ("scope", selected_scope), - ("snapshot", snapshot), - ("migration_entries", migrations.getattr("entries")?), - ("column_entries", columns.getattr("entries")?), - ], - )?), + &MigrationPhase { + runtime: runtime.clone(), + scope: selected_scope, + snapshot, + overrides: overrides.clone(), + deferral: deferral.clone(), + }, + ( + &migrations.getattr("entries")?, + &columns.getattr("entries")?, + ), )?; if !runtime.getattr("on_progress")?.is_none() { progress( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs index 9626282229..6d8260bb67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_model.rs @@ -1,7 +1,7 @@ //! Model cursor orchestration over live warehouse snapshot facts. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::model_config::config_values::{ config_cursor_bound, config_str as authored_config_str, }; @@ -12,12 +12,12 @@ use crate::bindings::_helpers::planner_resolution::constants::CURSOR_START; use crate::bindings::_helpers::planner_resolution::cursor_policies::{ future_cursor_config, start_cursor_config, }; +use crate::bindings::_helpers::planner_resolution::models::ModelSqlRequest; use crate::bindings::_helpers::planner_resolution::values::string; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::PyDict; -use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::{IntoPyObject, PyErr}; fn config<'py>(model: &Bound<'py, PyAny>, key: &str) -> PyResult> { let values: Bound<'py, PyAny> = model.getattr("config")?.getattr("values")?; @@ -65,35 +65,16 @@ fn history_redirected(model: &Bound<'_, PyAny>, snapshot: &Bound<'_, PyAny>) -> && target.ne(model_reads::destination(model)?.getattr("qualified_name")?)?) } -#[pyfunction] -#[pyo3(signature = (**kwargs))] +/// One model's cursor bounds from its snapshot, overrides and cursor policies; `None` when unbounded by a full refresh. pub(crate) fn compute_model_cursor_bounds<'py>( - py: Python<'py>, - kwargs: Option>, + request: &ModelSqlRequest<'py>, + context: &Bound<'py, PyAny>, ) -> PyResult> { - let args = KeywordArguments::read( - py, - "compute_model_cursor_bounds", - kwargs, - ( - &[ - "model", - "snapshot", - "backfill", - "full_refresh", - "start_cursor_override", - "end_cursor_override", - "runtime_cursor_producer_names", - "context", - ], - &["replaces_relation"], - ), - )?; - let model = args.value("model")?; + let model = request.model.clone(); + let py = model.py(); if !config(&model, "materialized")?.eq("incremental")? || config(&model, "cursor")?.is_none() { return Ok(py.None().into_bound(py)); } - let context = args.value("context")?; let incremental_mode = config(&model, "incremental_mode")?; let future_config = future_cursor_config(&model, &context.getattr("future_cursor_config")?)?; let start_config = start_cursor_config(&model, &context.getattr("start_cursor_config")?)?; @@ -102,8 +83,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( } let cursor_type = config(&model, "cursor_type")?; let cursor_grain = config(&model, "cursor_grain")?; - let start_override = args.value("start_cursor_override")?; - let end_override = args.value("end_cursor_override")?; + let (start_override, end_override) = request.cursor_overrides.clone(); let bounded = call_with( ®istry::native(py, NativeAttr::ResolveBoundedCursorOverride)?, &[ @@ -116,7 +96,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( if !bounded.is_none() { return Ok(bounded); } - let snapshot = args.value("snapshot")?; + let snapshot = &request.snapshot; let mut cursor_snapshot = snapshot .getattr("cursor_snapshots")? .call_method1("get", (model_reads::name_object(&model)?,))?; @@ -124,7 +104,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ®istry::native(py, NativeAttr::ResolveCursorInputRoles)?, &[("model", model.clone())], )?; - let producers = args.value("runtime_cursor_producer_names")?; + let producers = context.getattr("runtime_cursor_producer_names")?; let mut runtime_owned = false; for name in roles.getattr("watermark_inputs")?.try_iter()? { if producers.contains(name?)? { @@ -132,8 +112,8 @@ pub(crate) fn compute_model_cursor_bounds<'py>( break; } } - let full_refresh = args.flag("full_refresh")?; - let replaces = args.flag("replaces_relation")?; + let full_refresh = request.full_refresh; + let replaces = request.replaces_relation; if !full_refresh && !runtime_owned && !cursor_snapshot.is_none() { let empty = call_with( ®istry::native(py, NativeAttr::CursorInputsWithoutRows)?, @@ -142,7 +122,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( .getattr("input_names")?; if empty.is_truthy()? && replaces - && (destination_exists(&model, &snapshot)? + && (destination_exists(&model, snapshot)? || history_redirected(&model, &cursor_snapshot)?) { let refusal = call_with( @@ -206,7 +186,7 @@ pub(crate) fn compute_model_cursor_bounds<'py>( &[("cursor_snapshot", cursor_snapshot)], )?; } - let backfill = args.value("backfill")?; + let backfill = &request.backfill; let duration = if backfill.getattr("action")?.eq("bounded")? { backfill.getattr("duration")? } else { @@ -264,7 +244,3 @@ pub(crate) fn compute_model_cursor_bounds<'py>( ], ) } - -pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(compute_model_cursor_bounds, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs index 8c069d8dd4..90173837c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_overrides.rs @@ -13,9 +13,9 @@ use pyo3::types::PyDict; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cursor::planner::main::override_kind::override_kind; -#[pyfunction] +#[pyfunction(name = "type_cursor_overrides")] #[pyo3(signature = (**kwargs))] -pub(crate) fn type_cursor_overrides<'py>( +fn type_cursor_overrides_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -25,18 +25,27 @@ pub(crate) fn type_cursor_overrides<'py>( kwargs, (&["cursor_overrides"], &["selected_models"]), )?; - let overrides = args.value("cursor_overrides")?; - if overrides.is_none() || !overrides.getattr("has_untyped")?.is_truthy()? { - return Ok(overrides); - } - let mut models: Vec<(String, String)> = Vec::new(); - let selected = if args.passed("selected_models")? { + let selected: Vec> = if args.passed("selected_models")? { args.value("selected_models")? + .try_iter()? + .collect::>()? } else { - pyo3::types::PyTuple::empty(py).into_any() + Vec::new() }; - for model in selected.try_iter()? { - let model = model?; + type_cursor_overrides(&args.value("cursor_overrides")?, &selected) +} + +/// Untyped cursor overrides typed by the cursor type of the selected incremental models. +pub(crate) fn type_cursor_overrides<'py>( + overrides: &Bound<'py, PyAny>, + selected: &[Bound<'py, PyAny>], +) -> PyResult> { + let py = overrides.py(); + if overrides.is_none() || !overrides.getattr("has_untyped")?.is_truthy()? { + return Ok(overrides.clone()); + } + let mut models: Vec<(String, String)> = Vec::new(); + for model in selected { let values = model.getattr("config")?.getattr("values")?; let config = |key: &str| -> PyResult> { Ok(authored_config_str(&values, key)?.unwrap_or_else(|| py.None().into_bound(py))) @@ -46,10 +55,7 @@ pub(crate) fn type_cursor_overrides<'py>( } let kind = config("cursor_type")?; if kind.eq("integer")? || kind.eq("timestamp")? { - models.push(( - model_reads::name_object(&model)?.extract()?, - kind.extract()?, - )); + models.push((model_reads::name_object(model)?.extract()?, kind.extract()?)); } } let mut integer_literals = true; @@ -75,9 +81,9 @@ pub(crate) fn type_cursor_overrides<'py>( ])) { Ok(error) => PyErr::from_value(error), Err(error) => error } })?; let kind = registry::class(py, RecordId::CursorType)?.call1((kind,))?; - method(&overrides, "typed_as", &[("cursor_type", kind)]) + method(overrides, "typed_as", &[("cursor_type", kind)]) } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(type_cursor_overrides, module)?) + module.add_function(wrap_pyfunction!(type_cursor_overrides_of, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs index d5f276315b..063cba97d8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/mod.rs @@ -2,7 +2,7 @@ mod constants; mod cursor_bounds; -mod cursor_model; +pub(crate) mod cursor_model; pub(crate) mod cursor_overrides; pub(crate) mod cursor_policies; pub(crate) mod cursor_ranges; @@ -12,6 +12,7 @@ mod cursor_windows; mod future_safety; mod locations; mod maximum_safety; +pub(crate) mod models; pub(crate) mod orchestration; mod refs; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs new file mode 100644 index 0000000000..db9d2e26f5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs @@ -0,0 +1,24 @@ +//! Typed requests of planner SQL resolution. + +use pyo3::prelude::{Bound, PyAny}; + +/// The cursor window that filters the cursor inputs one resolution reads. +#[derive(Debug)] +pub(crate) struct CursorFiltering<'py> { + pub(crate) bounds: Bound<'py, PyAny>, + pub(crate) filter_inputs: Bound<'py, PyAny>, + pub(crate) cursor_type: Bound<'py, PyAny>, + pub(crate) lower_bound_inclusive: bool, +} + +/// One model's SQL resolution decisions. +#[derive(Debug)] +pub(crate) struct ModelSqlRequest<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) backfill: Bound<'py, PyAny>, + pub(crate) full_refresh: bool, + pub(crate) cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) replaces_relation: bool, + pub(crate) full_rebuild_cause: Option, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs index f4f0192106..81d9bbcea8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/orchestration.rs @@ -5,29 +5,29 @@ use crate::bindings::_helpers::attachments::cursor_intrinsics::resolve_cursor_in use crate::bindings::_helpers::attachments::reference_markers::assert_no_unresolved_sql_markers; use crate::bindings::_helpers::compile_attachment::cursor_roles::resolve_cursor_input_roles; use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, imported, keywords, + KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::config_str as authored_config_str; use crate::bindings::_helpers::planner_resolution::cursor_model::compute_model_cursor_bounds; +use crate::bindings::_helpers::planner_resolution::models::{CursorFiltering, ModelSqlRequest}; use crate::bindings::_helpers::planner_resolution::refs::{ - resolve_dbt_ref_references, resolve_ref_references, resolve_table_function_references, + resolve_dbt_ref_references, resolve_refs, resolve_table_function_references, resolve_udf_references, }; -use crate::bindings::_helpers::planner_resolution::sources::resolve_source_references; +use crate::bindings::_helpers::planner_resolution::sources::resolve_sources; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; -use pyo3::{IntoPyObject, PyErr, pyfunction, wrap_pyfunction}; +use pyo3::{PyErr, pyfunction, wrap_pyfunction}; /// What resolving one model's, hook's or function's references reads. -struct Resolution<'py> { - adapter: Bound<'py, PyAny>, +pub(crate) struct Resolution<'py> { + pub(crate) adapter: Bound<'py, PyAny>, external: Bound<'py, PyAny>, source_map: Bound<'py, PyAny>, source_warehouse_columns: Bound<'py, PyAny>, - star_exclude_keyword: Bound<'py, PyAny>, model_locations: Bound<'py, PyAny>, seed_locations: Bound<'py, PyAny>, function_locations: Bound<'py, PyAny>, @@ -35,7 +35,7 @@ struct Resolution<'py> { impl<'py> Resolution<'py> { /// Resolution against a `ModelPlanContext`'s relations. - fn of_context( + pub(crate) fn of_context( adapter: Bound<'py, PyAny>, context: &Bound<'py, PyAny>, external: Bound<'py, PyAny>, @@ -45,39 +45,26 @@ impl<'py> Resolution<'py> { external, source_map: context.getattr("source_map")?, source_warehouse_columns: context.getattr("source_warehouse_columns")?, - star_exclude_keyword: context.getattr("star_exclude_keyword")?, model_locations: context.getattr("model_locations")?, seed_locations: context.getattr("seed_locations")?, function_locations: context.getattr("function_locations")?, }) } - fn resolve( - &self, - sql: Bound<'py, PyAny>, - cursor: &[(&str, Bound<'py, PyAny>)], - ) -> PyResult { + fn resolve(&self, sql: &str, filtering: &CursorFiltering<'py>) -> PyResult { let py = self.adapter.py(); - let mut source = vec![ - ("query_sql", sql), - ("adapter", self.adapter.clone()), - ("source_map", self.source_map.clone()), - ( - "source_warehouse_columns", - self.source_warehouse_columns.clone(), - ), - ("star_exclude_keyword", self.star_exclude_keyword.clone()), - ]; - source.extend_from_slice(cursor); - let resolved: String = resolve_source_references(py, Some(keywords(py, &source)?))?; - let mut refs = vec![ - ("query_sql", PyString::new(py, &resolved).into_any()), - ("adapter", self.adapter.clone()), - ("model_locations", self.model_locations.clone()), - ("seed_locations", self.seed_locations.clone()), - ]; - refs.extend_from_slice(cursor); - let resolved: String = resolve_ref_references(py, Some(keywords(py, &refs)?))?; + let resolved: String = resolve_sources( + sql, + (&self.source_map, &self.source_warehouse_columns), + &self.adapter, + filtering, + )?; + let resolved: String = resolve_refs( + &resolved, + (&self.model_locations, &self.seed_locations), + &self.adapter, + filtering, + )?; let resolved: String = resolve_dbt_ref_references(&resolved, Some(&self.external))?; let locations = if self.function_locations.is_truthy()? { self.function_locations.clone() @@ -100,16 +87,13 @@ impl<'py> Resolution<'py> { fn unbounded(&self, sql: Bound<'py, PyAny>, label: &str) -> PyResult { let py = self.adapter.py(); let resolved = self.resolve( - sql, - &[ - ("cursor_bounds", py.None().into_bound(py)), - ("cursor_filter_inputs", PyDict::new(py).into_any()), - ("cursor_type", py.None().into_bound(py)), - ( - "lower_bound_inclusive", - true.into_pyobject(py)?.to_owned().into_any(), - ), - ], + &sql.extract::()?, + &CursorFiltering { + bounds: py.None().into_bound(py), + filter_inputs: PyDict::new(py).into_any(), + cursor_type: py.None().into_bound(py), + lower_bound_inclusive: true, + }, )?; self.assert_resolved(&resolved, label)?; Ok(resolved) @@ -141,9 +125,9 @@ impl<'py> Resolution<'py> { } } -#[pyfunction] +#[pyfunction(name = "resolve_model_sql")] #[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_model_sql<'py>( +fn resolve_model_sql_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -168,79 +152,76 @@ pub(crate) fn resolve_model_sql<'py>( ], ), )?; - let model = args.value("model")?; let context = args.value("context")?; let resolution = Resolution::of_context( args.value("adapter")?, &context, args.value("external_sql_reference_resolver")?, )?; + let overrides = args.value("cursor_overrides")?; + let cause = args.value("full_rebuild_cause")?; + let request = ModelSqlRequest { + model: args.value("model")?, + snapshot: args.value("snapshot")?, + backfill: args.value("backfill")?, + full_refresh: args.flag("full_refresh")?, + cursor_overrides: ( + overrides.getattr("start_cursor_override")?, + overrides.getattr("end_cursor_override")?, + ), + replaces_relation: args.flag("replaces_relation")?, + full_rebuild_cause: if cause.is_truthy()? { + Some(cause.extract()?) + } else { + None + }, + }; + let sql: String = resolve_model_sql(&resolution, &context, &request)?; + Ok(PyString::new(py, &sql).into_any()) +} + +/// One model's planned SQL: references resolved, cursor inputs filtered and intrinsics bound. +pub(crate) fn resolve_model_sql<'py>( + resolution: &Resolution<'py>, + context: &Bound<'py, PyAny>, + request: &ModelSqlRequest<'py>, +) -> PyResult { + let model = &request.model; + let py = model.py(); let cursor_type = authored_config_str(&model.getattr("config")?.getattr("values")?, "cursor_type")? .unwrap_or_else(|| py.None().into_bound(py)); - let overrides = args.value("cursor_overrides")?; - let bounds = compute_model_cursor_bounds( - py, - Some(keywords( - py, - &[ - ("model", model.clone()), - ("snapshot", args.value("snapshot")?), - ("backfill", args.value("backfill")?), - ("full_refresh", args.value("full_refresh")?), - ( - "start_cursor_override", - overrides.getattr("start_cursor_override")?, - ), - ( - "end_cursor_override", - overrides.getattr("end_cursor_override")?, - ), - ( - "runtime_cursor_producer_names", - context.getattr("runtime_cursor_producer_names")?, - ), - ("context", context.clone()), - ( - "replaces_relation", - args.value_or_false("replaces_relation")?, - ), - ], - )?), - )?; - let roles = resolve_cursor_input_roles(&model)?; - let inclusive = append_cursor_inclusive(&model)?; + let bounds = compute_model_cursor_bounds(request, context)?; + let roles = resolve_cursor_input_roles(model)?; + let filtering = CursorFiltering { + bounds: bounds.clone(), + filter_inputs: roles.getattr("filter_inputs")?, + cursor_type: cursor_type.clone(), + lower_bound_inclusive: append_cursor_inclusive(model)?, + }; let mut sql = resolution.resolve( - model_reads::query_sql(&model)?, - &[ - ("cursor_bounds", bounds.clone()), - ("cursor_filter_inputs", roles.getattr("filter_inputs")?), - ("cursor_type", cursor_type.clone()), - ("lower_bound_inclusive", inclusive), - ], + &model_reads::query_sql(model)?.extract::()?, + &filtering, )?; let (_, uses_intrinsics) = resolve_cursor_intrinsics(py, &sql, None, None)?; - let name: String = model_reads::model_name(&model)?; + let name: String = model_reads::model_name(model)?; if uses_intrinsics { if bounds.is_none() { - let cause = args.value("full_rebuild_cause")?; - let cause = if cause.is_truthy()? { - Some(cause.extract::()?) - } else if args.flag("full_refresh")? { - Some("--full-refresh".to_owned()) - } else { - None + let cause = match &request.full_rebuild_cause { + Some(cause) => Some(cause.clone()), + None if request.full_refresh => Some("--full-refresh".to_owned()), + None => None, }; return Err(unresolved_bounds_error(py, &name, cause.as_deref())?); } sql = intrinsic_bounds(&sql, &bounds, &cursor_type, &resolution.adapter)?; } resolution.assert_resolved(&sql, &format!("Model '{name}'"))?; - Ok(PyString::new(py, &sql).into_any()) + Ok(sql) } /// Effective append cursor lower-bound inclusivity. -fn append_cursor_inclusive<'py>(model: &Bound<'py, PyAny>) -> PyResult> { +fn append_cursor_inclusive(model: &Bound<'_, PyAny>) -> PyResult { let py = model.py(); let strategy = authored_config_str( &model.getattr("config")?.getattr("values")?, @@ -251,9 +232,7 @@ fn append_cursor_inclusive<'py>(model: &Bound<'py, PyAny>) -> PyResult() || raw.is_truthy()?; - Ok(inclusive.into_pyobject(py)?.to_owned().into_any()) + Ok(!strategy.eq("append")? || !raw.is_instance_of::() || raw.is_truthy()?) } /// `__cursor_start()` / `__cursor_end()` replaced with the adapter's typed bound literals. @@ -325,7 +304,6 @@ fn resolve_function_sql<'py>( external: py.None().into_bound(py), source_map: args.value("source_map")?, source_warehouse_columns: args.value("source_warehouse_columns")?, - star_exclude_keyword: args.value("star_exclude_keyword")?, model_locations: args.value("model_locations")?, seed_locations: args.value("seed_locations")?, function_locations: args.value("function_locations")?, @@ -339,13 +317,21 @@ fn resolve_function_sql<'py>( #[pyfunction] #[pyo3(signature = (*, model, adapter, context, external_sql_reference_resolver))] -pub(crate) fn resolve_model_hook_entries<'py>( +fn resolve_model_hook_entries<'py>( model: &Bound<'py, PyAny>, adapter: Bound<'py, PyAny>, context: Bound<'py, PyAny>, external_sql_reference_resolver: Bound<'py, PyAny>, ) -> PyResult> { let resolution = Resolution::of_context(adapter, &context, external_sql_reference_resolver)?; + model_hook_entries(model, &resolution) +} + +/// One model's pre and post hooks with their SQL resolved. +pub(crate) fn model_hook_entries<'py>( + model: &Bound<'py, PyAny>, + resolution: &Resolution<'py>, +) -> PyResult> { let values = model.getattr("config")?.getattr("values")?; let name: String = model_reads::model_name(model)?; let mut hooks: Vec> = Vec::new(); @@ -359,7 +345,7 @@ pub(crate) fn resolve_model_hook_entries<'py>( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resolve_model_sql, module)?)?; + module.add_function(wrap_pyfunction!(resolve_model_sql_of, module)?)?; module.add_function(wrap_pyfunction!(resolve_function_sql, module)?)?; module.add_function(wrap_pyfunction!(resolve_model_hook_entries, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs index 63475c1b12..a614a1269d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/refs.rs @@ -3,7 +3,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::planner_resolution::values::{cursor_wrap, method, string}; +use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; +use crate::bindings::_helpers::planner_resolution::values::{ + cursor_wrap, filtering_of, method, string, +}; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObject; @@ -14,10 +17,7 @@ use sqlbuild_sqltext::resolution::main::{ref_alias::ref_alias, reference_spans:: #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_ref_references( - py: Python<'_>, - kwargs: Option>, -) -> PyResult { +fn resolve_ref_references(py: Python<'_>, kwargs: Option>) -> PyResult { let args = KeywordArguments::read( py, "resolve_ref_references", @@ -36,10 +36,27 @@ pub(crate) fn resolve_ref_references( &[], ), )?; - let original: String = args.value("query_sql")?.extract()?; - let mut sql = original.clone(); - for (kind, key) in [("ref", "model_locations"), ("seed", "seed_locations")] { - let locations = args.value(key)?; + resolve_refs( + &args.value("query_sql")?.extract::()?, + ( + &args.value("model_locations")?, + &args.value("seed_locations")?, + ), + &args.value("adapter")?, + &filtering_of(&args)?, + ) +} + +/// `__ref` and `__seed` references replaced with their relations, cursor inputs filtered. +pub(crate) fn resolve_refs( + original: &str, + (model_locations, seed_locations): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + adapter: &Bound<'_, PyAny>, + filtering: &CursorFiltering<'_>, +) -> PyResult { + let py = adapter.py(); + let mut sql = original.to_owned(); + for (kind, locations) in [("ref", model_locations), ("seed", seed_locations)] { let mut output = String::new(); let mut last = 0; for span in reference_spans(&sql, kind) { @@ -50,7 +67,7 @@ pub(crate) fn resolve_ref_references( } else { let relation: String = call_with( &host::entry(py, AdapterHost::ResolveRelationLocationQualifiedName)?, - &[("adapter", args.value("adapter")?), ("location", target)], + &[("adapter", adapter.clone()), ("location", target)], )? .extract()?; let char_end = sql[..span.end].chars().count(); @@ -59,10 +76,10 @@ pub(crate) fn resolve_ref_references( .nth(char_end) .map_or(original.len(), |(offset, _)| offset); output.push_str(&cursor_wrap( - &args, + adapter, + filtering, relation, - &span.name, - !ref_alias(&original, original_end), + (&span.name, !ref_alias(original, original_end)), )?); } last = span.end; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs index 94a90504d0..5433bfbe90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/registration.rs @@ -4,7 +4,6 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::cursor_bounds::register(module)?; - super::cursor_model::register(module)?; super::cursor_overrides::register(module)?; super::cursor_policies::register(module)?; super::cursor_ranges::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs index d6d44f8596..7e9c530b0e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/sources.rs @@ -1,7 +1,10 @@ //! Source contract validation and adapter-controlled projection rendering. use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; -use crate::bindings::_helpers::planner_resolution::values::{cursor_wrap, method, string}; +use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; +use crate::bindings::_helpers::planner_resolution::values::{ + cursor_wrap, filtering_of, method, string, +}; use crate::bindings::_helpers::planner_strategy::values::input_error; use crate::bindings::_helpers::records::ids::ErrorId; use crate::bindings::_helpers::references::source_relations::render_source_relation; @@ -245,16 +248,19 @@ fn render_source_read_relation( render_source_relation(source_entry, Some(adapter)) } -fn resolve_one(args: &KeywordArguments<'_>, name: &str, original: &str) -> PyResult { - let entry = args.value("source_map")?.call_method1("get", (name,))?; +fn resolve_one( + (source_map, warehouse_columns): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + adapter: &Bound<'_, PyAny>, + filtering: &CursorFiltering<'_>, + (name, original): (&str, &str), +) -> PyResult { + let entry = source_map.call_method1("get", (name,))?; if entry.is_none() { return Ok(original.to_owned()); } - let adapter = args.value("adapter")?; + let adapter = adapter.clone(); let mut relation = render_source_read_relation(&adapter, &entry)?; - let columns = args - .value("source_warehouse_columns")? - .call_method1("get", (name,))?; + let columns = warehouse_columns.call_method1("get", (name,))?; let expression = !entry.getattr("expression")?.is_none(); if columns.is_truthy()? && (!expression @@ -274,12 +280,12 @@ fn resolve_one(args: &KeywordArguments<'_>, name: &str, original: &str) -> PyRes if entry.getattr("type_enforcement")?.is_truthy()? { relation = cast_source(&entry, &columns, &adapter, relation)?; } - cursor_wrap(args, relation, name, false) + cursor_wrap(&adapter, filtering, relation, (name, false)) } #[pyfunction] #[pyo3(signature = (**kwargs))] -pub(crate) fn resolve_source_references( +fn resolve_source_references( py: Python<'_>, kwargs: Option>, ) -> PyResult { @@ -302,19 +308,40 @@ pub(crate) fn resolve_source_references( &[], ), )?; - let sql: String = args.value("query_sql")?.extract()?; - let aliases = args - .value("adapter")? + resolve_sources( + &args.value("query_sql")?.extract::()?, + ( + &args.value("source_map")?, + &args.value("source_warehouse_columns")?, + ), + &args.value("adapter")?, + &filtering_of(&args)?, + ) +} + +/// `__source` references replaced with their read relations, cursor inputs filtered; `lookups` holds the source map and warehouse columns. +pub(crate) fn resolve_sources( + sql: &str, + lookups: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + adapter: &Bound<'_, PyAny>, + filtering: &CursorFiltering<'_>, +) -> PyResult { + let aliases = adapter .call_method0("requires_derived_table_aliases")? .is_truthy()?; let mut output = String::new(); let mut last = 0; - for span in reference_spans(&sql, "source") { - let mut relation = resolve_one(&args, &span.name, &sql[span.start..span.end])?; + for span in reference_spans(sql, "source") { + let mut relation = resolve_one( + lookups, + adapter, + filtering, + (&span.name, &sql[span.start..span.end]), + )?; output.push_str(sql.get(last..span.start).unwrap_or_default()); last = span.end; if aliases && relation.trim_start().starts_with('(') { - let (end, alias) = source_alias(&sql, span.end, &span.name); + let (end, alias) = source_alias(sql, span.end, &span.name); relation.push_str(&format!(" AS {alias}")); last = end; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs index 489e2b8256..0bcd5c6a96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs @@ -1,8 +1,8 @@ //! Boundary calls shared by reference and source resolution. use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::records::ids::NativeAttr; -use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; +use crate::bindings::_helpers::planner_resolution::models::CursorFiltering; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use sqlbuild_sqltext::resolution::main::cursor_subquery::cursor_subquery; @@ -21,34 +21,33 @@ pub(super) fn method<'py>( } pub(super) fn cursor_wrap( - arguments: &KeywordArguments<'_>, + adapter: &Bound<'_, PyAny>, + filtering: &CursorFiltering<'_>, relation: String, - name: &str, - inject_alias: bool, + (name, inject_alias): (&str, bool), ) -> PyResult { - let bounds = arguments.value("cursor_bounds")?; + let bounds = &filtering.bounds; if bounds.is_none() { return Ok(relation); } - let column: Option = arguments - .value("cursor_filter_inputs")? + let column: Option = filtering + .filter_inputs .call_method1("get", (name,))? .extract()?; let Some(column) = column else { return Ok(relation); }; - let adapter = arguments.value("adapter")?; let py = adapter.py(); - let cursor_type = arguments.value("cursor_type")?; + let cursor_type = &filtering.cursor_type; let literal = |bound: &str| -> PyResult { - let token = call_with( - ®istry::native(py, NativeAttr::CursorSentinelToToken)?, - &[("sentinel", bounds.getattr(bound)?)], - )?; + let token = sentinel_to_token(&bounds.getattr(bound)?)?; method( - &adapter, + adapter, "render_cursor_bound_literal", - &[("value", token), ("cursor_type", cursor_type.clone())], + &[ + ("value", string(py, &token)), + ("cursor_type", cursor_type.clone()), + ], )? .extract() }; @@ -64,9 +63,21 @@ pub(super) fn cursor_wrap( column: &column, start: &start, end: &end, - lower_inclusive: arguments.value("lower_bound_inclusive")?.is_truthy()?, + lower_inclusive: filtering.lower_bound_inclusive, typed: !cursor_type.is_none(), }, alias, )) } + +/// The cursor filtering of a keyword-argument resolution request. +pub(super) fn filtering_of<'py>( + arguments: &KeywordArguments<'py>, +) -> PyResult> { + Ok(CursorFiltering { + bounds: arguments.value("cursor_bounds")?, + filter_inputs: arguments.value("cursor_filter_inputs")?, + cursor_type: arguments.value("cursor_type")?, + lower_bound_inclusive: arguments.value("lower_bound_inclusive")?.is_truthy()?, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs index 9e7835d0c6..d22cd1591b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/cli.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_scenario::constants::SCENARIO_PLAN_GRAPH_VALIDATION; use crate::bindings::_helpers::planner_scenario::graph::plan_scenario_graph; +use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; use crate::bindings::_helpers::planner_scenario::relations::build_scenario_relation_plan; use crate::bindings::_helpers::planner_scenario::values::string; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; @@ -82,21 +83,14 @@ pub(crate) fn build_cli_scenario_plan<'py>( ], )?; let (database, schema) = namespace(&project, &graph)?; - let relation_args = PyDict::new(py); - for (name, value) in [ - ("project", project.clone()), - ("graph_plan", graph.clone()), - ("relation_map", map), - ( - "render_qualified_name", - adapter.getattr("render_qualified_name")?, - ), - ("database", database), - ("schema", schema), - ] { - relation_args.set_item(name, value)?; - } - let relations = build_scenario_relation_plan(py, Some(relation_args))?; + let relations = build_scenario_relation_plan(&ScenarioRelations { + project: project.clone(), + graph_plan: graph.clone(), + relation_map: map, + render_qualified_name: adapter.getattr("render_qualified_name")?, + database, + schema, + })?; let result = call_with( ®istry::native(py, NativeAttr::BuildScenarioExecutionPlan)?, &[ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs index 0c97f32fd3..70ff75b823 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/models.rs @@ -9,3 +9,14 @@ pub(crate) struct SqlContext<'py> { pub syntax: Bound<'py, PyAny>, pub label: String, } + +/// One scenario's graph and the relation map and qualification that locate its relations. +#[derive(Debug)] +pub(crate) struct ScenarioRelations<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) graph_plan: Bound<'py, PyAny>, + pub(crate) relation_map: Bound<'py, PyAny>, + pub(crate) render_qualified_name: Bound<'py, PyAny>, + pub(crate) database: Bound<'py, PyAny>, + pub(crate) schema: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs index a6e7cbe60c..9abe33e278 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/registration.rs @@ -54,7 +54,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { )?)?; module.add_function(wrap_pyfunction!(graph::plan_scenario_graph, module)?)?; module.add_function(wrap_pyfunction!( - relations::build_scenario_relation_plan, + relations::build_scenario_relation_plan_of, module )?)?; module.add_function(wrap_pyfunction!(cli::build_cli_scenario_plan, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs index 07a8992cd9..f26919e2b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_scenario/relations.rs @@ -7,15 +7,16 @@ use crate::bindings::_helpers::planner_scenario::constants::{REF, SOURCE}; use crate::bindings::_helpers::planner_scenario::constants::{ SCENARIO_PLAN_INTERNAL, SCENARIO_PLAN_MISSING_RELATION_TARGET, }; +use crate::bindings::_helpers::planner_scenario::models::ScenarioRelations; use crate::bindings::_helpers::planner_scenario::values::error; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; -#[pyfunction] +#[pyfunction(name = "build_scenario_relation_plan")] #[pyo3(signature = (**kwargs))] -pub(crate) fn build_scenario_relation_plan<'py>( +pub(crate) fn build_scenario_relation_plan_of<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -33,13 +34,24 @@ pub(crate) fn build_scenario_relation_plan<'py>( &["database", "schema"], ), )?; - let graph = args.value("graph_plan")?; + build_scenario_relation_plan(&ScenarioRelations { + project: args.value("project")?, + graph_plan: args.value("graph_plan")?, + relation_map: args.value("relation_map")?, + render_qualified_name: args.value("render_qualified_name")?, + database: args.value("database")?, + schema: args.value("schema")?, + }) +} + +/// The scenario's model, seed, source and fixture locations under its relation map. +pub(crate) fn build_scenario_relation_plan<'py>( + args: &ScenarioRelations<'py>, +) -> PyResult> { + let py = args.project.py(); + let graph = &args.graph_plan; let artifacts = PyDict::new(py); - for artifact in args - .value("relation_map")? - .getattr("artifacts")? - .try_iter()? - { + for artifact in args.relation_map.getattr("artifacts")?.try_iter()? { let artifact = artifact?; let identity = artifact.getattr("identity")?; artifacts.set_item( @@ -48,7 +60,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( )?; } let sources = PyDict::new(py); - for source in args.value("project")?.getattr("sources")?.try_iter()? { + for source in args.project.getattr("sources")?.try_iter()? { let source = source?; sources.set_item(source.getattr("name")?, source.getattr("source_entry")?)?; } @@ -68,7 +80,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( ] { for name in graph.getattr(field)?.try_iter()? { let name: String = name?.extract()?; - let target = target_for_artifact(&args, &artifacts, kind, &name)?; + let target = target_for_artifact(args, &artifacts, kind, &name)?; locations.set_item(&name, &target)?; if kind == REF { models.set_item(&name, &target)?; @@ -105,7 +117,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( ®istry::class(py, RecordId::ScenarioRelationPlan)?, &[ ("scenario_name", graph.getattr("name")?), - ("relation_map", args.value("relation_map")?), + ("relation_map", args.relation_map.clone()), ("model_locations", models.into_any()), ("seed_locations", seeds.into_any()), ("project_source_map", sources.into_any()), @@ -119,7 +131,7 @@ pub(crate) fn build_scenario_relation_plan<'py>( } fn target_for_artifact<'py>( - args: &KeywordArguments<'py>, + args: &ScenarioRelations<'py>, artifacts: &Bound<'py, PyDict>, kind: &str, name: &str, @@ -133,10 +145,10 @@ fn target_for_artifact<'py>( Some("This is likely a SQLBuild bug. Please file an issue with the scenario name."), )?); }; - let database = args.value("database")?; - let schema = args.value("schema")?; + let database = args.database.clone(); + let schema = args.schema.clone(); let qualified = call_with( - &args.value("render_qualified_name")?, + &args.render_qualified_name, &[ ("database", database.clone()), ("schema", schema.clone()), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs index 6c190d691c..4e5f95a646 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/audit_entries.rs @@ -15,24 +15,25 @@ use crate::bindings::_helpers::compile_driver::python_calls::{call_with, importe use crate::bindings::_helpers::planner_strategy::constants::{ AUDIT_TYPES_MODULE, COMPILE_TYPES_MODULE, MODEL_ATTACHMENT, SOURCE_ATTACHMENT, }; +use crate::bindings::_helpers::planner_strategy::models::AuditPlanning; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; /// Render and validate one audit query with the supplied adapter callbacks. fn render_query<'py>( - request: &Request<'_, 'py>, + request: &AuditPlanning<'py>, sql: Bound<'py, PyAny>, context: &str, ) -> PyResult> { - let py = request.py(); + let py = sql.py(); let resolved = call_with( ®istry::native(py, NativeAttr::RenderAuditSql)?, &[ ("unresolved_sql", sql), - ("model_locations", request.get("model_locations")?), - ("seed_locations", request.get("seed_locations")?), - ("source_map", request.get("source_map")?), - ("adapter", request.get("adapter")?), + ("model_locations", request.model_locations.clone()), + ("seed_locations", request.seed_locations.clone()), + ("source_map", request.source_map.clone()), + ("adapter", request.adapter.clone()), ], )?; call_with( @@ -62,15 +63,36 @@ fn audit_enum<'py>( } /// Build an audit entry while retaining the public Python plan dataclass. -#[pyfunction] +#[pyfunction(name = "plan_audit")] #[pyo3(signature = (**request))] -pub(crate) fn plan_audit<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { +pub(crate) fn plan_audit_of<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { let request = Request::new("plan_audit", request)?; - let py = request.py(); - let audit = request.get("audit")?; + plan_audit( + &request.get("audit")?, + &AuditPlanning { + model_locations: request.get("model_locations")?, + seed_locations: request.get("seed_locations")?, + source_map: request.get("source_map")?, + adapter: request.get("adapter")?, + model_materializations: request.get("model_materializations")?, + lineage_upstream_deps: request.get("lineage_upstream_deps")?, + lineage_downstream_deps: request.get("lineage_downstream_deps")?, + }, + ) +} + +/// One audit's plan entry, rendered against the planning relations. +fn plan_audit<'py>( + audit: &Bound<'py, PyAny>, + request: &AuditPlanning<'py>, +) -> PyResult> { + let py = audit.py(); + let audit = audit.clone(); let name: String = audit.getattr("name")?.extract()?; let resolved = render_query( - &request, + request, audit.getattr("sql_body")?, &format!("audit '{name}' planned SQL"), )?; @@ -79,19 +101,19 @@ pub(crate) fn plan_audit<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult< py.None().into_bound(py) } else { render_query( - &request, + request, evidence, &format!("audit '{name}' planned evidence SQL"), )? }; let attachment: Bound<'_, PyTuple> = - resolve_attachment_kind(&audit, &request.get("lineage_downstream_deps")?)?; + resolve_attachment_kind(&audit, &request.lineage_downstream_deps)?; let kind = attachment.get_item(0)?; let target = attachment.get_item(1)?; let kind_text: String = kind.extract()?; let materialization = if kind_text == MODEL_ATTACHMENT && !target.is_none() { request - .get("model_materializations")? + .model_materializations .call_method1("get", (&target,))? } else { py.None().into_bound(py) @@ -146,7 +168,7 @@ pub(crate) fn plan_audit<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult< "reads_outside_target_lineage", PyBool::new( py, - reads_outside_target_lineage(&audit, &request.get("lineage_upstream_deps")?)?, + reads_outside_target_lineage(&audit, &request.lineage_upstream_deps)?, ) .to_owned() .into_any(), @@ -210,24 +232,17 @@ pub(crate) fn build_selected_audit_entries<'py>( } let lineage = registry::native(py, NativeAttr::ProjectLineageViews)?.call1((project,))?; let relations = request.get("relations")?; - let arguments = PyDict::new(py); - for field in ["model_locations", "seed_locations"] { - arguments.set_item(field, relations.getattr(field)?)?; - } - arguments.set_item("source_map", relations.getattr("source_read_map")?)?; - arguments.set_item("adapter", request.get("adapter")?)?; - arguments.set_item( - "model_materializations", - request.get("model_materializations")?, - )?; - arguments.set_item("lineage_upstream_deps", lineage.getattr("upstream_deps")?)?; - arguments.set_item( - "lineage_downstream_deps", - lineage.getattr("downstream_deps")?, - )?; + let planning = AuditPlanning { + model_locations: relations.getattr("model_locations")?, + seed_locations: relations.getattr("seed_locations")?, + source_map: relations.getattr("source_read_map")?, + adapter: request.get("adapter")?, + model_materializations: request.get("model_materializations")?, + lineage_upstream_deps: lineage.getattr("upstream_deps")?, + lineage_downstream_deps: lineage.getattr("downstream_deps")?, + }; for audit in selected { - arguments.set_item("audit", audit)?; - entries.append(plan_audit(Some(&arguments))?)?; + entries.append(plan_audit(&audit, &planning)?)?; } Ok(entries) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs index 2497ab3b8b..aa89cdde7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_inputs.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::planner_strategy::constants::{ SOURCE_REFERENCE, TIMESTAMP_CURSOR, }; use crate::bindings::_helpers::planner_strategy::lineage::resolve_lineage_reference; +use crate::bindings::_helpers::planner_strategy::models::CursorInputLookups; use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; @@ -58,32 +59,61 @@ fn check_column(check: CursorInputColumnCheck, py: Python<'_>) -> PyResult<()> { } } +/// Cursor input lookups from keyword arguments, empty where the caller left one out. +fn lookups_of<'py>(request: &Request<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.py(); + let lookup = |name: &str| or_empty(request.optional(name)?, py); + Ok(CursorInputLookups { + models_by_name: lookup("models_by_name")?, + functions_by_name: lookup("functions_by_name")?, + source_map: lookup("source_map")?, + source_warehouse_columns: lookup("source_warehouse_columns")?, + model_locations: lookup("model_locations")?, + seed_locations: lookup("seed_locations")?, + runtime_cursor_producer_names: match request.optional("runtime_cursor_producer_names")? { + Some(names) => names, + None => PyFrozenSet::empty(py)?.into_any(), + }, + }) +} + /// Validate authored cursor input columns against contracts, then source warehouse metadata. #[pyfunction(name = "validate_source_cursor_input_columns")] #[pyo3(signature = (**request))] +pub(crate) fn cursor_input_columns_of<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult<()> { + let request = Request::new("validate_source_cursor_input_columns", request)?; + request.get("source_warehouse_columns")?; + reject_unexposed_cursor_input_columns( + &request.get("model")?, + &request.get("cursor_column")?, + &lookups_of(&request)?, + ) +} + +/// Reject authored cursor input columns a contract or the source warehouse metadata does not expose. pub(crate) fn reject_unexposed_cursor_input_columns<'py>( - request: Option<&Bound<'py, PyDict>>, + model: &Bound<'py, PyAny>, + cursor_column: &Bound<'py, PyAny>, + lookups: &CursorInputLookups<'py>, ) -> PyResult<()> { - let request = Request::new("validate_source_cursor_input_columns", request)?; - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; - if !incremental_cursor_flag(&model, &request.get("cursor_column")?)? { + let py: Python<'py> = model.py(); + if !incremental_cursor_flag(model, cursor_column)? { return Ok(()); } - let roles: Bound<'py, PyAny> = resolve_cursor_input_roles(&model)?; - let models_by_name = or_empty(request.optional("models_by_name")?, py)?; - let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; - let source_map = or_empty(request.optional("source_map")?, py)?; - let warehouse_columns: Bound<'py, PyAny> = request.get("source_warehouse_columns")?; - let model_name: String = model_reads::model_name(&model)?; + let model: &Bound<'py, PyAny> = model; + let roles: Bound<'py, PyAny> = resolve_cursor_input_roles(model)?; + let models_by_name: &Bound<'py, PyAny> = &lookups.models_by_name; + let source_map: &Bound<'py, PyAny> = &lookups.source_map; + let warehouse_columns: &Bound<'py, PyAny> = &lookups.source_warehouse_columns; + let model_name: String = model_reads::model_name(model)?; for role in ["filter_inputs", "watermark_inputs"] { for item in roles.getattr(role)?.call_method0("items")?.try_iter()? { let (input_name, column): (String, String) = item?.extract()?; let reference: Bound<'py, PyAny> = resolve_lineage_reference( - &model, + model, &input_name, models_by_name.clone(), - functions_by_name.clone(), + lookups.functions_by_name.clone(), )?; let kind: String = text(&reference.getattr("ref_kind")?)?; let name: String = text(&reference.getattr("ref_name")?)?; @@ -136,34 +166,26 @@ pub(crate) fn reject_unexposed_cursor_input_columns<'py>( /// The qualified relation one cursor input reference reads, when it resolves. fn input_relation<'py>( - request: &Request<'_, 'py>, + (lookups, adapter): (&CursorInputLookups<'py>, &Bound<'py, PyAny>), (kind, name): (&str, &str), ) -> PyResult> { - let py: Python<'py> = request.py(); + let py: Python<'py> = adapter.py(); let location: Bound<'py, PyAny> = match kind { MODEL_REFERENCE => { - let model: Bound<'py, PyAny> = request - .get("model_locations")? - .call_method1("get", (name,))?; + let model: Bound<'py, PyAny> = lookups.model_locations.call_method1("get", (name,))?; if model.is_none() { - request - .get("seed_locations")? - .call_method1("get", (name,))? + lookups.seed_locations.call_method1("get", (name,))? } else { model } } - SEED_REFERENCE => request - .get("seed_locations")? - .call_method1("get", (name,))?, + SEED_REFERENCE => lookups.seed_locations.call_method1("get", (name,))?, SOURCE_REFERENCE => { - let source: Bound<'py, PyAny> = - request.get("source_map")?.call_method1("get", (name,))?; + let source: Bound<'py, PyAny> = lookups.source_map.call_method1("get", (name,))?; if source.is_none() { return Ok(None); } - let adapter: Bound<'py, PyAny> = request.get("adapter")?; - return render_source_relation(&source, Some(&adapter)).map(Some); + return render_source_relation(&source, Some(adapter)).map(Some); } _ => py.None().into_bound(py), }; @@ -203,21 +225,44 @@ fn terminal_bound<'py>( } /// Cursor-bearing input relations of an incremental model, for runtime range discovery. -#[pyfunction] +#[pyfunction(name = "build_cursor_input_relations")] #[pyo3(signature = (**request))] -pub(crate) fn build_cursor_input_relations<'py>( +pub(crate) fn cursor_input_relations_of<'py>( request: Option<&Bound<'py, PyDict>>, ) -> PyResult> { let request = Request::new("build_cursor_input_relations", request)?; - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; - if !incremental_cursor_flag(&model, &request.get("cursor_column")?)? { + for name in [ + "adapter", + "model_locations", + "models_by_name", + "seed_locations", + "source_map", + "runtime_cursor_producer_names", + ] { + request.get(name)?; + } + build_cursor_input_relations( + (&request.get("model")?, &request.get("adapter")?), + &request.get("cursor_column")?, + &lookups_of(&request)?, + ) +} + +/// Cursor-bearing input relations of an incremental model, for runtime range discovery. +pub(crate) fn build_cursor_input_relations<'py>( + (model, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + cursor_column: &Bound<'py, PyAny>, + lookups: &CursorInputLookups<'py>, +) -> PyResult> { + let py: Python<'py> = model.py(); + if !incremental_cursor_flag(model, cursor_column)? { return Ok(PyTuple::empty(py)); } - let models_by_name: Bound<'py, PyAny> = request.get("models_by_name")?; - let functions_by_name = or_empty(request.optional("functions_by_name")?, py)?; - let model_locations: Bound<'py, PyAny> = request.get("model_locations")?; - let producers: Bound<'py, PyAny> = request.get("runtime_cursor_producer_names")?; + let model: Bound<'py, PyAny> = model.clone(); + let models_by_name: &Bound<'py, PyAny> = &lookups.models_by_name; + let functions_by_name: &Bound<'py, PyAny> = &lookups.functions_by_name; + let model_locations: &Bound<'py, PyAny> = &lookups.model_locations; + let producers: &Bound<'py, PyAny> = &lookups.runtime_cursor_producer_names; let cursor_type: Bound<'py, PyAny> = match config_str(&model_values(&model)?, "cursor_type")? { Some(cursor_type) if cursor_type.is_truthy()? => cursor_type, _ => PyString::new(py, TIMESTAMP_CURSOR).into_any(), @@ -237,7 +282,7 @@ pub(crate) fn build_cursor_input_relations<'py>( let kind: String = text(&reference.getattr("ref_kind")?)?; let name: Bound<'py, PyAny> = reference.getattr("ref_name")?; let name_text: String = text(&name)?; - let Some(relation) = input_relation(&request, (&kind, &name_text))? else { + let Some(relation) = input_relation((lookups, adapter), (&kind, &name_text))? else { continue; }; let producer: Option> = if kind == MODEL_REFERENCE { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs index 29fcf254be..ff00ec515e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/cursor_types.rs @@ -69,7 +69,7 @@ fn classify_cursor_sql_type<'py>( /// A `PlanWarning` when the warehouse cursor column type contradicts `cursor_type`. #[pyfunction] #[pyo3(signature = (**options))] -pub(crate) fn check_cursor_type_consistency<'py>( +fn check_cursor_type_consistency<'py>( py: Python<'py>, options: Option>, ) -> PyResult>> { @@ -89,28 +89,41 @@ pub(crate) fn check_cursor_type_consistency<'py>( ), )?; let model_name: Bound<'py, PyAny> = arguments.value("model_name")?; - let mut warehouse_columns: Vec<(String, String)> = Vec::new(); - for column in arguments.value("warehouse_columns")?.try_iter()? { - let column: Bound<'py, PyAny> = column?; - warehouse_columns.push(( - text(&column.getattr("name")?)?, - text(&column.getattr("type")?)?, - )); - } let check = CursorTypeCheck { model_name: text(&model_name)?, cursor_column: optional_string(&arguments.value("cursor_column")?)?, cursor_type: optional_string(&arguments.value("cursor_type")?)?, - warehouse_columns, + warehouse_columns: warehouse_column_types(&arguments.value("warehouse_columns")?)?, sql_analysis_enabled: arguments.flag("sql_analysis_enabled")?, }; - let (warning, parse_failure) = cursor_type_warning(&check); + cursor_type_warning_record(&model_name, &check) +} + +/// The `(name, type)` pairs of warehouse `ColumnInfo`s. +pub(crate) fn warehouse_column_types( + columns: &Bound<'_, PyAny>, +) -> PyResult> { + let mut pairs: Vec<(String, String)> = Vec::new(); + for column in columns.try_iter()? { + let column = column?; + pairs.push(( + text(&column.getattr("name")?)?, + text(&column.getattr("type")?)?, + )); + } + Ok(pairs) +} + +/// The `PlanWarning` of one cursor type check, logging an unclassifiable warehouse type. +pub(crate) fn cursor_type_warning_record<'py>( + model_name: &Bound<'py, PyAny>, + check: &CursorTypeCheck, +) -> PyResult>> { + let (warning, parse_failure) = cursor_type_warning(check); if let Some(failure) = parse_failure { - log_parse_failure(py, &failure)?; + log_parse_failure(model_name.py(), &failure)?; } - warning - .map(|row| plan_warning(&model_name, row)) - .transpose() + warning.map(|row| plan_warning(model_name, row)).transpose() } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 9143134c54..d66e0fb1a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -7,11 +7,10 @@ use pyo3::{IntoPyObject, pyfunction}; use sqlbuild_analysis::planner_strategy::main::microbatch_lookback::microbatch_lookback; use sqlbuild_analysis::planner_strategy::main::table_type_ddl::table_type_ddl; -use sqlbuild_analysis::planner_strategy::models::LookbackConfig; +use sqlbuild_analysis::planner_strategy::models::{CursorTypeCheck, LookbackConfig}; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::cursor::scalars::sentinel_to_token; use crate::bindings::_helpers::model_config::config_values::{ config_bool, config_cursor_bound, config_int, config_str, config_string_tuple, @@ -23,13 +22,16 @@ use crate::bindings::_helpers::planner_strategy::constants::{ CUSTOM_MATERIALIZATION, DDL_SEPARATOR, ENFORCED_CONTRACT, MICROBATCH_MODE, }; use crate::bindings::_helpers::planner_strategy::cursor_inputs::model_values; -use crate::bindings::_helpers::planner_strategy::cursor_types::check_cursor_type_consistency; +use crate::bindings::_helpers::planner_strategy::cursor_types::{ + cursor_type_warning_record, warehouse_column_types, +}; use crate::bindings::_helpers::planner_strategy::microbatch_limits::{replay_grain, unknown_grain}; use crate::bindings::_helpers::planner_strategy::microbatches::microbatch_limit_config; +use crate::bindings::_helpers::planner_strategy::models::ModelEntryDecisions; use crate::bindings::_helpers::planner_strategy::strategy::{ - build_model_warnings, resolve_schema_actions, + model_warning_records, resolve_schema_actions, warning_inputs, }; -use crate::bindings::_helpers::planner_strategy::values::{input_error, text}; +use crate::bindings::_helpers::planner_strategy::values::{input_error, optional_string, text}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; @@ -56,25 +58,6 @@ const STRING_FIELDS: [(&str, &str); 15] = [ ]; /// Model config keys copied as optional strings that complete the snapshot settings. const SNAPSHOT_POLICY_FIELDS: [&str; 2] = ["snapshot_full_refresh", "snapshot_schema_change"]; -/// Plan decisions passed through to the entry unchanged. -const PASSED_FIELDS: [&str; 16] = [ - "action", - "reason", - "materialization_type", - "resolved_sql", - "cursor_column", - "cursor_bounds", - "empty_cursor_inputs", - "waiting_on_empty_inputs", - "cursor_input_relations", - "microbatch_range", - "start_cursor_override", - "end_cursor_override", - "future_cursor_config", - "start_cursor_config", - "invocation_time", - "backfill", -]; /// Change-detection facts copied to the entry. const CHANGE_FIELDS: [&str; 8] = [ "fingerprint_metadata_json", @@ -232,29 +215,29 @@ pub(crate) fn extract_seed_columns<'py>(seed: &Bound<'py, PyAny>) -> PyResult( - request: &Request<'_, 'py>, + request: &ModelEntryDecisions<'py>, (unique_key, merge_exclude): (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>), (warehouse_columns, cursor_bounds): (&Bound<'py, PyAny>, Bound<'py, PyAny>), ) -> PyResult { - let py: Python<'py> = request.py(); - let adapter: Bound<'py, PyAny> = request.get("adapter")?; - let model: Bound<'py, PyAny> = request.get("model")?; - let destination: Bound<'py, PyAny> = model_reads::destination(&model)?; + let py: Python<'py> = request.model.py(); + let adapter: &Bound<'py, PyAny> = &request.adapter; + let model: &Bound<'py, PyAny> = &request.model; + let destination: Bound<'py, PyAny> = model_reads::destination(model)?; let qualified: Bound<'py, PyAny> = destination.getattr("qualified_name")?; let qualified: Bound<'py, PyAny> = if qualified.is_truthy()? { qualified } else { destination.getattr("name")? }; - let sql: Bound<'py, PyAny> = request.get("resolved_sql")?; + let sql: Bound<'py, PyAny> = request.resolved_sql.clone(); let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = vec![("destination", qualified), ("sql", sql)]; - let cursor_column: Bound<'py, PyAny> = request.get("cursor_column")?; - let method: &str = match text(&request.get("action")?)?.as_str() { + let cursor_column: Bound<'py, PyAny> = request.cursor_column.clone(); + let method: &str = match text(&request.action)?.as_str() { "create_view" => "render_create_view_as", "create_table" => "render_create_table_as", "incremental_append" => "render_append", "incremental_delete_insert" if !cursor_column.is_none() && !cursor_bounds.is_none() => { - let values: Bound<'py, PyAny> = model_values(&model)?; + let values: Bound<'py, PyAny> = model_values(model)?; let start = sentinel_to_token(&cursor_bounds.getattr("start")?)?; let end = sentinel_to_token(&cursor_bounds.getattr("end")?)?; pairs.push(("cursor_column", cursor_column)); @@ -331,59 +314,42 @@ fn custom_fields<'py>( /// The model's base warnings followed by its cursor type warning. fn model_warnings<'py>( - request: &Request<'_, 'py>, + request: &ModelEntryDecisions<'py>, (schema_actions, policy): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (enforcement, warehouse_columns): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; - let mut warnings: Vec> = build_model_warnings( - py, - Some(keywords( - py, - &[ - ("model_name", name.clone()), - ("materialization_type", request.get("materialization_type")?), - ("change_result", request.get("change_result")?), - ("schema_actions", schema_actions.clone()), - ("on_schema_change", policy.clone()), - ("type_enforcement", enforcement.clone()), - ], - )?), - )? - .try_iter()? - .collect::>()?; - let cursor_warning: Option> = check_cursor_type_consistency( - py, - Some(keywords( - py, - &[ - ("model_name", name), - ("cursor_column", request.get("cursor_column")?), - ( - "cursor_type", - or_none(py, config_str(&model_values(&model)?, "cursor_type")?), - ), - ("warehouse_columns", warehouse_columns.clone()), - ("sql_analysis_enabled", request.get("sql_analysis_enabled")?), - ], - )?), + let py: Python<'py> = request.model.py(); + let model: &Bound<'py, PyAny> = &request.model; + let name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let inputs = warning_inputs( + &name, + &request.materialization_type, + (schema_actions, policy, enforcement), )?; - if let Some(cursor_warning) = cursor_warning.filter(|warning| !warning.is_none()) { + let mut warnings: Vec> = + model_warning_records(&name, &inputs, &request.change_result)?; + let check = CursorTypeCheck { + model_name: text(&name)?, + cursor_column: optional_string(&request.cursor_column)?, + cursor_type: optional_string(&or_none( + py, + config_str(&model_values(model)?, "cursor_type")?, + ))?, + warehouse_columns: warehouse_column_types(warehouse_columns)?, + sql_analysis_enabled: request.sql_analysis_enabled, + }; + if let Some(cursor_warning) = cursor_type_warning_record(&name, &check)? { warnings.push(cursor_warning); } PyTuple::new(py, warnings) } /// The cursor bounds the logical DDL renders: placeholders for runtime-owned windows, else the batch range or cursor bounds. -fn ddl_cursor_bounds<'py>(request: &Request<'_, 'py>) -> PyResult> { - let py: Python<'py> = request.py(); - let microbatch_range: Bound<'py, PyAny> = request.get("microbatch_range")?; +fn ddl_cursor_bounds<'py>(request: &ModelEntryDecisions<'py>) -> PyResult> { + let py: Python<'py> = request.model.py(); + let microbatch_range: &Bound<'py, PyAny> = &request.microbatch_range; Ok( - if request.get("runtime_owned_cursor_bounds")?.is_truthy()? - && !request.get("cursor_column")?.is_none() - { + if request.runtime_owned_cursor_bounds && !request.cursor_column.is_none() { let sentinel: Bound<'py, PyAny> = host::entry(py, ExecutorHost::BoundSentinel)?; call_with( ®istry::class(py, RecordId::CursorBounds)?, @@ -395,19 +361,19 @@ fn ddl_cursor_bounds<'py>(request: &Request<'_, 'py>) -> PyResult( - request: &Request<'_, 'py>, + request: &ModelEntryDecisions<'py>, values: &Bound<'py, PyAny>, entry: &Bound<'py, PyDict>, ) -> PyResult<()> { - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; + let py: Python<'py> = request.model.py(); + let model: &Bound<'py, PyAny> = &request.model; for (key, field) in STRING_FIELDS { entry.set_item(field, or_none(py, config_str(values, key)?))?; } @@ -433,9 +399,9 @@ fn config_fields<'py>( read("microbatch_strategy")?, read("cursor_grain")?, ), - &request.get("cursor_input_relations")?, + &request.cursor_input_relations, )?; - lookback_of(&model, grain)? + lookback_of(model, grain)? } else { config_str(values, "lookback")? .map(|value| text(&value)) @@ -489,28 +455,62 @@ fn contract_fields<'py>( Ok(()) } -/// Build a model plan entry and its warnings from resolved plan decisions, keeping the Python dataclasses. -#[pyfunction] -#[pyo3(signature = (**request))] +/// Set the plan decisions passed through to the entry unchanged. +fn passed_fields<'py>( + request: &ModelEntryDecisions<'py>, + entry: &Bound<'py, PyDict>, +) -> PyResult<()> { + let py: Python<'py> = request.model.py(); + let flag = |value: bool| PyBool::new(py, value).to_owned().into_any(); + let fields: [(&str, Bound<'py, PyAny>); 16] = [ + ("action", request.action.clone()), + ("reason", request.reason.clone()), + ("materialization_type", request.materialization_type.clone()), + ("resolved_sql", request.resolved_sql.clone()), + ("cursor_column", request.cursor_column.clone()), + ("cursor_bounds", request.cursor_bounds.clone()), + ("empty_cursor_inputs", request.empty_cursor_inputs.clone()), + ( + "waiting_on_empty_inputs", + flag(request.waiting_on_empty_inputs), + ), + ( + "cursor_input_relations", + request.cursor_input_relations.clone(), + ), + ("microbatch_range", request.microbatch_range.clone()), + ("start_cursor_override", request.cursor_overrides.0.clone()), + ("end_cursor_override", request.cursor_overrides.1.clone()), + ("future_cursor_config", request.cursor_configs.0.clone()), + ("start_cursor_config", request.cursor_configs.1.clone()), + ("invocation_time", request.invocation_time.clone()), + ("backfill", request.backfill.clone()), + ]; + for (field, value) in fields { + entry.set_item(field, value)?; + } + Ok(()) +} + +/// Build a model plan entry and its warnings from resolved plan decisions. pub(crate) fn assemble_model_plan_entry<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("assemble_model_plan_entry", request)?; - let py: Python<'py> = request.py(); - let model: Bound<'py, PyAny> = request.get("model")?; - let values: Bound<'py, PyAny> = model_values(&model)?; - let change: Bound<'py, PyAny> = request.get("change_result")?; - let snapshot: Bound<'py, PyAny> = request.get("snapshot")?; - let name: Bound<'py, PyAny> = model_reads::name_object(&model)?; - let policy: Bound<'py, PyAny> = on_schema_change(&model)?; + request: &ModelEntryDecisions<'py>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { + let py: Python<'py> = request.model.py(); + let model: &Bound<'py, PyAny> = &request.model; + let values: Bound<'py, PyAny> = model_values(model)?; + let change: &Bound<'py, PyAny> = &request.change_result; + let snapshot: &Bound<'py, PyAny> = &request.snapshot; + let name: Bound<'py, PyAny> = model_reads::name_object(model)?; + let policy: Bound<'py, PyAny> = on_schema_change(model)?; let schema_actions: Bound<'py, PyAny> = resolve_schema_actions(&change.getattr("schema_findings")?, &policy)?.into_any(); - let enforcement: Bound<'py, PyAny> = type_enforcement(&model)?; + let enforcement: Bound<'py, PyAny> = type_enforcement(model)?; let warehouse_columns: Bound<'py, PyAny> = snapshot .getattr("existing_columns")? .call_method1("get", (&name, PyTuple::empty(py)))?; let warnings: Bound<'py, PyTuple> = model_warnings( - &request, + request, (&schema_actions, &policy), (&enforcement, &warehouse_columns), )?; @@ -523,33 +523,30 @@ pub(crate) fn assemble_model_plan_entry<'py>( .getattr("value")? .getattr("value")?; let ddl: String = logical_ddl( - &request, + request, (&unique_key, &merge_exclude), - (&warehouse_columns, ddl_cursor_bounds(&request)?), + (&warehouse_columns, ddl_cursor_bounds(request)?), )?; let entry: Bound<'py, PyDict> = PyDict::new(py); - entry.set_item("key", model_reads::model_key(&model)?)?; + entry.set_item("key", model_reads::model_key(model)?)?; entry.set_item("name", &name)?; entry.set_item("relative_path", model.getattr("relative_path")?)?; - entry.set_item("destination", model_reads::destination(&model)?)?; - entry.set_item("fingerprint_query_sql", model_reads::query_sql(&model)?)?; + entry.set_item("destination", model_reads::destination(model)?)?; + entry.set_item("fingerprint_query_sql", model_reads::query_sql(model)?)?; entry.set_item("logical_ddl", table_type_ddl(&ddl, &text(&table_type)?))?; - for field in PASSED_FIELDS { - entry.set_item(field, request.get(field)?)?; - } + passed_fields(request, &entry)?; for field in CHANGE_FIELDS { entry.set_item(field, change.getattr(field)?)?; } - config_fields(&request, &values, &entry)?; + config_fields(request, &values, &entry)?; entry.set_item("unique_key", unique_key)?; entry.set_item("merge_exclude_columns", merge_exclude)?; entry.set_item("table_type", table_type)?; entry.set_item("on_schema_change", policy)?; entry.set_item("type_enforcement", enforcement)?; - contract_fields(&model, &values, &entry)?; - for field in ["pre_hooks", "post_hooks"] { - entry.set_item(field, request.get(field)?)?; - } + contract_fields(model, &values, &entry)?; + entry.set_item("pre_hooks", &request.hooks.0)?; + entry.set_item("post_hooks", &request.hooks.1)?; let fingerprint: Bound<'py, PyAny> = snapshot .getattr("fingerprints")? .getattr("models")? @@ -565,10 +562,10 @@ pub(crate) fn assemble_model_plan_entry<'py>( entry.set_item("schema_actions", schema_actions)?; custom_fields( &values, - text(&request.get("materialization_type")?)? == CUSTOM_MATERIALIZATION, + text(&request.materialization_type)? == CUSTOM_MATERIALIZATION, &entry, )?; let plan_entry: Bound<'py, PyAny> = registry::class(py, RecordId::ModelPlanEntry)?.call((), Some(&entry))?; - PyTuple::new(py, [plan_entry, warnings.into_any()]) + Ok((plan_entry, warnings)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs index 6c998ddebe..cc125e3fc3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/models.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeSet, HashMap}; use std::sync::{Arc, Mutex}; -use pyo3::prelude::Py; +use pyo3::prelude::{Bound, Py}; use pyo3::pyclass; use pyo3::types::{PyAny, PyDict}; use sqlbuild_analysis::planner_fixtures::models::{ @@ -56,3 +56,53 @@ pub(crate) struct ExpectedTypes<'a> { pub(crate) overlay: Vec<(FixtureKey, Vec<(String, String)>)>, pub(crate) base: &'a HashMap>, } + +/// One model's resolved plan decisions, assembled into its `ModelPlanEntry`. +#[derive(Debug)] +pub(crate) struct ModelEntryDecisions<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) change_result: Bound<'py, PyAny>, + pub(crate) action: Bound<'py, PyAny>, + pub(crate) reason: Bound<'py, PyAny>, + pub(crate) materialization_type: Bound<'py, PyAny>, + pub(crate) resolved_sql: Bound<'py, PyAny>, + pub(crate) sql_analysis_enabled: bool, + pub(crate) hooks: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) cursor_column: Bound<'py, PyAny>, + pub(crate) cursor_input_relations: Bound<'py, PyAny>, + pub(crate) runtime_owned_cursor_bounds: bool, + pub(crate) cursor_bounds: Bound<'py, PyAny>, + pub(crate) microbatch_range: Bound<'py, PyAny>, + pub(crate) empty_cursor_inputs: Bound<'py, PyAny>, + pub(crate) waiting_on_empty_inputs: bool, + pub(crate) backfill: Bound<'py, PyAny>, + pub(crate) cursor_overrides: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) cursor_configs: (Bound<'py, PyAny>, Bound<'py, PyAny>), + pub(crate) invocation_time: Bound<'py, PyAny>, +} + +/// The project lookups that resolve one model's cursor inputs. +#[derive(Debug)] +pub(crate) struct CursorInputLookups<'py> { + pub(crate) models_by_name: Bound<'py, PyAny>, + pub(crate) functions_by_name: Bound<'py, PyAny>, + pub(crate) source_map: Bound<'py, PyAny>, + pub(crate) source_warehouse_columns: Bound<'py, PyAny>, + pub(crate) model_locations: Bound<'py, PyAny>, + pub(crate) seed_locations: Bound<'py, PyAny>, + pub(crate) runtime_cursor_producer_names: Bound<'py, PyAny>, +} + +/// The relations and lineage that render and schedule selected audits. +#[derive(Debug)] +pub(crate) struct AuditPlanning<'py> { + pub(crate) model_locations: Bound<'py, PyAny>, + pub(crate) seed_locations: Bound<'py, PyAny>, + pub(crate) source_map: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) model_materializations: Bound<'py, PyAny>, + pub(crate) lineage_upstream_deps: Bound<'py, PyAny>, + pub(crate) lineage_downstream_deps: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs index 89538ed245..26e4c400b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/registration.rs @@ -8,29 +8,28 @@ use crate::bindings::_helpers::planner_strategy::{ }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(super::audit_entries::plan_audit, module)?)?; module.add_function(wrap_pyfunction!( - super::audit_entries::audit_is_selected, + super::audit_entries::plan_audit_of, module )?)?; module.add_function(wrap_pyfunction!( - super::audit_entries::build_selected_audit_entries, + super::audit_entries::audit_is_selected, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::reject_unexposed_cursor_input_columns, + super::audit_entries::build_selected_audit_entries, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::build_cursor_input_relations, + super::cursor_inputs::cursor_input_columns_of, module )?)?; module.add_function(wrap_pyfunction!( - super::cursor_inputs::runtime_cursor_producer_names, + super::cursor_inputs::cursor_input_relations_of, module )?)?; module.add_function(wrap_pyfunction!( - super::model_entries::assemble_model_plan_entry, + super::cursor_inputs::runtime_cursor_producer_names, module )?)?; module.add_function(wrap_pyfunction!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs index ad4e0aa5eb..f3057d8e3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/strategy.rs @@ -138,10 +138,41 @@ pub(crate) fn resolve_schema_actions<'py>( PyTuple::new(py, actions) } +/// The warning inputs of one model's plan decisions. +pub(crate) fn warning_inputs( + model_name: &Bound<'_, PyAny>, + materialization: &Bound<'_, PyAny>, + (schema_actions, on_schema_change, type_enforcement): ( + &Bound<'_, PyAny>, + &Bound<'_, PyAny>, + &Bound<'_, PyAny>, + ), +) -> PyResult { + Ok(WarningInputs { + model_name: text(model_name)?, + materialization: materialization_type(Some(&text(materialization)?)), + has_schema_actions: schema_actions.is_truthy()?, + on_schema_change: schema_policy(on_schema_change)?, + type_enforcement: type_enforcement.is_truthy()?, + }) +} + +/// The `PlanWarning`s of one model's warning inputs and change result. +pub(crate) fn model_warning_records<'py>( + model_name: &Bound<'py, PyAny>, + inputs: &WarningInputs, + change_result: &Bound<'py, PyAny>, +) -> PyResult>> { + model_warnings(inputs, &change_facts(change_result)?) + .into_iter() + .map(|row| plan_warning(model_name, row)) + .collect() +} + /// Warnings for one model from its change detection and plan decisions. #[pyfunction] #[pyo3(signature = (**options))] -pub(crate) fn build_model_warnings<'py>( +fn build_model_warnings<'py>( py: Python<'py>, options: Option>, ) -> PyResult> { @@ -162,21 +193,19 @@ pub(crate) fn build_model_warnings<'py>( ), )?; let model_name: Bound<'py, PyAny> = arguments.value("model_name")?; - let inputs = WarningInputs { - model_name: text(&model_name)?, - materialization: materialization_type(Some(&text( - &arguments.value("materialization_type")?, - )?)), - has_schema_actions: arguments.flag("schema_actions")?, - on_schema_change: schema_policy(&arguments.value("on_schema_change")?)?, - type_enforcement: arguments.flag("type_enforcement")?, - }; - let rows = model_warnings(&inputs, &change_facts(&arguments.value("change_result")?)?); - let warnings: Vec> = rows - .into_iter() - .map(|row| plan_warning(&model_name, row)) - .collect::>()?; - PyTuple::new(py, warnings) + let inputs: WarningInputs = warning_inputs( + &model_name, + &arguments.value("materialization_type")?, + ( + &arguments.value("schema_actions")?, + &arguments.value("on_schema_change")?, + &arguments.value("type_enforcement")?, + ), + )?; + PyTuple::new( + py, + model_warning_records(&model_name, &inputs, &arguments.value("change_result")?)?, + ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs index cbe7d5efed..5d6419d550 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/registration.rs @@ -54,7 +54,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module )?)?; module.add_function(wrap_pyfunction!( - super::source_freshness::build_planner_source_freshness_result, + super::source_freshness::build_planner_source_freshness_result_of, module )?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs index cb56a20b1e..f26f61aa14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/source_freshness.rs @@ -103,9 +103,9 @@ fn state_schemas( Ok(schemas) } -#[pyfunction] +#[pyfunction(name = "build_planner_source_freshness_result")] #[pyo3(signature = (**options))] -pub(crate) fn build_planner_source_freshness_result<'py>( +pub(crate) fn build_planner_source_freshness_result_of<'py>( py: pyo3::Python<'py>, options: Option>, ) -> PyResult> { @@ -118,13 +118,21 @@ pub(crate) fn build_planner_source_freshness_result<'py>( &["freshness_state_schemas"], ), )?; - let project = &args.value("project")?; - let adapter = &args.value("adapter")?; - let connection = &args.value("connection")?; - let scope = &args.value("scope")?; - let relations = &args.value("relations")?; - let freshness_state_schemas = &args.value("freshness_state_schemas")?; + build_planner_source_freshness_result( + (&args.value("project")?, &args.value("scope")?), + (&args.value("adapter")?, &args.value("connection")?), + &args.value("relations")?, + &args.value("freshness_state_schemas")?, + ) +} +/// The source freshness evaluation of one inspection scope against the warehouse. +pub(crate) fn build_planner_source_freshness_result<'py>( + (project, scope): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (adapter, connection): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + relations: &Bound<'py, PyAny>, + freshness_state_schemas: &Bound<'py, PyAny>, +) -> PyResult> { let py = project.py(); let schemas = state_schemas(project, scope)?; let mut lowered_state: BTreeSet = BTreeSet::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 7bfd4ebf72..ee83b2488a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -688,7 +688,6 @@ native_ids! { ApplyMaximumStartPolicy => "apply_maximum_start_policy", ApplyTypedCursorReplayPolicy => "apply_typed_cursor_replay_policy", ApplyUnitTestSelectors => "apply_unit_test_selectors", - AssembleModelPlanEntry => "assemble_model_plan_entry", AssembleNativeProjectResources => "assemble_native_project_resources", AssembleNativeSqlTests => "assemble_native_sql_tests", AssembleProject => "assemble_project", @@ -739,8 +738,6 @@ native_ids! { BuildModelVersionIdentityMetadataJson => "build_model_version_identity_metadata_json", BuildModelWarnings => "build_model_warnings", BuildPhaseTimingTracker => "BuildPhaseTimingTracker", - BuildPlanEntries => "build_plan_entries", - BuildPlannerEntryResults => "build_planner_entry_results", BuildPlannerIdentityContext => "build_planner_identity_context", BuildPlannerRelationsContext => "build_planner_relations_context", BuildPlannerScope => "build_planner_scope", @@ -798,7 +795,6 @@ native_ids! { CompiledProjectRecord => "compiled_project_record", CompleteSemanticProject => "complete_semantic_project", ComputeCursorBounds => "compute_cursor_bounds", - ComputeModelCursorBounds => "compute_model_cursor_bounds", ComputeQueryHash => "compute_query_hash", ConnectionProgressReporter => "ConnectionProgressReporter", ContentDigest => "content_digest", @@ -1001,8 +997,6 @@ native_ids! { PlanDbtInteropFromProject => "plan_dbt_interop_from_project", PlanHasExecutableWork => "plan_has_executable_work", PlanModel => "plan_model", - PlanModelMigrations => "plan_model_migrations", - PlanOldNameViews => "plan_old_name_views", PlanReferenceRenames => "plan_reference_renames", PlanRetention => "plan_retention", PlanRuleFixes => "plan_rule_fixes", @@ -1099,7 +1093,6 @@ native_ids! { ResolveSchemaActions => "resolve_schema_actions", ResolveSkippedViewChain => "resolve_skipped_view_chain", ResolveSourceReferences => "resolve_source_references", - ResolveSqlTestModelChains => "resolve_sql_test_model_chains", ResolveSqlbuildExecutionPlanOutput => "resolve_sqlbuild_execution_plan_output", ResolveSqlbuildModelSelectorNames => "resolve_sqlbuild_model_selector_names", ResolveStartCursorConfig => "resolve_start_cursor_config", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index b067230958..f36ea6e975 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -4,6 +4,7 @@ mod compiled_tests; pub(crate) mod constants; mod cursor_window; mod fixture_validation; +pub(crate) mod models; pub(crate) mod plan_assembly; pub(crate) mod plan_driver; mod plan_entries; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs new file mode 100644 index 0000000000..75520b0694 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/models.rs @@ -0,0 +1,20 @@ +//! Typed requests of SQL-test planning. + +use pyo3::prelude::{Bound, PyAny}; + +/// How one batch of SQL tests is planned. +#[derive(Debug)] +pub(crate) struct SqlTestPlanOptions<'py> { + pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, + pub(crate) render_sql: bool, + pub(crate) include_plan: bool, + pub(crate) artifact: bool, +} + +/// How assembling SQL-test plan results validates fixtures. +#[derive(Debug)] +pub(crate) struct SqlTestAssembly<'py> { + pub(crate) sql_analysis_enabled: Bound<'py, PyAny>, + pub(crate) validate_fixtures: bool, + pub(crate) fixture_planning_context: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs index dca5cd13fe..25b43046fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_assembly.rs @@ -4,18 +4,17 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; -use crate::bindings::_helpers::sql_test_glue::plan_driver::plan_sql_tests_natively; +use crate::bindings::_helpers::sql_test_glue::models::{SqlTestAssembly, SqlTestPlanOptions}; +use crate::bindings::_helpers::sql_test_glue::plan_driver::planned as planned_tests; -#[pyfunction(name = "plan_sql_tests")] +#[pyfunction] #[pyo3(signature = (**kwargs))] -pub(crate) fn assemble_sql_test_plans<'py>( +pub(super) fn plan_sql_tests<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -33,47 +32,54 @@ pub(crate) fn assemble_sql_test_plans<'py>( ), )?; let tests: Vec> = args.value("tests")?.extract()?; - let project = args.value("project")?; - let adapter = args.value("adapter")?; + assemble_sql_test_plans( + &tests, + (&args.value("project")?, &args.value("adapter")?), + &SqlTestAssembly { + sql_analysis_enabled: args.value_or_false("sql_analysis_enabled")?, + validate_fixtures: args.flag("validate_fixtures")?, + fixture_planning_context: args.value("fixture_planning_context")?, + }, + ) +} + +/// Plan results of SQL tests, with fixture and expected-output diagnostics in place of failing plans. +pub(crate) fn assemble_sql_test_plans<'py>( + tests: &[Bound<'py, PyAny>], + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + assembly: &SqlTestAssembly<'py>, +) -> PyResult> { + let py = project.py(); let mut context: Option> = None; - let (planned, diagnostics) = if args.flag("sql_analysis_enabled")? - && args.flag("validate_fixtures")? + let (planned, diagnostics) = if assembly.sql_analysis_enabled.is_truthy()? + && assembly.validate_fixtures { - let given = args.value("fixture_planning_context")?; + let given = assembly.fixture_planning_context.clone(); let shared = if given.is_none() { - Bound::new(py, planning_context(&project, None)?)? + Bound::new(py, planning_context(project, None)?)? } else { given.cast_into::()? }; - let validated = super::plan_fixtures::validate(&tests, (&project, &adapter), shared.get())?; + let validated = super::plan_fixtures::validate(tests, (project, adapter), shared.get())?; context = Some(shared); validated } else { - (tests.clone(), Default::default()) + (tests.to_vec(), Default::default()) }; let plannable: Vec<_> = planned .into_iter() .enumerate() .filter_map(|(index, test)| (!diagnostics.contains_key(&index)).then_some(test)) .collect(); - let plans = plan_sql_tests_natively( - py, - Some(keywords( - py, - &[ - ("project", project.clone()), - ("tests", PyTuple::new(py, plannable)?.into_any()), - ("adapter", adapter), - ( - "sql_analysis_enabled", - args.value_or_false("sql_analysis_enabled")?, - ), - ( - "render_sql", - pyo3::types::PyBool::new(py, false).to_owned().into_any(), - ), - ], - )?), + let plans = planned_tests( + (project, adapter), + &PyTuple::new(py, plannable)?.into_any(), + &SqlTestPlanOptions { + sql_analysis_enabled: assembly.sql_analysis_enabled.clone(), + render_sql: false, + include_plan: true, + artifact: false, + }, )?; let plans: Vec> = plans.extract()?; if plans.len() != tests.len() - diagnostics.len() { @@ -129,8 +135,8 @@ pub(crate) fn assemble_sql_test_plans<'py>( super::plan_entries::entry( test, &plan, - (&project, &models), - &args.value_or_false("sql_analysis_enabled")?, + (project, &models), + &assembly.sql_analysis_enabled, )?, ), ("warnings", plan.getattr("warnings")?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs index 4e5d1a0190..d10b065dd6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_driver.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ use crate::bindings::_helpers::planner_strategy::fixture_context::count_chain_resolution; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::sql_test_glue::models::SqlTestPlanOptions; use crate::bindings::_helpers::sql_test_glue::plan_values::tuple; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyTuple}; @@ -47,36 +48,32 @@ pub(super) fn chains<'py>( Ok(PyTuple::new(py, result)?.into_any()) } -#[pyfunction] -#[pyo3(signature = (**kwargs))] +/// The model chain of each SQL test, resolved in one project batch and counted. pub(crate) fn resolve_sql_test_model_chains<'py>( - py: Python<'py>, - kwargs: Option>, + project: &Bound<'py, PyAny>, + tests: &Bound<'py, PyAny>, ) -> PyResult> { - let args = KeywordArguments::read( - py, - "resolve_sql_test_model_chains", - kwargs, - (&["project", "tests"], &[]), - )?; - count_chain_resolution(py)?; - chains(&args.value("project")?, &args.value("tests")?) + count_chain_resolution(project.py())?; + chains(project, tests) } -fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult> { - let project = args.value("project")?; +/// Plans of one SQL-test batch, or their compiled artifacts. +pub(crate) fn planned<'py>( + (project, adapter): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + tests: &Bound<'py, PyAny>, + options: &SqlTestPlanOptions<'py>, +) -> PyResult> { let py = project.py(); - let tests = args.value("tests")?; + let artifact = options.artifact; if !tests.is_truthy()? { return Ok(PyTuple::empty(py)); } let start: u64 = imported(py, "time", "perf_counter_ns")? .call0()? .extract()?; - let adapter = args.value("adapter")?; - let model_sql = super::plan_rendering::models(&project, &adapter)?; - let functions = super::plan_rendering::functions(&project, &adapter)?; - let overrides = super::plan_rendering::overrides(&project, &tests, &adapter)?; + let model_sql = super::plan_rendering::models(project, adapter)?; + let functions = super::plan_rendering::functions(project, adapter)?; + let overrides = super::plan_rendering::overrides(project, tests, adapter)?; let request = call_with( ®istry::class(py, RecordId::NativeSqlTestPlanningRequest)?, &[ @@ -85,7 +82,7 @@ fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult(args: &KeywordArguments<'py>, artifact: bool) -> PyResult(args: &KeywordArguments<'py>, artifact: bool) -> PyResult value, Err(error) if error.is_instance_of::(py) => { - return Err(super::plan_errors::planning_error(py, error, &tests)?); + return Err(super::plan_errors::planning_error(py, error, tests)?); } Err(error) => return Err(error), }; @@ -175,7 +169,7 @@ fn planned<'py>(args: &KeywordArguments<'py>, artifact: bool) -> PyResult( +pub(super) fn plan_sql_tests_natively<'py>( py: Python<'py>, kwargs: Option>, ) -> PyResult> { @@ -194,7 +188,16 @@ pub(crate) fn plan_sql_tests_natively<'py>( &["include_plan"], ), )?; - planned(&args, false) + planned( + (&args.value("project")?, &args.value("adapter")?), + &args.value("tests")?, + &SqlTestPlanOptions { + sql_analysis_enabled: args.value("sql_analysis_enabled")?, + render_sql: args.flag("render_sql")?, + include_plan: !args.passed("include_plan")? || args.flag("include_plan")?, + artifact: false, + }, + ) } #[pyfunction] @@ -212,7 +215,16 @@ pub(crate) fn plan_compiled_sql_test_artifacts<'py>( &[], ), )?; - planned(&args, true) + planned( + (&args.value("project")?, &args.value("adapter")?), + &args.value("tests")?, + &SqlTestPlanOptions { + sql_analysis_enabled: args.value("sql_analysis_enabled")?, + render_sql: true, + include_plan: false, + artifact: true, + }, + ) } #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs index 4e8d3f69c5..4d68666e41 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_fixtures.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; +use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::planner_strategy::models::RelationFixturePlanningContext; use crate::bindings::_helpers::sql_test_glue::authored_ctes::prefixed_ctes; use crate::bindings::_helpers::sql_test_glue::fixture_validation::validated_fixtures; @@ -92,16 +92,8 @@ pub(super) fn validate<'py>( if indexes.is_empty() { return Ok((tests.to_vec(), Default::default())); } - let chains = resolve_sql_test_model_chains( - py, - Some(keywords( - py, - &[ - ("project", inputs.0.clone()), - ("tests", PyTuple::new(py, model_tests)?.into_any()), - ], - )?), - )?; + let chains = + resolve_sql_test_model_chains(inputs.0, &PyTuple::new(py, model_tests)?.into_any())?; let chains: Vec> = chains.extract()?; if chains.len() != indexes.len() { return Err(pyo3::exceptions::PyValueError::new_err( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index c6df114194..2e414fa4ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -7,17 +7,13 @@ use crate::bindings::_helpers::sql_test_glue::{authored_ctes, compiled_tests, pl pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!( - super::plan_assembly::assemble_sql_test_plans, + super::plan_assembly::plan_sql_tests, module )?)?; module.add_function(wrap_pyfunction!( super::plan_driver::plan_sql_tests_natively, module )?)?; - module.add_function(wrap_pyfunction!( - super::plan_driver::resolve_sql_test_model_chains, - module - )?)?; module.add_function(wrap_pyfunction!( super::plan_driver::plan_compiled_sql_test_artifacts, module diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0ed383f959..07bfe17ff3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -252,11 +252,6 @@ def plan_compiled_sql_test_artifacts( adapter: BaseAdapter, sql_analysis_enabled: bool, ) -> tuple[NativeSqlTestArtifact, ...]: ... -def resolve_sql_test_model_chains( - *, - project: CompiledProjectView, - tests: tuple[CompiledSqlTest, ...], -) -> tuple[tuple[str, ...], ...]: ... def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[str, ...]: ... STAGE_CAPTURE_OMITTED_ATTRIBUTES: dict[str, frozenset[str]] @@ -2127,18 +2122,6 @@ class NativeCompiledProject: def compiled_model_record(model: CompiledModelView) -> CompiledModel: ... def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: ... -def compute_model_cursor_bounds( - *, - model: CompiledModelView, - snapshot: WarehouseSnapshot, - backfill: BackfillResult, - full_refresh: bool, - start_cursor_override: str | None, - end_cursor_override: str | None, - runtime_cursor_producer_names: frozenset[str], - context: ModelPlanContext, - replaces_relation: bool = False, -) -> CursorBounds | None: ... def compute_cursor_bounds( *, cursor_snapshot: ModelCursorSnapshot, @@ -6852,18 +6835,6 @@ def build_planner_relations_context( known_source_columns: dict[str, tuple[ColumnInfo, ...]] | None = None, relation_keys: frozenset[CompiledObjectKey] | None = None, ) -> PlannerRelationsContext: ... -def build_plan_entries( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - resolved_actions: PlannerResolvedActions, - cursor_overrides: CursorOverrides | None, - full_refresh: bool, - build_inputs: PlanEntryBuildInputs | None = None, -) -> PlannerModelEntryResults: ... def build_model_materializations( model_entries: tuple[ModelPlanEntry, ...], / ) -> dict[str, str]: ... @@ -7199,14 +7170,6 @@ def migration_destination_fingerprints( ) -> dict[str, tuple[str, ...]]: ... def stored_migration_fingerprint(fingerprint: Fingerprint) -> str | None: ... def declared_column_aliases(model: CompiledModelView) -> tuple[tuple[str, str], ...]: ... -def plan_old_name_views( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - migration_entries: tuple[ModelMigrationPlanEntry, ...], - column_entries: tuple[ColumnMigrationPlanEntry, ...], -) -> OldNameViewPlanning: ... def manual_migration_requests(*, scope: PlannerScope) -> tuple[ModelMigrationRequest, ...]: ... def discover_model_migrations( *, @@ -7229,15 +7192,6 @@ def decide_model_migration( state: MigrationStateInspection, snapshots_config: SnapshotsConfig, ) -> tuple[ModelMigrationPlanEntry, Fingerprint | None, bool]: ... -def plan_model_migrations( - *, - runtime: PlannerRuntime, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - overrides: PlannerOverrides, - deferral: DeferralInputs, - fingerprints: MigrationFingerprintCache, -) -> ModelMigrationPlanning: ... def plan_column_migrations( *, runtime: PlannerRuntime, @@ -7361,33 +7315,6 @@ def resolve_microbatch_lookback( *, model: CompiledModelView, effective_grain: str | None = None ) -> str | None: ... def extract_seed_columns(seed: CompiledSeedView) -> tuple[ColumnInfo, ...]: ... -def assemble_model_plan_entry( - *, - model: CompiledModelView, - adapter: BaseAdapter, - snapshot: WarehouseSnapshot, - change_result: ChangeDetectionResult, - action: PlanAction, - reason: PlanReason, - materialization_type: MaterializationType, - resolved_sql: str, - sql_analysis_enabled: bool, - pre_hooks: object, - post_hooks: object, - cursor_column: str | None, - cursor_input_relations: tuple[CursorInputRelation, ...], - runtime_owned_cursor_bounds: bool, - cursor_bounds: CursorBounds | None, - microbatch_range: CursorBounds | None, - empty_cursor_inputs: tuple[str, ...], - waiting_on_empty_inputs: bool, - backfill: BackfillResult, - start_cursor_override: str | None, - end_cursor_override: str | None, - future_cursor_config: FutureCursorsConfig | None, - start_cursor_config: StartCursorsConfig | None, - invocation_time: datetime | None, -) -> tuple[ModelPlanEntry, tuple[PlanWarning, ...]]: ... def plan_audit( *, audit: CompiledAuditView, @@ -7769,18 +7696,6 @@ def reconcile_execution_changes( pruning: PlannerScopePruningResult, changes: PlannerChangeResults, ) -> PlannerChangeReconciliation: ... -def build_planner_entry_results( - *, - runtime: PlannerRuntime, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - overrides: PlannerOverrides, - policies: PlannerPolicies, - deferral: DeferralInputs, - pruning: PlannerScopePruningResult, - reconciliation: PlannerChangeReconciliation, - source_freshness: DirectSourceFreshnessPlanningResult, -) -> PlannerEntryResults: ... def resolve_model_full_refresh(*, model: CompiledModelView, cli_full_refresh: bool) -> bool: ... def effectively_full_refreshed_model_names( *, project: CompiledProjectView, cli_full_refresh: bool From 1cf2c04387ce61f32abb4921202b3807cdb62215 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:34:23 +0100 Subject: [PATCH 08/12] refactor(planner): keep plan phase results as Rust data Change reconciliation, model entry results, plan output inputs and model, column and old-name migration planning results are Rust structs passed between phases instead of native records read back by attribute name. The no-op execution-scope pruning step is gone; plan output metadata is unchanged. Removes the unused build_plan_output, prune and reconcile entry points and their records. --- .../planner_entries/execution_plan.rs | 24 ++-- .../_helpers/planner_entries/model_entries.rs | 16 +-- .../_helpers/planner_entries/models.rs | 15 +++ .../_helpers/planner_entries/plan_output.rs | 54 +++------ .../planner_migrations/column_planning.rs | 39 +++--- .../planner_migrations/model_planning.rs | 22 ++-- .../_helpers/planner_migrations/models.rs | 25 ++++ .../planner_migrations/old_name_planning.rs | 20 ++-- .../_helpers/planner_phases/constants.rs | 11 -- .../planner_phases/execution_changes.rs | 76 ++++-------- .../_helpers/planner_phases/models.rs | 14 +++ .../planner_phases/output_assembly.rs | 84 ++++--------- .../_helpers/planner_phases/registration.rs | 8 -- .../planner_phases/warehouse_state.rs | 36 +++--- .../_helpers/planner_records/constants.rs | 67 ----------- .../src/bindings/_helpers/records/ids.rs | 9 -- src/sqlbuild/_native.pyi | 113 +----------------- 17 files changed, 195 insertions(+), 438 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 945c6d688d..80d9e5bdf0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::planner_entries::test_planning::{ use crate::bindings::_helpers::planner_entries::values::{COMPILE_TYPES, native, replaced}; use crate::bindings::_helpers::planner_phases::buildability::check_selected_scope_buildability; use crate::bindings::_helpers::planner_phases::execution_changes::{ - build_planner_entry_results, prune_planner_execution_scope, reconcile_execution_changes, + build_planner_entry_results, reconcile_execution_changes, }; use crate::bindings::_helpers::planner_phases::identities::{ build_planner_identity_context, detect_stale_warning_changes, @@ -128,28 +128,36 @@ fn planned_output<'py>( &warehouse.getattr("inspection_relations")?, &snapshot.getattr("source_freshness_state_schemas")?, )?; - let pruning = prune_planner_execution_scope(scopes, &resolved_actions)?; - let reconciliation = reconcile_execution_changes(&warehouse, &identities, &pruning, &changes)?; + let reconciliation = reconcile_execution_changes( + (&inspection_scope, &snapshot), + &identities, + &resolved_actions, + &changes, + )?; let entries = build_planner_entry_results( (runtime, &warehouse), (overrides, &deferral), - (&pruning, &reconciliation), + (&inspection_scope, &reconciliation), &source_freshness, )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); let output = base_plan_output( (runtime, &warehouse, &identities, overrides), - (&pruning, &reconciliation, &entries, &source_freshness), + ( + &inspection_scope, + &reconciliation, + entries, + &source_freshness, + ), &planned_sql_tests, )?; let output = add_storage_policies(output, (runtime, &warehouse, scopes, &policies))?; let output = add_plan_warnings( output, (scopes, &warehouse, &identities, &stale_warning_changes), - (&pruning, &source_freshness, &policies), + (&source_freshness, &policies), )?; - let plan_output = - add_plan_metadata(output, &pruning, &source_freshness, &policies)?.build(py)?; + let plan_output = add_plan_metadata(output, &source_freshness, &policies)?.build(py)?; let on_progress = args.value("on_progress")?; if !on_progress.is_none() { on_progress.call1((format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 40bda792c4..5b9898a4b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::planner_entries::values::{ use crate::bindings::_helpers::planner_phases::full_refresh::{ check_full_rebuild_allowed, describe_full_rebuild_cause, resolve_model_full_refresh, }; +use crate::bindings::_helpers::planner_phases::models::ModelEntryResults; use crate::bindings::_helpers::planner_resolution::cursor_ranges::{ CursorWindowInputs, config_str, cursor_safety_configs, empty_cursor_inputs, microbatch_range, model_cursor_overrides, plan_cursor_bounds, @@ -36,7 +37,7 @@ use crate::bindings::_helpers::planner_strategy::values::text; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; const MICROBATCH_COMPLEXITY_CODE: &str = "MICROBATCH_CONCURRENCY_COMPLEXITY"; @@ -368,7 +369,9 @@ fn check_microbatch_concurrency<'py>( } /// Build model plan entries from snapshot and per-model resolved actions. -pub(crate) fn plan_entries<'py>(request: PlanEntriesRequest<'py>) -> PyResult> { +pub(crate) fn plan_entries<'py>( + request: PlanEntriesRequest<'py>, +) -> PyResult> { let PlanEntriesRequest { project, adapter, @@ -527,14 +530,7 @@ pub(crate) fn plan_entries<'py>(request: PlanEntriesRequest<'py>) -> PyResult { pub(crate) full_refresh: bool, pub(crate) build_inputs: Bound<'py, PyAny>, } + +/// What assembling the plan output's fields reads. +#[derive(Debug)] +pub(crate) struct PlanOutputInputs<'py> { + pub(crate) project: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) scope: Bound<'py, PyAny>, + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) relations: Bound<'py, PyAny>, + pub(crate) changes: Bound<'py, PyAny>, + pub(crate) model_entries: Vec>, + pub(crate) model_warnings: Vec>, + pub(crate) reload_sources: Bound<'py, PyAny>, + pub(crate) extras: Bound<'py, PyAny>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs index ffee05797b..d4f97a8f78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/plan_output.rs @@ -2,10 +2,10 @@ use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; use crate::bindings::_helpers::planner_entries::model_entries::scope_overlaps; +use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::values::{ error_of, native, none, planner_model, planner_type, string, }; -use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; use crate::bindings::_helpers::planner_strategy::fixture_context::planning_context; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -541,16 +541,21 @@ fn planned_sql_tests<'py>( /// The plan output's fields from planned entries and resolved relation inputs. pub(crate) fn plan_output_fields<'py>( - args: &KeywordArguments<'py>, + inputs: PlanOutputInputs<'py>, ) -> PyResult)>> { - let project = args.value("project")?; + let PlanOutputInputs { + project, + adapter, + scope, + snapshot, + relations, + changes, + model_entries, + model_warnings, + reload_sources, + extras, + } = inputs; let py = project.py(); - let adapter = args.value("adapter")?; - let scope = args.value("scope")?; - let snapshot = args.value("snapshot")?; - let relations = args.value("relations")?; - let model_entry_results = args.value("model_entry_results")?; - let extras = args.value("extras")?; let extras = if extras.is_none() { native(py, RecordId::PlanOutputExtras, &[])? } else { @@ -567,20 +572,18 @@ pub(crate) fn plan_output_fields<'py>( ("execution_order", execution_order.clone()), ("selected_keys", selected_keys.clone()), ("source_map", source_map.clone()), - ("is_reload", args.value("reload_sources")?), + ("is_reload", reload_sources), ], )?; require_intermediate_loader_targets(&project, &snapshot, &source_map, &source_load_entries)?; let functions = function_entries( (&project, &adapter, &snapshot), &relations, - &args.value("changes")?, + &changes, &selected_keys, )?; - let model_entries = model_entry_results.getattr("entries")?; let materializations = PyDict::new(py); - for entry in model_entries.try_iter()? { - let entry = entry?; + for entry in &model_entries { materializations.set_item( entry.getattr("name")?, entry.getattr("materialization_type")?, @@ -618,13 +621,11 @@ pub(crate) fn plan_output_fields<'py>( scoped_order.push(key); } } - let mut warnings: Vec> = Vec::new(); - for warning in model_entry_results.getattr("warnings")?.try_iter()? { - warnings.push(warning?); - } + let mut warnings: Vec> = model_warnings; for warning in planned.getattr("warnings")?.try_iter()? { warnings.push(warning?); } + let model_entries = tuple_of(py, model_entries)?; Ok(vec![ ("execution_order", tuple_of(py, scoped_order)?), ("model_entries", model_entries.clone()), @@ -680,22 +681,6 @@ pub(crate) fn plan_output_fields<'py>( ]) } -/// Assemble the plan output from planned entries and resolved relation inputs. -#[pyfunction] -#[pyo3(signature = (**kwargs))] -fn build_plan_output<'py>( - py: Python<'py>, - kwargs: Option>, -) -> PyResult> { - let args = KeywordArguments::read( - py, - "build_plan_output", - kwargs, - (PLAN_OUTPUT_INPUTS, &["extras"]), - )?; - planner_model(py, RecordId::PlanOutput, &plan_output_fields(&args)?) -} - /// Build one canonical function entry from resolved relation inputs. #[pyfunction] #[pyo3(signature = (**kwargs))] @@ -797,7 +782,6 @@ fn build_selected_test_entries<'py>( } pub(super) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(build_plan_output, module)?)?; module.add_function(wrap_pyfunction!(plan_function, module)?)?; module.add_function(wrap_pyfunction!(plan_selected_sql_tests, module)?)?; module.add_function(wrap_pyfunction!(sql_test_is_selected, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index d9c61e3e77..2f143e1f1e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -2,16 +2,15 @@ use crate::bindings::_helpers::analysis::model_reads; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; -use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::replace; use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInputColumns; use crate::bindings::_helpers::planner_migrations::models::{ - ColumnMigrationInputs, MigrationPhase, + ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PySet, PySetMethods, PyTuple}; +use pyo3::types::{PyDict, PySet, PySetMethods}; use pyo3::{pyfunction, wrap_pyfunction}; /// Decide column renames and return the existing planning result model. @@ -20,7 +19,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; fn plan_column_migrations_of<'py>( py: Python<'py>, options: Option>, -) -> PyResult> { +) -> PyResult<()> { let args = KeywordArguments::read( py, "plan_column_migrations", @@ -53,13 +52,14 @@ fn plan_column_migrations_of<'py>( source_columns: args.value("source_columns")?, }, ) + .map(|_| ()) } /// Decide column renames for the selected incremental and snapshot models. pub(crate) fn plan_column_migrations<'py>( phase: &MigrationPhase<'py>, columns: &ColumnMigrationInputs<'py>, -) -> PyResult> { +) -> PyResult> { let py = phase.runtime.py(); let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); @@ -69,11 +69,12 @@ pub(crate) fn plan_column_migrations<'py>( &adapter.call_method0("sql_analysis_dialect")?, )?; if candidates.is_empty() { - return native( - py, - RecordId::ColumnMigrationPlanning, - &[("snapshot", snapshot)], - ); + return Ok(ColumnMigrationPlanning { + snapshot, + entries: Vec::new(), + hints: Vec::new(), + warnings: Vec::new(), + }); } let database = call_with( ®istry::native(py, NativeAttr::MigrationPlanningDatabase)?, @@ -138,16 +139,12 @@ pub(crate) fn plan_column_migrations<'py>( } let overlaid = super::column_overlay::overlay(&snapshot, &candidates, &entries, &inputs)?; let snapshot = super::column_overlay::cursors((phase, columns), overlaid, &entries)?; - native( - py, - RecordId::ColumnMigrationPlanning, - &[ - ("snapshot", snapshot), - ("entries", PyTuple::new(py, entries)?.into_any()), - ("hints", PyTuple::new(py, hints)?.into_any()), - ("warnings", PyTuple::new(py, warnings)?.into_any()), - ], - ) + Ok(ColumnMigrationPlanning { + snapshot, + entries, + hints, + warnings, + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index f1586bd36c..420311124c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -2,7 +2,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_migrations::column_objects::{config, native, replace}; -use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; +use crate::bindings::_helpers::planner_migrations::models::{ + MigrationPhase, ModelMigrationPlanning, +}; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -105,7 +107,7 @@ fn decide_model_migration<'py>( pub(crate) fn plan_model_migrations<'py>( phase: &MigrationPhase<'py>, fingerprints: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let py = phase.runtime.py(); let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); @@ -193,7 +195,7 @@ fn finish<'py>( state: &Bound<'py, PyAny>, schemas: &Bound<'py, PyAny>, discovery: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let runtime = phase.runtime.clone(); let py = runtime.py(); let scope = phase.scope.clone(); @@ -323,15 +325,11 @@ fn finish<'py>( if !entries.is_empty() { fields.push(("renamed_models", PyFrozenSet::new(py, renamed)?.into_any())); } - native( - py, - RecordId::ModelMigrationPlanning, - &[ - ("snapshot", replace(&overlaid, &fields)?), - ("entries", PyTuple::new(py, entries)?.into_any()), - ("warnings", PyTuple::new(py, warnings)?.into_any()), - ], - ) + Ok(ModelMigrationPlanning { + snapshot: replace(&overlaid, &fields)?, + entries, + warnings, + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs index 9536eb9803..abac12f8cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs @@ -44,3 +44,28 @@ pub(crate) struct ColumnMigrationInputs<'py> { pub(crate) physical_relations: Bound<'py, PyAny>, pub(crate) source_columns: Bound<'py, PyAny>, } + +/// The snapshot after model migrations overlay it, with the planned entries and their warnings. +#[derive(Debug)] +pub(crate) struct ModelMigrationPlanning<'py> { + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) entries: Vec>, + pub(crate) warnings: Vec>, +} + +/// The snapshot after column renames overlay it, with the planned entries, rename hints and warnings. +#[derive(Debug)] +pub(crate) struct ColumnMigrationPlanning<'py> { + pub(crate) snapshot: Bound<'py, PyAny>, + pub(crate) entries: Vec>, + pub(crate) hints: Vec>, + pub(crate) warnings: Vec>, +} + +/// The old-name view steps, the recorded live views and the warnings they raise. +#[derive(Debug)] +pub(crate) struct OldNameViewPlanning<'py> { + pub(crate) entries: Bound<'py, PyAny>, + pub(crate) views: Bound<'py, PyAny>, + pub(crate) warnings: Vec>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index f771290e8b..901ea28453 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -4,9 +4,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::native; -use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; +use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PySet, PySetMethods, PyTuple}; @@ -71,7 +71,7 @@ fn schemas<'py>( pub(crate) fn plan_old_name_views<'py>( phase: &MigrationPhase<'py>, moves: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), -) -> PyResult> { +) -> PyResult> { let py = phase.runtime.py(); let runtime = phase.runtime.clone(); let scope = phase.scope.clone(); @@ -171,13 +171,9 @@ pub(crate) fn plan_old_name_views<'py>( warnings.extend(super::old_name_references::errors( &runtime, &views, &entries, )?); - native( - py, - RecordId::OldNameViewPlanning, - &[ - ("entries", entries), - ("views", views), - ("warnings", PyTuple::new(py, warnings)?.into_any()), - ], - ) + Ok(OldNameViewPlanning { + entries, + views, + warnings, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs index ac9c86d04a..5c45d15791 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/constants.rs @@ -12,14 +12,3 @@ pub(crate) const MISSING_UPSTREAM_CODE: &str = "S301"; pub(crate) const RETENTION_DECREASE: &str = "decrease"; pub(crate) const RETENTION_INCREASE: &str = "increase"; pub(crate) const RETENTION_MATCH: &str = "match"; -/// The required inputs of plan output assembly. -pub(crate) const PLAN_OUTPUT_INPUTS: &[&str] = &[ - "project", - "adapter", - "scope", - "snapshot", - "relations", - "changes", - "model_entry_results", - "reload_sources", -]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 762a1811b4..267a5bfdd7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,54 +1,29 @@ -//! Execution-scope pruning, write-hash reconciliation and plan entry construction. +//! Write-hash reconciliation and plan entry construction. use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; use crate::bindings::_helpers::planner_entries::models::{PlanEntriesRequest, RelationsRequest}; use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; +use crate::bindings::_helpers::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; use crate::bindings::_helpers::planner_phases::objects::native; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; -use pyo3::pyfunction; -use pyo3::types::{PyFrozenSet, PyTuple}; +use pyo3::types::PyFrozenSet; -#[pyfunction] -#[pyo3(signature = (*, scopes, resolved_actions))] -pub(crate) fn prune_planner_execution_scope<'py>( - scopes: &Bound<'py, PyAny>, - resolved_actions: &Bound<'py, PyAny>, -) -> PyResult> { - let py = scopes.py(); - let inspection = scopes.getattr("inspection_scope")?; - native( - py, - RecordId::PlannerScopePruningResult, - &[ - ("inspection_scope", inspection.clone()), - ("execution_scope", inspection), - ("resolved_actions", resolved_actions.clone()), - ("pruned_direct_model_names", PyTuple::empty(py).into_any()), - ( - "direct_identity_stale_model_names", - PyFrozenSet::empty(py)?.into_any(), - ), - ], - ) -} - -#[pyfunction] -#[pyo3(signature = (*, warehouse, identities, pruning, changes))] +/// The execution scope's changes with honest write hashes, and the actions merged from them. pub(crate) fn reconcile_execution_changes<'py>( - warehouse: &Bound<'py, PyAny>, + (execution_scope, snapshot): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), identities: &Bound<'py, PyAny>, - pruning: &Bound<'py, PyAny>, + resolved_actions: &Bound<'py, PyAny>, changes: &Bound<'py, PyAny>, -) -> PyResult> { - let py = warehouse.py(); +) -> PyResult> { + let py = execution_scope.py(); let honest = native( py, NativeAttr::WithHonestModelWriteHashes, &[ - ("scope", pruning.getattr("execution_scope")?), - ("snapshot", warehouse.getattr("snapshot")?), + ("scope", execution_scope.clone()), + ("snapshot", snapshot.clone()), ("changes", changes.clone()), ( "version_identities", @@ -60,15 +35,14 @@ pub(crate) fn reconcile_execution_changes<'py>( py, NativeAttr::MergeRecomputedModelChanges, &[ - ("resolved_actions", pruning.getattr("resolved_actions")?), + ("resolved_actions", resolved_actions.clone()), ("changes", honest.clone()), ], )?; - native( - py, - RecordId::PlannerChangeReconciliation, - &[("changes", honest), ("resolved_actions", resolved)], - ) + Ok(ChangeReconciliation { + changes: honest, + resolved_actions: resolved, + }) } fn config_or<'py>( @@ -90,13 +64,12 @@ fn config_or<'py>( pub(crate) fn build_planner_entry_results<'py>( (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), (overrides, deferral): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - (pruning, reconciliation): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (execution_scope, reconciliation): (&Bound<'py, PyAny>, &ChangeReconciliation<'py>), source_freshness: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let py = runtime.py(); let project = runtime.getattr("project")?; let config = runtime.getattr("project_config")?; - let execution_scope = pruning.getattr("execution_scope")?; let relations = planner_relations_context(RelationsRequest { project: project.clone(), adapter: runtime.getattr("adapter")?, @@ -167,22 +140,15 @@ pub(crate) fn build_planner_entry_results<'py>( ("microbatch_limit_action", limit_action), ], )?; - let entries = plan_entries(PlanEntriesRequest { + plan_entries(PlanEntriesRequest { project, adapter: runtime.getattr("adapter")?, - scope: execution_scope, + scope: execution_scope.clone(), snapshot: warehouse.getattr("snapshot")?, relations, - resolved_models: reconciliation - .getattr("resolved_actions")? - .getattr("models")?, + resolved_models: reconciliation.resolved_actions.getattr("models")?, cursor_overrides: overrides.getattr("cursor_overrides")?, full_refresh: overrides.getattr("full_refresh")?.is_truthy()?, build_inputs, - })?; - native( - py, - RecordId::PlannerEntryResults, - &[("model_entry_results", entries)], - ) + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index 0fb08904de..d5ab8b6e63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -8,3 +8,17 @@ pub(crate) struct PlanOutputFields<'py> { pub(crate) fields: Vec<(&'static str, Bound<'py, PyAny>)>, pub(crate) metadata: Vec<(&'static str, Bound<'py, PyAny>)>, } + +/// The model changes and resolved actions after write-hash reconciliation. +#[derive(Debug)] +pub(crate) struct ChangeReconciliation<'py> { + pub(crate) changes: Bound<'py, PyAny>, + pub(crate) resolved_actions: Bound<'py, PyAny>, +} + +/// Planned model entries and the warnings planning them raised. +#[derive(Debug)] +pub(crate) struct ModelEntryResults<'py> { + pub(crate) entries: Vec>, + pub(crate) warnings: Vec>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 3bf915e1a4..eb78f38e42 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -1,14 +1,14 @@ //! Plan output assembly: storage policies, warnings and direct-mode metadata. use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::compile_driver::python_calls::{ - KeywordArguments, call_with, keywords, -}; +use crate::bindings::_helpers::compile_driver::python_calls::call_with; +use crate::bindings::_helpers::planner_entries::models::PlanOutputInputs; use crate::bindings::_helpers::planner_entries::plan_output::plan_output_fields; -use crate::bindings::_helpers::planner_phases::constants::PLAN_OUTPUT_INPUTS; -use crate::bindings::_helpers::planner_phases::models::PlanOutputFields; +use crate::bindings::_helpers::planner_phases::models::{ + ChangeReconciliation, ModelEntryResults, PlanOutputFields, +}; use crate::bindings::_helpers::planner_phases::objects::{ - model_key_names, native, replace, sorted_names, strings, text, tuple, + native, replace, sorted_names, strings, text, tuple, }; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; @@ -16,7 +16,6 @@ use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use std::collections::BTreeSet; fn concat<'py>(py: Python<'py>, parts: &[Bound<'py, PyAny>]) -> PyResult> { let mut items: Vec<_> = Vec::new(); @@ -65,10 +64,10 @@ pub(crate) fn base_plan_output<'py>( &Bound<'py, PyAny>, &Bound<'py, PyAny>, ), - (pruning, reconciliation, entries, source_freshness): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, + (execution_scope, reconciliation, entries, source_freshness): ( &Bound<'py, PyAny>, + &ChangeReconciliation<'py>, + ModelEntryResults<'py>, &Bound<'py, PyAny>, ), planned_sql_tests: &Bound<'py, PyAny>, @@ -90,38 +89,23 @@ pub(crate) fn base_plan_output<'py>( ("planned_sql_tests", planned_sql_tests.clone()), ], )?; - let inputs = keywords( - py, - &[ - ("project", runtime.getattr("project")?), - ("adapter", runtime.getattr("adapter")?), - ("scope", pruning.getattr("execution_scope")?), - ("snapshot", warehouse.getattr("snapshot")?), - ("relations", warehouse.getattr("inspection_relations")?), - ("changes", reconciliation.getattr("changes")?), - ( - "model_entry_results", - entries.getattr("model_entry_results")?, - ), - ("reload_sources", overrides.getattr("reload_sources")?), - ("extras", extras), - ], - )?; let mut output = PlanOutputFields { - fields: plan_output_fields(&KeywordArguments::read( - py, - "build_plan_output", - Some(inputs), - (PLAN_OUTPUT_INPUTS, &["extras"]), - )?)?, + fields: plan_output_fields(PlanOutputInputs { + project: runtime.getattr("project")?, + adapter: runtime.getattr("adapter")?, + scope: execution_scope.clone(), + snapshot: warehouse.getattr("snapshot")?, + relations: warehouse.getattr("inspection_relations")?, + changes: reconciliation.changes.clone(), + model_entries: entries.entries, + model_warnings: entries.warnings, + reload_sources: overrides.getattr("reload_sources")?, + extras, + })?, metadata: Vec::new(), }; output.set("source_freshness", source_freshness.clone()); output.set("cursor_overrides", overrides.getattr("cursor_overrides")?); - let pruned = pruning.getattr("pruned_direct_model_names")?; - if pruned.is_truthy()? { - output.metadata.push(("direct_pruned_model_names", pruned)); - } Ok(output) } @@ -237,11 +221,7 @@ pub(crate) fn add_plan_warnings<'py>( &Bound<'py, PyAny>, &Bound<'py, PyAny>, ), - (pruning, source_freshness, policies): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + (source_freshness, policies): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = scopes.py(); let freshness = source_freshness_unknown_warnings(source_freshness)?; @@ -257,7 +237,7 @@ pub(crate) fn add_plan_warnings<'py>( NativeAttr::BuildStaleOutOfSelectionWarnings, &[ ("original_scope", scopes.getattr("stale_warning_scope")?), - ("execution_scope", pruning.getattr("execution_scope")?), + ("execution_scope", scopes.getattr("inspection_scope")?), ("changes", stale_warning_changes.clone()), ("snapshot", warehouse.getattr("snapshot")?), ( @@ -355,24 +335,10 @@ fn freshness_metadata<'py>(source_freshness: &Bound<'py, PyAny>) -> PyResult( mut output: PlanOutputFields<'py>, - pruning: &Bound<'py, PyAny>, source_freshness: &Bound<'py, PyAny>, policies: &Bound<'py, PyAny>, ) -> PyResult> { - let py = pruning.py(); - let selected: BTreeSet = model_key_names( - &pruning - .getattr("inspection_scope")? - .getattr("selected_keys")?, - )? - .into_iter() - .collect(); - let remaining: Vec = strings(&pruning.getattr("direct_identity_stale_model_names")?)? - .into_iter() - .filter(|name| !selected.contains(name)) - .collect::>() - .into_iter() - .collect(); + let py = source_freshness.py(); let diagnostics = PyDict::new(py); diagnostics.set_item("mode", text(py, "direct"))?; diagnostics.set_item("enabled", policies.getattr("selection_diagnostics")?)?; @@ -382,7 +348,7 @@ pub(crate) fn add_plan_metadata<'py>( )); output.metadata.push(( "direct_remaining_stale_model_names", - sorted_names(py, remaining)?, + sorted_names(py, Vec::new())?, )); output .metadata diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index 85b98dfdee..dae4b5c3df 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -20,14 +20,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { super::buildability::check_selected_scope_buildability, module )?)?; - module.add_function(wrap_pyfunction!( - super::execution_changes::prune_planner_execution_scope, - module - )?)?; - module.add_function(wrap_pyfunction!( - super::execution_changes::reconcile_execution_changes, - module - )?)?; module.add_function(wrap_pyfunction!( super::full_refresh::resolve_model_full_refresh, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index 62ecf3ee5e..a39957f827 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -112,8 +112,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( relation_keys: py.None().into_bound(py), })?; let physical = PyDict::new(py); - for entry in migrations.getattr("entries")?.try_iter()? { - let entry = entry?; + for entry in &migrations.entries { if entry .getattr("decision")? .getattr("moves_data")? @@ -133,7 +132,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( } let columns = plan_column_migrations( &MigrationPhase { - snapshot: migrations.getattr("snapshot")?, + snapshot: migrations.snapshot.clone(), ..phase }, &ColumnMigrationInputs { @@ -145,6 +144,8 @@ pub(crate) fn gather_planner_warehouse_state<'py>( source_columns: inspection_relations.getattr("source_warehouse_columns")?, }, )?; + let migration_entries = PyTuple::new(py, &migrations.entries)?.into_any(); + let column_entries = PyTuple::new(py, &columns.entries)?.into_any(); let old_names = plan_old_name_migration_views( &MigrationPhase { runtime: runtime.clone(), @@ -153,10 +154,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( overrides: overrides.clone(), deferral: deferral.clone(), }, - ( - &migrations.getattr("entries")?, - &columns.getattr("entries")?, - ), + (&migration_entries, &column_entries), )?; if !runtime.getattr("on_progress")?.is_none() { progress( @@ -168,24 +166,24 @@ pub(crate) fn gather_planner_warehouse_state<'py>( )?; progress(runtime, "Generating plan...")?; } - let mut warnings: Vec<_> = Vec::new(); - for source in [&migrations, &columns, &old_names] { - for warning in source.getattr("warnings")?.try_iter()? { - warnings.push(warning?); - } - } + let mut warnings: Vec<_> = migrations.warnings; + warnings.extend(columns.warnings); + warnings.extend(old_names.warnings); native( py, RecordId::PlannerWarehouseState, &[ - ("snapshot", columns.getattr("snapshot")?), + ("snapshot", columns.snapshot), ("inspection_relations", inspection_relations), - ("migration_entries", migrations.getattr("entries")?), + ("migration_entries", migration_entries), ("migration_warnings", PyTuple::new(py, warnings)?.into_any()), - ("column_migration_entries", columns.getattr("entries")?), - ("column_rename_hints", columns.getattr("hints")?), - ("old_name_view_entries", old_names.getattr("entries")?), - ("old_name_views", old_names.getattr("views")?), + ("column_migration_entries", column_entries), + ( + "column_rename_hints", + PyTuple::new(py, columns.hints)?.into_any(), + ), + ("old_name_view_entries", old_names.entries), + ("old_name_views", old_names.views), ("migration_fingerprints", fingerprints), ], ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index a77e1e89d3..aeebfb176b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -1254,14 +1254,6 @@ const R_DIRECT_MODEL_VERSION_IDENTITIES: RecordSpec = RecordSpec { FieldSpec::required("model_version_hashes"), ], }; -const R_PLANNER_MODEL_ENTRY_RESULTS: RecordSpec = RecordSpec { - name: "PlannerModelEntryResults", - doc: "Model plan-entry phase output.", - fields: &[ - FieldSpec::required("entries"), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - ], -}; const R_SCHEMA_ACTION: RecordSpec = RecordSpec { name: "SchemaAction", doc: "One concrete schema change action to apply to a target table.", @@ -1482,34 +1474,6 @@ const R_MIGRATION_COMPATIBILITY_RESULT: RecordSpec = RecordSpec { FieldSpec::new("findings", FieldDefault::EmptyTuple), ], }; -const R_MODEL_MIGRATION_PLANNING: RecordSpec = RecordSpec { - name: "ModelMigrationPlanning", - doc: "Migration decisions plus the warehouse snapshot as it will look after moves.", - fields: &[ - FieldSpec::required("snapshot"), - FieldSpec::new("entries", FieldDefault::EmptyTuple), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - ], -}; -const R_OLD_NAME_VIEW_PLANNING: RecordSpec = RecordSpec { - name: "OldNameViewPlanning", - doc: "Old-name steps for this build plus every compatibility view still recorded.", - fields: &[ - FieldSpec::new("entries", FieldDefault::EmptyTuple), - FieldSpec::new("views", FieldDefault::EmptyTuple), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - ], -}; -const R_COLUMN_MIGRATION_PLANNING: RecordSpec = RecordSpec { - name: "ColumnMigrationPlanning", - doc: "Column rename decisions plus the warehouse snapshot as it will look after renames.", - fields: &[ - FieldSpec::required("snapshot"), - FieldSpec::new("entries", FieldDefault::EmptyTuple), - FieldSpec::new("hints", FieldDefault::EmptyTuple), - FieldSpec::new("warnings", FieldDefault::EmptyTuple), - ], -}; const R_PLANNER_IDENTITY_CONTEXT: RecordSpec = RecordSpec { name: "PlannerIdentityContext", doc: "Expected version identities for inspection and stale-warning scopes.", @@ -1518,30 +1482,6 @@ const R_PLANNER_IDENTITY_CONTEXT: RecordSpec = RecordSpec { FieldSpec::required("stale_warning_identities"), ], }; -const R_PLANNER_SCOPE_PRUNING_RESULT: RecordSpec = RecordSpec { - name: "PlannerScopePruningResult", - doc: "Scope pruning outcome with pruned scopes, actions, and staleness state.", - fields: &[ - FieldSpec::required("inspection_scope"), - FieldSpec::required("execution_scope"), - FieldSpec::required("resolved_actions"), - FieldSpec::required("pruned_direct_model_names"), - FieldSpec::required("direct_identity_stale_model_names"), - ], -}; -const R_PLANNER_CHANGE_RECONCILIATION: RecordSpec = RecordSpec { - name: "PlannerChangeReconciliation", - doc: "Write-hash-honest changes merged back into resolved actions.", - fields: &[ - FieldSpec::required("changes"), - FieldSpec::required("resolved_actions"), - ], -}; -const R_PLANNER_ENTRY_RESULTS: RecordSpec = RecordSpec { - name: "PlannerEntryResults", - doc: "Model plan entries for one plan build.", - fields: &[FieldSpec::required("model_entry_results")], -}; /// Planner value records, in registration order. pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ @@ -1569,7 +1509,6 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_RESOLVED_MODEL_ACTION, R_PLANNER_RESOLVED_ACTIONS, R_DIRECT_MODEL_VERSION_IDENTITIES, - R_PLANNER_MODEL_ENTRY_RESULTS, R_SCHEMA_ACTION, R_PLAN_WARNING, R_COLUMN_RENAME_HINT, @@ -1590,13 +1529,7 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_MODEL_MIGRATION_DECLARATION, R_MODEL_MIGRATION_DISCOVERY, R_MIGRATION_COMPATIBILITY_RESULT, - R_MODEL_MIGRATION_PLANNING, - R_OLD_NAME_VIEW_PLANNING, - R_COLUMN_MIGRATION_PLANNING, R_PLANNER_IDENTITY_CONTEXT, - R_PLANNER_SCOPE_PRUNING_RESULT, - R_PLANNER_CHANGE_RECONCILIATION, - R_PLANNER_ENTRY_RESULTS, ]; /// Planner string enums, in declaration order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index ee83b2488a..00b5795f29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -67,7 +67,6 @@ native_ids! { ColumnMigrationDecision => "ColumnMigrationDecision", ColumnMigrationEvent => "ColumnMigrationEvent", ColumnMigrationPlanEntry => "ColumnMigrationPlanEntry", - ColumnMigrationPlanning => "ColumnMigrationPlanning", ColumnRenameHint => "ColumnRenameHint", ColumnTransformKind => "ColumnTransformKind", CommandWarehouseScope => "CommandWarehouseScope", @@ -337,7 +336,6 @@ native_ids! { ModelMigrationDeclaration => "ModelMigrationDeclaration", ModelMigrationDiscovery => "ModelMigrationDiscovery", ModelMigrationPlanEntry => "ModelMigrationPlanEntry", - ModelMigrationPlanning => "ModelMigrationPlanning", ModelMigrationRequest => "ModelMigrationRequest", ModelPlanContext => "ModelPlanContext", ModelPlanEntry => "ModelPlanEntry", @@ -372,7 +370,6 @@ native_ids! { OldNameViewEventType => "OldNameViewEventType", OldNameViewHistory => "OldNameViewHistory", OldNameViewPlanEntry => "OldNameViewPlanEntry", - OldNameViewPlanning => "OldNameViewPlanning", OldNameViewStatus => "OldNameViewStatus", OnSchemaChange => "OnSchemaChange", OwnershipRoot => "OwnershipRoot", @@ -392,18 +389,14 @@ native_ids! { PlanSafetyGates => "PlanSafetyGates", PlanWarning => "PlanWarning", PlannedSqlTests => "PlannedSqlTests", - PlannerChangeReconciliation => "PlannerChangeReconciliation", PlannerChangeResults => "PlannerChangeResults", - PlannerEntryResults => "PlannerEntryResults", PlannerIdentityContext => "PlannerIdentityContext", - PlannerModelEntryResults => "PlannerModelEntryResults", PlannerOverrides => "PlannerOverrides", PlannerPolicies => "PlannerPolicies", PlannerRelationsContext => "PlannerRelationsContext", PlannerResolvedActions => "PlannerResolvedActions", PlannerRuntime => "PlannerRuntime", PlannerScope => "PlannerScope", - PlannerScopePruningResult => "PlannerScopePruningResult", PlannerScopeResolution => "PlannerScopeResolution", PlannerSelection => "PlannerSelection", PlannerWarehouseState => "PlannerWarehouseState", @@ -1014,7 +1007,6 @@ native_ids! { ProjectWith => "project_with", PromotionConflictDiagnostics => "promotion_conflict_diagnostics", ProveNativeDynamicContracts => "prove_native_dynamic_contracts", - PrunePlannerExecutionScope => "prune_planner_execution_scope", PublicDeclarationIndexes => "public_declaration_indexes", PublicModelSchemaIndex => "public_model_schema_index", PythonReadSourceNamesFor => "python_read_source_names_for", @@ -1024,7 +1016,6 @@ native_ids! { ReadSelectedSeedFingerprints => "read_selected_seed_fingerprints", ReadsOutsideTargetLineage => "reads_outside_target_lineage", RebindDeclarationScope => "rebind_declaration_scope", - ReconcileExecutionChanges => "reconcile_execution_changes", RecordCompileInputReads => "record_compile_input_reads", RecordModelAnalyses => "record_model_analyses", RecordSettingsClassRead => "record_settings_class_read", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 07bfe17ff3..aed8be74f5 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1103,16 +1103,6 @@ class DirectModelVersionIdentities(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> DirectModelVersionIdentities: ... -class PlannerModelEntryResults(NativeRecord): - """Model plan-entry phase output.""" - - entries: tuple[ModelPlanEntry, ...] - warnings: tuple[PlanWarning, ...] - def __init__( - self, entries: tuple[ModelPlanEntry, ...], warnings: tuple[PlanWarning, ...] = ... - ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerModelEntryResults: ... - class SchemaAction(NativeRecord): """One concrete schema change action to apply to a target table.""" @@ -1445,50 +1435,6 @@ class MigrationCompatibilityResult(NativeRecord): def __init__(self, status: MigrationCompatibility, findings: tuple[str, ...] = ...) -> None: ... def __replace__(self, **changes: Any) -> MigrationCompatibilityResult: ... -class ModelMigrationPlanning(NativeRecord): - """Migration decisions plus the warehouse snapshot as it will look after moves.""" - - snapshot: WarehouseSnapshot - entries: tuple[ModelMigrationPlanEntry, ...] - warnings: tuple[PlanWarning, ...] - def __init__( - self, - snapshot: WarehouseSnapshot, - entries: tuple[ModelMigrationPlanEntry, ...] = ..., - warnings: tuple[PlanWarning, ...] = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> ModelMigrationPlanning: ... - -class OldNameViewPlanning(NativeRecord): - """Old-name steps for this build plus every compatibility view still recorded.""" - - entries: tuple[OldNameViewPlanEntry, ...] - views: tuple[OldNameView, ...] - warnings: tuple[PlanWarning, ...] - def __init__( - self, - entries: tuple[OldNameViewPlanEntry, ...] = ..., - views: tuple[OldNameView, ...] = ..., - warnings: tuple[PlanWarning, ...] = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> OldNameViewPlanning: ... - -class ColumnMigrationPlanning(NativeRecord): - """Column rename decisions plus the warehouse snapshot as it will look after renames.""" - - snapshot: WarehouseSnapshot - entries: tuple[ColumnMigrationPlanEntry, ...] - hints: tuple[ColumnRenameHint, ...] - warnings: tuple[PlanWarning, ...] - def __init__( - self, - snapshot: WarehouseSnapshot, - entries: tuple[ColumnMigrationPlanEntry, ...] = ..., - hints: tuple[ColumnRenameHint, ...] = ..., - warnings: tuple[PlanWarning, ...] = ..., - ) -> None: ... - def __replace__(self, **changes: Any) -> ColumnMigrationPlanning: ... - class PlannerIdentityContext(NativeRecord): """Expected version identities for inspection and stale-warning scopes.""" @@ -1501,41 +1447,6 @@ class PlannerIdentityContext(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> PlannerIdentityContext: ... -class PlannerScopePruningResult(NativeRecord): - """Scope pruning outcome with pruned scopes, actions, and staleness state.""" - - inspection_scope: PlannerScope - execution_scope: PlannerScope - resolved_actions: PlannerResolvedActions - pruned_direct_model_names: tuple[str, ...] - direct_identity_stale_model_names: frozenset[str] - def __init__( - self, - inspection_scope: PlannerScope, - execution_scope: PlannerScope, - resolved_actions: PlannerResolvedActions, - pruned_direct_model_names: tuple[str, ...], - direct_identity_stale_model_names: frozenset[str], - ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerScopePruningResult: ... - -class PlannerChangeReconciliation(NativeRecord): - """Write-hash-honest changes merged back into resolved actions.""" - - changes: PlannerChangeResults - resolved_actions: PlannerResolvedActions - def __init__( - self, changes: PlannerChangeResults, resolved_actions: PlannerResolvedActions - ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerChangeReconciliation: ... - -class PlannerEntryResults(NativeRecord): - """Model plan entries for one plan build.""" - - model_entry_results: PlannerModelEntryResults - def __init__(self, model_entry_results: PlannerModelEntryResults) -> None: ... - def __replace__(self, **changes: Any) -> PlannerEntryResults: ... - class NativeModelInput: @property def reference_rows(self) -> list[tuple[str, str, str | None, int | None]]: ... @@ -6615,18 +6526,6 @@ def apply_maximum_start_policy( policy: MaximumStartPolicyInputs, has_start_override: bool, ) -> CursorBounds: ... -def build_plan_output( - *, - project: CompiledProjectView, - adapter: BaseAdapter, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - relations: PlannerRelationsContext, - changes: PlannerChangeResults, - model_entry_results: PlannerModelEntryResults, - reload_sources: bool, - extras: PlanOutputExtras | None = None, -) -> PlanOutput: ... def plan_function( *, function: CompiledFunctionView, @@ -7202,7 +7101,7 @@ def plan_column_migrations( overrides: PlannerOverrides, deferral: DeferralInputs, source_columns: Mapping[str, tuple[ColumnInfo, ...]], -) -> ColumnMigrationPlanning: ... +) -> None: ... def old_name_schemas_to_read(*, snapshot: WarehouseSnapshot, needed: set[str]) -> set[str]: ... def old_name_claim_message(*, model_name: str, view: OldNameView, now: datetime) -> str: ... def old_name_claim_errors( @@ -7686,16 +7585,6 @@ def check_selected_scope_buildability( warehouse: PlannerWarehouseState, deferral: DeferralInputs, ) -> None: ... -def prune_planner_execution_scope( - *, scopes: PlannerScopeResolution, resolved_actions: PlannerResolvedActions -) -> PlannerScopePruningResult: ... -def reconcile_execution_changes( - *, - warehouse: PlannerWarehouseState, - identities: PlannerIdentityContext, - pruning: PlannerScopePruningResult, - changes: PlannerChangeResults, -) -> PlannerChangeReconciliation: ... def resolve_model_full_refresh(*, model: CompiledModelView, cli_full_refresh: bool) -> bool: ... def effectively_full_refreshed_model_names( *, project: CompiledProjectView, cli_full_refresh: bool From 5abe26b41acab3e318aa00c33faf99663b651a29 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:54:13 +0100 Subject: [PATCH 09/12] refactor(planner): hold the planner runtime and warehouse state in Rust PlannerRuntime and PlannerWarehouseState are frozen native classes with typed fields instead of generic native records. Planner phases read their fields directly rather than by attribute name; Python still constructs and reads them with the same keywords and attributes. The unused class-call record default is removed. --- .../planner_entries/execution_plan.rs | 30 +++-- .../planner_migrations/column_entries.rs | 6 +- .../planner_migrations/column_planning.rs | 8 +- .../planner_migrations/discovery_models.rs | 8 +- .../planner_migrations/model_overlay.rs | 6 +- .../planner_migrations/model_planning.rs | 18 ++- .../planner_migrations/old_name_entries.rs | 5 +- .../planner_migrations/old_name_planning.rs | 18 ++- .../planner_migrations/planning_boundary.rs | 25 +++- .../planner_migrations/planning_execution.rs | 5 +- .../planner_migrations/planning_origin.rs | 13 +- .../_helpers/planner_phases/buildability.rs | 15 ++- .../planner_phases/execution_changes.rs | 21 ++-- .../bindings/_helpers/planner_phases/mod.rs | 1 + .../_helpers/planner_phases/models.rs | 31 ++++- .../planner_phases/output_assembly.rs | 42 +++++-- .../_helpers/planner_phases/registration.rs | 2 + .../_helpers/planner_phases/retention.rs | 24 ++-- .../_helpers/planner_phases/runtime.rs | 116 ++++++++++++++++++ .../planner_phases/warehouse_state.rs | 56 +++++---- .../_helpers/planner_records/constants.rs | 36 +----- .../src/bindings/_helpers/records/fields.rs | 1 - .../src/bindings/_helpers/records/ids.rs | 2 - .../src/bindings/_helpers/records/models.rs | 4 +- src/sqlbuild/_native.pyi | 8 +- .../_helpers/planning/test_retention.py | 11 +- 26 files changed, 362 insertions(+), 150 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 80d9e5bdf0..74cbae755f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -1,5 +1,6 @@ //! Top-level planner orchestration producing an execution plan. +use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, within, }; @@ -18,9 +19,11 @@ use crate::bindings::_helpers::planner_phases::execution_changes::{ use crate::bindings::_helpers::planner_phases::identities::{ build_planner_identity_context, detect_stale_warning_changes, }; +use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; use crate::bindings::_helpers::planner_phases::output_assembly::{ add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, }; +use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_phases::scopes::resolve_planner_scopes; use crate::bindings::_helpers::planner_phases::warehouse_state::gather_planner_warehouse_state; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; @@ -113,7 +116,7 @@ fn planned_output<'py>( policies.getattr("selection_diagnostics")?.extract()?, )?; check_selected_scope_buildability(runtime, scopes, &warehouse, &deferral)?; - let snapshot = warehouse.getattr("snapshot")?; + let snapshot = warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?; let (changes, stale_warning_changes) = change_results( &project, (scopes, &snapshot, &identities), @@ -125,7 +128,7 @@ fn planned_output<'py>( let source_freshness = build_planner_source_freshness_result( (&project, &inspection_scope), (&args.value("adapter")?, &args.value("connection")?), - &warehouse.getattr("inspection_relations")?, + &warehouse_field(&warehouse, |warehouse| &warehouse.inspection_relations)?, &snapshot.getattr("source_freshness_state_schemas")?, )?; let reconciliation = reconcile_execution_changes( @@ -195,18 +198,19 @@ fn build_execution_plan<'py>( let project = args.value("project")?; let adapter = args.value("adapter")?; let connection = args.value("connection")?; - let runtime = native( + let runtime = Bound::new( py, - RecordId::PlannerRuntime, - &[ - ("project", project.clone()), - ("adapter", adapter.clone()), - ("connection", connection.clone()), - ("project_config", args.value("project_config")?), - ("local_config", args.value("local_config")?), - ("on_progress", args.value("on_progress")?), - ], - )?; + PlannerRuntime { + project: project.clone().unbind(), + adapter: adapter.clone().unbind(), + connection: connection.clone().unbind(), + project_config: args.value("project_config")?.unbind(), + local_config: args.value("local_config")?.unbind(), + on_progress: args.value("on_progress")?.unbind(), + invocation_time: utc_now(py)?.unbind(), + }, + )? + .into_any(); let (scopes, overrides) = scopes_and_cursor_overrides(&args)?; let inspection = scopes.getattr("inspection_scope")?; let test_planning = start_planning( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 9a8539396a..13028274ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::planner_migrations::column_objects::{ }; use crate::bindings::_helpers::planner_migrations::constants::{AUTOMATIC, DONE}; use crate::bindings::_helpers::planner_migrations::models::{CheckedColumns, ColumnCandidate}; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::state_rows::migration_projections::newest_column; @@ -168,7 +169,10 @@ pub(super) fn checked<'py>( .getattr("adapter")? .getattr("column_rename_unavailable_reason")?, &[ - ("connection", runtime.getattr("connection")?), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ("destination", relation), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs index 2f143e1f1e..929aa2a268 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_planning.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::planner_migrations::known_input_columns::KnownInp use crate::bindings::_helpers::planner_migrations::models::{ ColumnMigrationInputs, ColumnMigrationPlanning, MigrationPhase, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -63,7 +64,7 @@ pub(crate) fn plan_column_migrations<'py>( let py = phase.runtime.py(); let runtime = phase.runtime.clone(); let snapshot = phase.snapshot.clone(); - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; let candidates = super::column_candidates::candidates( (phase, columns), &adapter.call_method0("sql_analysis_dialect")?, @@ -84,7 +85,10 @@ pub(crate) fn plan_column_migrations<'py>( ®istry::native(py, NativeAttr::MigrationStateInspection)?, &[ ("adapter", adapter), - ("connection", runtime.getattr("connection")?), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ("database", database), ], )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 6676aacc5f..4f55a6d0b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -5,6 +5,7 @@ use crate::bindings::_helpers::graph::node_set::PyNodeSet; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -16,7 +17,10 @@ use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let item = item?; let name: String = item.getattr("name")?.extract()?; let mut upstream: Vec = Vec::new(); @@ -73,7 +77,7 @@ pub(super) fn metadata<'py>( models: &[Bound<'py, PyAny>], ) -> PyResult> { let py = runtime.py(); - let project = runtime.getattr("project")?; + let project = runtime_field(&runtime, |runtime| &runtime.project)?; let functions = native( py, NativeAttr::BuildFunctionLocalHashes, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs index 51d7ca507d..1db0e0aacc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::column_objects::{config, replace}; use crate::bindings::_helpers::planner_migrations::models::MigrationPhase; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -124,7 +125,10 @@ pub(super) fn overlay<'py>( } fn cursor_enabled(runtime: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult { - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let item = item?; if item.getattr("name")?.eq(name)? { return Ok(config(&item, "materialized")?.eq("incremental")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs index 420311124c..515c7577c7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_planning.rs @@ -5,6 +5,7 @@ use crate::bindings::_helpers::planner_migrations::column_objects::{config, nati use crate::bindings::_helpers::planner_migrations::models::{ MigrationPhase, ModelMigrationPlanning, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -118,8 +119,14 @@ pub(crate) fn plan_model_migrations<'py>( let state = call_with( ®istry::native(py, NativeAttr::MigrationStateInspection)?, &[ - ("adapter", runtime.getattr("adapter")?), - ("connection", runtime.getattr("connection")?), + ( + "adapter", + runtime_field(&runtime, |runtime| &runtime.adapter)?, + ), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ( "database", call_with( @@ -144,7 +151,10 @@ fn discover<'py>( let scope = phase.scope.clone(); let snapshot = phase.snapshot.clone(); let mut declared_requests: Vec> = Vec::new(); - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { if let Some(request) = declared(&item?)? { declared_requests.push(request); } @@ -237,7 +247,7 @@ fn finish<'py>( &state.getattr("inspect_relations")?, &[("locations", PyTuple::new(py, origins)?.into_any())], )?; - let project_config = runtime.getattr("project_config")?; + let project_config = runtime_field(&runtime, |runtime| &runtime.project_config)?; let snapshots_config = if project_config.is_none() { registry::class(py, RecordId::SnapshotsConfig)?.call0()? } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs index a398c75a9e..0b3fb70282 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_entries.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::planner_migrations::models::OldNamePlanner; use crate::bindings::_helpers::planner_migrations::old_name_objects::{ expires, location, relation, retention, stores_history, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -21,7 +22,7 @@ impl<'py> OldNamePlanner<'py> { ) -> PyResult { let runtime = runtime.clone(); let py = runtime.py(); - let project_config = runtime.getattr("project_config")?; + let project_config = runtime_field(&runtime, |runtime| &runtime.project_config)?; let config = if project_config.is_none() { registry::class(py, RecordId::MigrationsConfig)?.call0()? } else { @@ -33,7 +34,7 @@ impl<'py> OldNamePlanner<'py> { views, columns: columns.clone(), retention: config.getattr("old_name_views")?, - now: runtime.getattr("invocation_time")?, + now: runtime_field(&runtime, |runtime| &runtime.invocation_time)?, runtime, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs index 901ea28453..ff0045eecc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/old_name_planning.rs @@ -6,6 +6,7 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::native; use crate::bindings::_helpers::planner_migrations::models::{MigrationPhase, OldNameViewPlanning}; use crate::bindings::_helpers::planner_migrations::old_name_objects::recorded_view; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::NativeAttr; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -75,7 +76,7 @@ pub(crate) fn plan_old_name_views<'py>( let py = phase.runtime.py(); let runtime = phase.runtime.clone(); let scope = phase.scope.clone(); - let project = runtime.getattr("project")?; + let project = runtime_field(&runtime, |runtime| &runtime.project)?; let models: Vec> = project .getattr("models")? .try_iter()? @@ -97,8 +98,14 @@ pub(crate) fn plan_old_name_views<'py>( let state = call_with( ®istry::native(py, NativeAttr::MigrationStateInspection)?, &[ - ("adapter", runtime.getattr("adapter")?), - ("connection", runtime.getattr("connection")?), + ( + "adapter", + runtime_field(&runtime, |runtime| &runtime.adapter)?, + ), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ( "database", native( @@ -164,7 +171,10 @@ pub(crate) fn plan_old_name_views<'py>( ("selected", PyTuple::new(py, selected)?.into_any()), ("snapshot", phase.snapshot.clone()), ("views", views.clone()), - ("now", runtime.getattr("invocation_time")?), + ( + "now", + runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + ), ], )?; let mut warnings: Vec> = claims.try_iter()?.collect::>()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs index 05ae35389a..7afa215fba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_boundary.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::planner_migrations::column_objects::native; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -45,8 +46,8 @@ fn migration_redirected_cursor_snapshots<'py>( let runtime = args.value("runtime")?; let scope = args.value("scope")?; let overrides = args.value("overrides")?; - let project = runtime.getattr("project")?; - let config = runtime.getattr("project_config")?; + let project = runtime_field(&runtime, |runtime| &runtime.project)?; + let config = runtime_field(&runtime, |runtime| &runtime.project_config)?; let start = if config.is_none() { py.None().into_bound(py) } else { @@ -58,7 +59,10 @@ fn migration_redirected_cursor_snapshots<'py>( &[ ("model_keys", scope.getattr("selected_keys")?), ("runtime_producer_keys", scope.getattr("selected_keys")?), - ("invocation_time", runtime.getattr("invocation_time")?), + ( + "invocation_time", + runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + ), ("start_cursor_config", start), ("cursor_overrides", overrides.getattr("cursor_overrides")?), ], @@ -74,8 +78,14 @@ fn migration_redirected_cursor_snapshots<'py>( ®istry::native(py, NativeAttr::GatherRedirectedCursorSnapshots)?, &[ ("project", project), - ("adapter", runtime.getattr("adapter")?), - ("connection", runtime.getattr("connection")?), + ( + "adapter", + runtime_field(&runtime, |runtime| &runtime.adapter)?, + ), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ("existing_relations", args.value("existing_relations")?), ("target_relations", args.value("target_relations")?), ("cursor_scope", cursor_scope), @@ -84,7 +94,10 @@ fn migration_redirected_cursor_snapshots<'py>( "deferred_locations", args.value("deferral")?.getattr("deferred_locations")?, ), - ("on_progress", runtime.getattr("on_progress")?), + ( + "on_progress", + runtime_field(&runtime, |runtime| &runtime.on_progress)?, + ), ( "origin_cursor_columns", args.value("origin_cursor_columns")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs index 07f52831e7..2a86cbfd6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs @@ -4,6 +4,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_migrations::column_objects::replace; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; @@ -25,7 +26,7 @@ fn migration_promotion<'py>( runtime: &Bound<'py, PyAny>, decision: &Bound<'py, PyAny>, ) -> PyResult> { - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; let promotion = registry::class(runtime.py(), RecordId::MigrationPromotion)?; if adapter .call_method0("supports_transactional_ddl")? @@ -47,7 +48,7 @@ fn migration_with_execution<'py>( entry: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; let transient = if adapter.getattr("adapter_name")?.eq("snowflake")? { PyBool::new( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 3a14bdae93..9fea051c79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -2,6 +2,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::planner_migrations::constants::QUALIFIED_RELATION_MAX_PARTS; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; @@ -32,7 +33,10 @@ fn named<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, ) -> PyResult> { - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let item = item?; if item.getattr("name")?.eq(name)? { return Ok( @@ -70,7 +74,7 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { #[pyfunction] #[pyo3(signature = (*, runtime))] fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { - let project = runtime.getattr("project")?; + let project = runtime_field(&runtime, |runtime| &runtime.project)?; let fallback = project.getattr("effective_target_database")?; for item in project.getattr("models")?.try_iter()? { let database = item?.getattr("destination")?.getattr("database")?; @@ -85,7 +89,10 @@ fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; - for item in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for item in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let schema = item?.getattr("destination")?.getattr("schema")?; if !schema.is_none() { schemas.add(schema)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 0f02cc0ce8..08e0504cc9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -2,6 +2,8 @@ use crate::bindings::_helpers::planner_phases::constants::MISSING_UPSTREAM_CODE; use crate::bindings::_helpers::planner_phases::objects::{frozenset, native, planner_error}; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_tables::check_selected_source_tables_exist; use crate::bindings::_helpers::records::ids::NativeAttr; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -16,7 +18,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( deferral: &Bound<'py, PyAny>, ) -> PyResult<()> { let py = runtime.py(); - let project = runtime.getattr("project")?; + let project = runtime_field(&runtime, |runtime| &runtime.project)?; let selected_scope = scopes.getattr("selected_scope")?; let mut external_seeds: Vec<_> = Vec::new(); for seed in project.getattr("seeds")?.try_iter()? { @@ -31,7 +33,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( &[ ("selected_keys", selected_scope.getattr("selected_keys")?), ("upstream_deps", selected_scope.getattr("upstream_deps")?), - ("snapshot", warehouse.getattr("snapshot")?), + ( + "snapshot", + warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?, + ), ( "deferred_relations", deferral.getattr("deferred_relations")?, @@ -65,10 +70,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( check_selected_source_tables_exist( &project, ( - &runtime.getattr("adapter")?, - &runtime.getattr("connection")?, + &runtime_field(&runtime, |runtime| &runtime.adapter)?, + &runtime_field(&runtime, |runtime| &runtime.connection)?, ), &selected_scope, - &warehouse.getattr("inspection_relations")?, + &warehouse_field(&warehouse, |warehouse| &warehouse.inspection_relations)?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 267a5bfdd7..0a26626150 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -5,6 +5,8 @@ use crate::bindings::_helpers::planner_entries::models::{PlanEntriesRequest, Rel use crate::bindings::_helpers::planner_entries::relations::planner_relations_context; use crate::bindings::_helpers::planner_phases::models::{ChangeReconciliation, ModelEntryResults}; use crate::bindings::_helpers::planner_phases::objects::native; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; @@ -68,16 +70,16 @@ pub(crate) fn build_planner_entry_results<'py>( source_freshness: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let project = runtime.getattr("project")?; - let config = runtime.getattr("project_config")?; + let project = runtime_field(&runtime, |runtime| &runtime.project)?; + let config = runtime_field(&runtime, |runtime| &runtime.project_config)?; let relations = planner_relations_context(RelationsRequest { project: project.clone(), - adapter: runtime.getattr("adapter")?, - connection: runtime.getattr("connection")?, + adapter: runtime_field(&runtime, |runtime| &runtime.adapter)?, + connection: runtime_field(&runtime, |runtime| &runtime.connection)?, scope: execution_scope.clone(), deferral: deferral.clone(), project_config: config.clone(), - local_config: runtime.getattr("local_config")?, + local_config: runtime_field(&runtime, |runtime| &runtime.local_config)?, known_source_columns: warehouse .getattr("inspection_relations")? .getattr("source_warehouse_columns")?, @@ -128,7 +130,10 @@ pub(crate) fn build_planner_entry_results<'py>( "start_cursor_config", config_or(&config, &["cursors", "start"], none())?, ), - ("invocation_time", runtime.getattr("invocation_time")?), + ( + "invocation_time", + runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + ), ("max_microbatches", max_microbatches), ( "max_microbatches_is_override", @@ -142,9 +147,9 @@ pub(crate) fn build_planner_entry_results<'py>( )?; plan_entries(PlanEntriesRequest { project, - adapter: runtime.getattr("adapter")?, + adapter: runtime_field(&runtime, |runtime| &runtime.adapter)?, scope: execution_scope.clone(), - snapshot: warehouse.getattr("snapshot")?, + snapshot: warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?, relations, resolved_models: reconciliation.resolved_actions.getattr("models")?, cursor_overrides: overrides.getattr("cursor_overrides")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs index e750afe4fb..b75082aa1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/mod.rs @@ -10,5 +10,6 @@ pub(crate) mod objects; pub(crate) mod output_assembly; pub(crate) mod registration; pub(crate) mod retention; +pub(crate) mod runtime; pub(crate) mod scopes; pub(crate) mod warehouse_state; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index d5ab8b6e63..3df8d67d83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -1,6 +1,7 @@ //! Plan output fields gathered across the plan phases before the output is built once. -use pyo3::prelude::{Bound, PyAny}; +use pyo3::prelude::{Bound, Py, PyAny}; +use pyo3::pyclass; /// The plan output's fields and metadata entries, each set before the output is built. #[derive(Debug)] @@ -22,3 +23,31 @@ pub(crate) struct ModelEntryResults<'py> { pub(crate) entries: Vec>, pub(crate) warnings: Vec>, } + +/// Resolved execution environment shared by planner phases. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerRuntime { + pub(crate) project: Py, + pub(crate) adapter: Py, + pub(crate) connection: Py, + pub(crate) project_config: Py, + pub(crate) local_config: Py, + pub(crate) on_progress: Py, + pub(crate) invocation_time: Py, +} + +/// Warehouse snapshot, inspection relations and migration plans gathered once per plan. +#[pyclass(module = "sqlbuild._native", frozen, get_all)] +#[derive(Debug)] +pub(crate) struct PlannerWarehouseState { + pub(crate) snapshot: Py, + pub(crate) inspection_relations: Py, + pub(crate) migration_entries: Py, + pub(crate) migration_warnings: Py, + pub(crate) column_migration_entries: Py, + pub(crate) column_rename_hints: Py, + pub(crate) old_name_view_entries: Py, + pub(crate) old_name_views: Py, + pub(crate) migration_fingerprints: Py, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index eb78f38e42..b83327bd44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -10,6 +10,8 @@ use crate::bindings::_helpers::planner_phases::models::{ use crate::bindings::_helpers::planner_phases::objects::{ native, replace, sorted_names, strings, text, tuple, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; +use crate::bindings::_helpers::planner_phases::runtime::warehouse_field; use crate::bindings::_helpers::planner_warehouse::source_freshness::source_freshness_unknown_warnings; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; @@ -91,11 +93,11 @@ pub(crate) fn base_plan_output<'py>( )?; let mut output = PlanOutputFields { fields: plan_output_fields(PlanOutputInputs { - project: runtime.getattr("project")?, - adapter: runtime.getattr("adapter")?, + project: runtime_field(&runtime, |runtime| &runtime.project)?, + adapter: runtime_field(&runtime, |runtime| &runtime.adapter)?, scope: execution_scope.clone(), - snapshot: warehouse.getattr("snapshot")?, - relations: warehouse.getattr("inspection_relations")?, + snapshot: warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?, + relations: warehouse_field(&warehouse, |warehouse| &warehouse.inspection_relations)?, changes: reconciliation.changes.clone(), model_entries: entries.entries, model_warnings: entries.warnings, @@ -161,7 +163,10 @@ pub(crate) fn add_storage_policies<'py>( .is_truthy()? { let models = PyDict::new(py); - for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for model in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let model = model?; models.set_item(model_reads::name_object(&model)?, model)?; } @@ -171,7 +176,10 @@ pub(crate) fn add_storage_policies<'py>( &[ ("entries", entries), ("models_by_name", models.into_any()), - ("fingerprints", warehouse.getattr("migration_fingerprints")?), + ( + "fingerprints", + warehouse_field(&warehouse, |warehouse| &warehouse.migration_fingerprints)?, + ), ( "dialect", runtime @@ -181,15 +189,20 @@ pub(crate) fn add_storage_policies<'py>( ], )?; } - let entries = with_column_rename_hints(&entries, &warehouse.getattr("column_rename_hints")?)?; - let migration_entries = warehouse.getattr("migration_entries")?; - let column_entries = warehouse.getattr("column_migration_entries")?; - let old_name_entries = warehouse.getattr("old_name_view_entries")?; - let old_name_views = warehouse.getattr("old_name_views")?; + let entries = with_column_rename_hints( + &entries, + &warehouse_field(&warehouse, |warehouse| &warehouse.column_rename_hints)?, + )?; + let migration_entries = warehouse_field(&warehouse, |warehouse| &warehouse.migration_entries)?; + let column_entries = + warehouse_field(&warehouse, |warehouse| &warehouse.column_migration_entries)?; + let old_name_entries = + warehouse_field(&warehouse, |warehouse| &warehouse.old_name_view_entries)?; + let old_name_views = warehouse_field(&warehouse, |warehouse| &warehouse.old_name_views)?; let warnings = concat( py, &[ - warehouse.getattr("migration_warnings")?, + warehouse_field(&warehouse, |warehouse| &warehouse.migration_warnings)?, output.field("warnings")?, ], )?; @@ -239,7 +252,10 @@ pub(crate) fn add_plan_warnings<'py>( ("original_scope", scopes.getattr("stale_warning_scope")?), ("execution_scope", scopes.getattr("inspection_scope")?), ("changes", stale_warning_changes.clone()), - ("snapshot", warehouse.getattr("snapshot")?), + ( + "snapshot", + warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?, + ), ( "version_identities", identities.getattr("stale_warning_identities")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs index dae4b5c3df..3ed0682940 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/registration.rs @@ -4,6 +4,8 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::wrap_pyfunction; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!( super::scopes::resolve_planner_scopes, module diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index 0559d5227e..2c8eec692f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -13,6 +13,7 @@ use crate::bindings::_helpers::planner_phases::objects::{ construct, member, model_key_names, native, native_member, optional_text, planner_error, planner_model, replace, sorted_names, text, tuple, }; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use pyo3::IntoPyObject; @@ -30,8 +31,8 @@ pub(crate) fn effective_target_config<'py>( runtime: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let project_config = runtime.getattr("project_config")?; - let local_config = runtime.getattr("local_config")?; + let project_config = runtime_field(&runtime, |runtime| &runtime.project_config)?; + let local_config = runtime_field(&runtime, |runtime| &runtime.local_config)?; let target_name = runtime .getattr("project")? .getattr("effective_target_name")?; @@ -64,7 +65,10 @@ fn selected_models<'py>( .into_iter() .collect(); let mut models: Vec<_> = Vec::new(); - for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for model in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let model = model?; if names.contains(&model_reads::model_name(&model)?) { models.push(model); @@ -119,7 +123,7 @@ pub(crate) fn plan_table_types<'py>( let relations = warehouse .getattr("snapshot")? .getattr("existing_relations")?; - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; let materialization = registry::class(py, RecordId::MaterializationType)?; let mut entries: Vec<_> = Vec::new(); for model in models { @@ -282,9 +286,12 @@ fn inspect<'py>( request: &Bound<'py, PyAny>, ) -> PyResult> { call_with( - &runtime.getattr("adapter")?.getattr("inspect_retention")?, + &runtime_field(&runtime, |runtime| &runtime.adapter)?.getattr("inspect_retention")?, &[ - ("connection", runtime.getattr("connection")?), + ( + "connection", + runtime_field(&runtime, |runtime| &runtime.connection)?, + ), ("request", request.clone()), ], ) @@ -463,7 +470,10 @@ fn bigquery_retention<'py>( } type Scope = (Option, String); let mut managed: BTreeMap>> = BTreeMap::new(); - for model in runtime.getattr("project")?.getattr("models")?.try_iter()? { + for model in runtime_field(&runtime, |runtime| &runtime.project)? + .getattr("models")? + .try_iter()? + { let model = model?; let retention = model.getattr("config")?.getattr("time_travel_retention")?; let destination = model_reads::destination(&model)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs new file mode 100644 index 0000000000..96c8604552 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/runtime.rs @@ -0,0 +1,116 @@ +//! The planner runtime and warehouse state: built from keyword arguments and read by planner phases. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pymethods; +use pyo3::types::{PyDict, PyTuple}; + +use crate::bindings::_helpers::build_command::support::utc_now; +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::planner_phases::models::{PlannerRuntime, PlannerWarehouseState}; +use crate::bindings::_helpers::records::ids::NativeAttr; +use crate::bindings::_helpers::records::registry; + +#[pymethods] +impl PlannerRuntime { + #[new] + #[pyo3(signature = (**kwargs))] + fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "PlannerRuntime", + kwargs, + ( + &["project", "adapter", "connection"], + &[ + "project_config", + "local_config", + "on_progress", + "invocation_time", + ], + ), + )?; + let invocation_time = if args.passed("invocation_time")? { + args.value("invocation_time")? + } else { + utc_now(py)? + }; + Ok(Self { + project: args.value("project")?.unbind(), + adapter: args.value("adapter")?.unbind(), + connection: args.value("connection")?.unbind(), + project_config: args.value("project_config")?.unbind(), + local_config: args.value("local_config")?.unbind(), + on_progress: args.value("on_progress")?.unbind(), + invocation_time: invocation_time.unbind(), + }) + } +} + +/// One field of a planner runtime passed as a Python object. +pub(crate) fn runtime_field<'py>( + runtime: &Bound<'py, PyAny>, + field: fn(&PlannerRuntime) -> &Py, +) -> PyResult> { + Ok(field(runtime.cast::()?.get()) + .bind(runtime.py()) + .clone()) +} + +#[pymethods] +impl PlannerWarehouseState { + #[new] + #[pyo3(signature = (**kwargs))] + fn new(py: Python<'_>, kwargs: Option>) -> PyResult { + let args = KeywordArguments::read( + py, + "PlannerWarehouseState", + kwargs, + ( + &["snapshot", "inspection_relations"], + &[ + "migration_entries", + "migration_warnings", + "column_migration_entries", + "column_rename_hints", + "old_name_view_entries", + "old_name_views", + "migration_fingerprints", + ], + ), + )?; + let entries = |name: &str| -> PyResult> { + Ok(if args.passed(name)? { + args.value(name)?.unbind() + } else { + PyTuple::empty(py).into_any().unbind() + }) + }; + Ok(Self { + snapshot: args.value("snapshot")?.unbind(), + inspection_relations: args.value("inspection_relations")?.unbind(), + migration_entries: entries("migration_entries")?, + migration_warnings: entries("migration_warnings")?, + column_migration_entries: entries("column_migration_entries")?, + column_rename_hints: entries("column_rename_hints")?, + old_name_view_entries: entries("old_name_view_entries")?, + old_name_views: entries("old_name_views")?, + migration_fingerprints: if args.passed("migration_fingerprints")? { + args.value("migration_fingerprints")?.unbind() + } else { + registry::native(py, NativeAttr::MigrationFingerprintCache)? + .call0()? + .unbind() + }, + }) + } +} + +/// One field of a planner warehouse state passed as a Python object. +pub(crate) fn warehouse_field<'py>( + warehouse: &Bound<'py, PyAny>, + field: fn(&PlannerWarehouseState) -> &Py, +) -> PyResult> { + Ok(field(warehouse.cast::()?.get()) + .bind(warehouse.py()) + .clone()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index a39957f827..ee8e78bf1f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -9,7 +9,9 @@ use crate::bindings::_helpers::planner_migrations::models::{ }; use crate::bindings::_helpers::planner_migrations::old_name_planning::plan_old_name_views as plan_old_name_migration_views; use crate::bindings::_helpers::planner_phases::full_refresh::effectively_full_refreshed_model_names; +use crate::bindings::_helpers::planner_phases::models::PlannerWarehouseState; use crate::bindings::_helpers::planner_phases::objects::native; +use crate::bindings::_helpers::planner_phases::runtime::runtime_field; use crate::bindings::_helpers::planner_warehouse::fingerprint_cache::MigrationFingerprintCache; use crate::bindings::_helpers::planner_warehouse::snapshot::{SnapshotRequest, warehouse_snapshot}; use crate::bindings::_helpers::records::ids::RecordId; @@ -19,7 +21,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use std::time::Instant; fn progress(runtime: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { - let on_progress = runtime.getattr("on_progress")?; + let on_progress = runtime_field(&runtime, |runtime| &runtime.on_progress)?; if !on_progress.is_none() { on_progress.call1((message,))?; } @@ -36,7 +38,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ) -> PyResult> { let py = runtime.py(); let started = Instant::now(); - let project = runtime.getattr("project")?; + let project = runtime_field(&runtime, |runtime| &runtime.project)?; let fingerprints = Py::new( py, MigrationFingerprintCache::open(py, Some(project.getattr("compile_cache_dir")?))?, @@ -46,7 +48,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( progress(runtime, "Inspecting warehouse state...")?; let selected_scope = scopes.getattr("selected_scope")?; let full_refresh: bool = overrides.getattr("full_refresh")?.extract()?; - let config = runtime.getattr("project_config")?; + let config = runtime_field(&runtime, |runtime| &runtime.project_config)?; let cursor_scope = native( py, RecordId::CursorSnapshotScope, @@ -56,7 +58,10 @@ pub(crate) fn gather_planner_warehouse_state<'py>( "runtime_producer_keys", selected_scope.getattr("selected_keys")?, ), - ("invocation_time", runtime.getattr("invocation_time")?), + ( + "invocation_time", + runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + ), ( "start_cursor_config", if config.is_none() { @@ -68,7 +73,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ("cursor_overrides", overrides.getattr("cursor_overrides")?), ], )?; - let adapter = runtime.getattr("adapter")?; + let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; let stale_keys = pyo3::types::PyFrozenSet::new( py, scopes @@ -82,12 +87,12 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let snapshot = warehouse_snapshot(&SnapshotRequest { project: project.clone(), adapter: adapter.clone(), - connection: runtime.getattr("connection")?, + connection: runtime_field(&runtime, |runtime| &runtime.connection)?, execute: adapter.getattr("execute")?, selected_keys: stale_keys, full_refresh: false, full_refresh_model_names: effectively_full_refreshed_model_names(&project, full_refresh)?, - on_progress: runtime.getattr("on_progress")?, + on_progress: runtime_field(&runtime, |runtime| &runtime.on_progress)?, deferred_locations: deferral.getattr("deferred_locations")?, cursor_scope, })?; @@ -103,11 +108,11 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let inspection_relations = planner_relations_context(RelationsRequest { project: project.clone(), adapter: adapter.clone(), - connection: runtime.getattr("connection")?, + connection: runtime_field(&runtime, |runtime| &runtime.connection)?, scope: scopes.getattr("inspection_scope")?, deferral: deferral.clone(), project_config: config, - local_config: runtime.getattr("local_config")?, + local_config: runtime_field(&runtime, |runtime| &runtime.local_config)?, known_source_columns: py.None().into_bound(py), relation_keys: py.None().into_bound(py), })?; @@ -156,7 +161,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( }, (&migration_entries, &column_entries), )?; - if !runtime.getattr("on_progress")?.is_none() { + if !runtime_field(&runtime, |runtime| &runtime.on_progress)?.is_none() { progress( runtime, &format!( @@ -169,22 +174,19 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let mut warnings: Vec<_> = migrations.warnings; warnings.extend(columns.warnings); warnings.extend(old_names.warnings); - native( + Ok(Bound::new( py, - RecordId::PlannerWarehouseState, - &[ - ("snapshot", columns.snapshot), - ("inspection_relations", inspection_relations), - ("migration_entries", migration_entries), - ("migration_warnings", PyTuple::new(py, warnings)?.into_any()), - ("column_migration_entries", column_entries), - ( - "column_rename_hints", - PyTuple::new(py, columns.hints)?.into_any(), - ), - ("old_name_view_entries", old_names.entries), - ("old_name_views", old_names.views), - ("migration_fingerprints", fingerprints), - ], - ) + PlannerWarehouseState { + snapshot: columns.snapshot.unbind(), + inspection_relations: inspection_relations.unbind(), + migration_entries: migration_entries.unbind(), + migration_warnings: PyTuple::new(py, warnings)?.into_any().unbind(), + column_migration_entries: column_entries.unbind(), + column_rename_hints: PyTuple::new(py, columns.hints)?.into_any().unbind(), + old_name_view_entries: old_names.entries.unbind(), + old_name_views: old_names.views.unbind(), + migration_fingerprints: fingerprints.unbind(), + }, + )? + .into_any()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index aeebfb176b..d419863cd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -1,9 +1,8 @@ //! Layouts of the planner, migration, cursor and fingerprint value records hosted natively. -use crate::bindings::_helpers::build_command::support::utc_now; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::record_behaviours::planner::forward_only_backfill; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::models::{EnumSpec, FieldDefault, FieldSpec, RecordSpec}; const R_PARSED_SELECTOR: RecordSpec = RecordSpec { @@ -1395,19 +1394,6 @@ const R_PLAN_PROVIDER_USAGE: RecordSpec = RecordSpec { FieldSpec::new("annotation_module", FieldDefault::None), ], }; -const R_PLANNER_RUNTIME: RecordSpec = RecordSpec { - name: "PlannerRuntime", - doc: "Resolved execution environment shared by planner phases.", - fields: &[ - FieldSpec::required("project"), - FieldSpec::required("adapter"), - FieldSpec::required("connection"), - FieldSpec::new("project_config", FieldDefault::None), - FieldSpec::new("local_config", FieldDefault::None), - FieldSpec::new("on_progress", FieldDefault::None), - FieldSpec::new("invocation_time", FieldDefault::Factory(utc_now)), - ], -}; const R_PLANNER_SCOPE_RESOLUTION: RecordSpec = RecordSpec { name: "PlannerScopeResolution", doc: "Resolved planner scopes for one plan build.", @@ -1417,24 +1403,6 @@ const R_PLANNER_SCOPE_RESOLUTION: RecordSpec = RecordSpec { FieldSpec::required("inspection_scope"), ], }; -const R_PLANNER_WAREHOUSE_STATE: RecordSpec = RecordSpec { - name: "PlannerWarehouseState", - doc: "Warehouse snapshot and inspection relations gathered once per plan.", - fields: &[ - FieldSpec::required("snapshot"), - FieldSpec::required("inspection_relations"), - FieldSpec::new("migration_entries", FieldDefault::EmptyTuple), - FieldSpec::new("migration_warnings", FieldDefault::EmptyTuple), - FieldSpec::new("column_migration_entries", FieldDefault::EmptyTuple), - FieldSpec::new("column_rename_hints", FieldDefault::EmptyTuple), - FieldSpec::new("old_name_view_entries", FieldDefault::EmptyTuple), - FieldSpec::new("old_name_views", FieldDefault::EmptyTuple), - FieldSpec::new( - "migration_fingerprints", - FieldDefault::NativeClassCall(NativeAttr::MigrationFingerprintCache), - ), - ], -}; const R_MODEL_MIGRATION_REQUEST: RecordSpec = RecordSpec { name: "ModelMigrationRequest", doc: "One destination model whose data should come from an earlier relation.", @@ -1522,9 +1490,7 @@ pub(crate) const PLANNER_VALUE_RECORDS: &[RecordSpec] = &[ R_SCENARIO_ARTIFACT_IDENTITY, R_SCENARIO_ARTIFACT_NAME, R_PLAN_PROVIDER_USAGE, - R_PLANNER_RUNTIME, R_PLANNER_SCOPE_RESOLUTION, - R_PLANNER_WAREHOUSE_STATE, R_MODEL_MIGRATION_REQUEST, R_MODEL_MIGRATION_DECLARATION, R_MODEL_MIGRATION_DISCOVERY, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index d37c1c4cda..54b76127e7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -118,7 +118,6 @@ fn default_value<'py>( FieldDefault::HostMember(class, member) => host::entry(py, class)?.getattr(member), FieldDefault::NativeMember(class, member) => registry::class(py, class)?.getattr(member), FieldDefault::NativeCall(class) => registry::class(py, class)?.call0(), - FieldDefault::NativeClassCall(class) => registry::native(py, class)?.call0(), FieldDefault::Factory(factory) => factory(py), FieldDefault::CallText(module, name, text) => imported(py, module, name)?.call1((text,)), } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index 00b5795f29..4926d5bdc0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -395,11 +395,9 @@ native_ids! { PlannerPolicies => "PlannerPolicies", PlannerRelationsContext => "PlannerRelationsContext", PlannerResolvedActions => "PlannerResolvedActions", - PlannerRuntime => "PlannerRuntime", PlannerScope => "PlannerScope", PlannerScopeResolution => "PlannerScopeResolution", PlannerSelection => "PlannerSelection", - PlannerWarehouseState => "PlannerWarehouseState", PlaygroundCommandRequest => "PlaygroundCommandRequest", PlaygroundTemplate => "PlaygroundTemplate", PolyglotAnalysisDialect => "PolyglotAnalysisDialect", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs index 327da1b23d..7430d50770 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/models.rs @@ -13,7 +13,7 @@ use crate::bindings::_helpers::records::fields::{ build_values, cached_value, compared_values, frozen_error, ordered_comparison, record_spec, repr_text, store_cached, }; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::types::{ClassMethod, ClassProperty, RecordFactory}; /// How a record field is filled when the caller does not pass it. @@ -40,8 +40,6 @@ pub(crate) enum FieldDefault { HostMember(HostId, &'static str), /// A native record class called with no arguments, evaluated for every instance. NativeCall(RecordId), - /// A native Rust class called with no arguments, evaluated for every instance. - NativeClassCall(NativeAttr), /// A native factory, evaluated for every instance. Factory(RecordFactory), /// `getattr(import(module), name)(text)`, evaluated for every instance. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index aed8be74f5..4504f51c5a 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1318,7 +1318,7 @@ class PlanProviderUsage(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> PlanProviderUsage: ... -class PlannerRuntime(NativeRecord): +class PlannerRuntime: """Resolved execution environment shared by planner phases.""" project: CompiledProjectView @@ -1330,6 +1330,7 @@ class PlannerRuntime(NativeRecord): invocation_time: datetime def __init__( self, + *, project: CompiledProjectView, adapter: BaseAdapter, connection: Any, @@ -1338,7 +1339,6 @@ class PlannerRuntime(NativeRecord): on_progress: Callable[[str], None] | None = ..., invocation_time: datetime = ..., ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerRuntime: ... class PlannerScopeResolution(NativeRecord): """Resolved planner scopes for one plan build.""" @@ -1354,7 +1354,7 @@ class PlannerScopeResolution(NativeRecord): ) -> None: ... def __replace__(self, **changes: Any) -> PlannerScopeResolution: ... -class PlannerWarehouseState(NativeRecord): +class PlannerWarehouseState: """Warehouse snapshot and inspection relations gathered once per plan.""" snapshot: WarehouseSnapshot @@ -1368,6 +1368,7 @@ class PlannerWarehouseState(NativeRecord): migration_fingerprints: MigrationFingerprintCache def __init__( self, + *, snapshot: WarehouseSnapshot, inspection_relations: PlannerRelationsContext, migration_entries: tuple[ModelMigrationPlanEntry, ...] = ..., @@ -1378,7 +1379,6 @@ class PlannerWarehouseState(NativeRecord): old_name_views: tuple[OldNameView, ...] = ..., migration_fingerprints: MigrationFingerprintCache = ..., ) -> None: ... - def __replace__(self, **changes: Any) -> PlannerWarehouseState: ... class ModelMigrationRequest(NativeRecord): """One destination model whose data should come from an earlier relation.""" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py index 02f7d206a5..69a15cd0b7 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/planning/test_retention.py @@ -424,8 +424,8 @@ def test_given_changes_only_current_model_when_planning_type_then_selection_stil declared=True, ), ) - warehouse = replace( - warehouse, + warehouse = PlannerWarehouseState( + inspection_relations=warehouse.inspection_relations, snapshot=replace( warehouse.snapshot, fingerprints=WarehouseFingerprints( @@ -753,8 +753,11 @@ def test_given_missing_bigquery_dataset_when_planning_then_defers_retention_unti existing_relations={}, config_values={}, ) - runtime: PlannerRuntime = replace( - runtime, + runtime: PlannerRuntime = PlannerRuntime( + project=runtime.project, + adapter=runtime.adapter, + connection=runtime.connection, + invocation_time=runtime.invocation_time, project_config=ProjectConfig( name="test", adapter=BuiltinAdapter.BIGQUERY.value, From 15dd042b828ff2879b46ca7e1b85a97e85ea8d64 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:42:36 +0100 Subject: [PATCH 10/12] fix: stop early SQL lint promptly on Ctrl-C and match lint paths without the Windows verbatim prefix --- .../native_threads/interruptible_join.rs | 32 +++++++++++ .../bindings/_helpers/native_threads/mod.rs | 1 + .../planner_migrations/column_entries.rs | 2 +- .../planner_migrations/discovery_models.rs | 4 +- .../planner_migrations/model_overlay.rs | 2 +- .../planner_migrations/planning_execution.rs | 4 +- .../planner_migrations/planning_origin.rs | 6 +- .../_helpers/planner_phases/buildability.rs | 10 ++-- .../planner_phases/execution_changes.rs | 16 +++--- .../planner_phases/output_assembly.rs | 26 ++++----- .../_helpers/planner_phases/retention.rs | 14 ++--- .../planner_phases/warehouse_state.rs | 20 +++---- .../bindings/_helpers/rules/early_sql_lint.rs | 14 +++-- .../bindings/_helpers/rules/lint_pipeline.rs | 4 +- .../src/bindings/_helpers/rules/sql_rules.rs | 4 ++ .../src/project_tree/main/mod.rs | 1 + .../project_tree/main/project_real_path.rs | 8 +++ .../cli/commands/main/compile/_test_types.py | 10 ++++ .../cli/commands/main/compile/helpers.py | 43 ++++++++++++++ .../main/compile/test_compile_early_lint.py | 57 +++++++++++++++++++ 20 files changed, 220 insertions(+), 58 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs create mode 100644 crates/sqlbuild-rules/src/project_tree/main/project_real_path.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs new file mode 100644 index 0000000000..2c03b0b48e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/interruptible_join.rs @@ -0,0 +1,32 @@ +//! Joining a native thread from Python's main thread without hiding Ctrl+C. + +use std::thread::JoinHandle; +use std::time::Duration; + +use pyo3::prelude::{PyErr, Python}; + +/// How often a join checks for pending signals such as Ctrl+C. +const SIGNAL_POLL_INTERVAL: Duration = Duration::from_millis(5); + +/// A joined thread's outcome, or the signal raised while waiting for it. +pub(crate) enum Joined { + Finished(std::thread::Result), + Interrupted(PyErr), +} + +/// Join `thread` without the GIL; on a signal call `interrupt`, finish the join and return the signal. +pub(crate) fn join_interruptibly( + py: Python<'_>, + thread: JoinHandle, + interrupt: impl FnOnce(), +) -> Joined { + while !thread.is_finished() { + if let Err(signal) = py.check_signals() { + interrupt(); + let _ = py.detach(|| thread.join()); + return Joined::Interrupted(signal); + } + py.detach(|| std::thread::sleep(SIGNAL_POLL_INTERVAL)); + } + Joined::Finished(py.detach(|| thread.join())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs index 9d58356de9..c379120844 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_threads/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod constants; pub(crate) mod exit_signal; +pub(crate) mod interruptible_join; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs index 13028274ff..668ac1047d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/column_entries.rs @@ -171,7 +171,7 @@ pub(super) fn checked<'py>( &[ ( "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, + runtime_field(runtime, |runtime| &runtime.connection)?, ), ("destination", relation), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs index 4f55a6d0b6..41f8d72f48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/discovery_models.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; pub(super) fn ordered<'py>(runtime: &Bound<'py, PyAny>) -> PyResult>> { let mut models: BTreeMap> = BTreeMap::new(); let mut dependencies: BTreeMap> = BTreeMap::new(); - for item in runtime_field(&runtime, |runtime| &runtime.project)? + for item in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { @@ -77,7 +77,7 @@ pub(super) fn metadata<'py>( models: &[Bound<'py, PyAny>], ) -> PyResult> { let py = runtime.py(); - let project = runtime_field(&runtime, |runtime| &runtime.project)?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; let functions = native( py, NativeAttr::BuildFunctionLocalHashes, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs index 1db0e0aacc..6d8cca941a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_overlay.rs @@ -125,7 +125,7 @@ pub(super) fn overlay<'py>( } fn cursor_enabled(runtime: &Bound<'_, PyAny>, name: &Bound<'_, PyAny>) -> PyResult { - for item in runtime_field(&runtime, |runtime| &runtime.project)? + for item in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs index 2a86cbfd6d..2fd4af82e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_execution.rs @@ -26,7 +26,7 @@ fn migration_promotion<'py>( runtime: &Bound<'py, PyAny>, decision: &Bound<'py, PyAny>, ) -> PyResult> { - let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let promotion = registry::class(runtime.py(), RecordId::MigrationPromotion)?; if adapter .call_method0("supports_transactional_ddl")? @@ -48,7 +48,7 @@ fn migration_with_execution<'py>( entry: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let transient = if adapter.getattr("adapter_name")?.eq("snowflake")? { PyBool::new( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs index 9fea051c79..0f3289d8a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/planning_origin.rs @@ -33,7 +33,7 @@ fn named<'py>( runtime: &Bound<'py, PyAny>, state: &Bound<'py, PyAny>, ) -> PyResult> { - for item in runtime_field(&runtime, |runtime| &runtime.project)? + for item in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { @@ -74,7 +74,7 @@ fn input_error(py: Python<'_>, message: String) -> PyResult { #[pyfunction] #[pyo3(signature = (*, runtime))] fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult> { - let project = runtime_field(&runtime, |runtime| &runtime.project)?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; let fallback = project.getattr("effective_target_database")?; for item in project.getattr("models")?.try_iter()? { let database = item?.getattr("destination")?.getattr("database")?; @@ -89,7 +89,7 @@ fn migration_planning_database<'py>(runtime: &Bound<'py, PyAny>) -> PyResult(runtime: &Bound<'py, PyAny>) -> PyResult> { let schemas = PySet::empty(runtime.py())?; - for item in runtime_field(&runtime, |runtime| &runtime.project)? + for item in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs index 08e0504cc9..4372fa93b3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/buildability.rs @@ -18,7 +18,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( deferral: &Bound<'py, PyAny>, ) -> PyResult<()> { let py = runtime.py(); - let project = runtime_field(&runtime, |runtime| &runtime.project)?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; let selected_scope = scopes.getattr("selected_scope")?; let mut external_seeds: Vec<_> = Vec::new(); for seed in project.getattr("seeds")?.try_iter()? { @@ -35,7 +35,7 @@ pub(crate) fn check_selected_scope_buildability<'py>( ("upstream_deps", selected_scope.getattr("upstream_deps")?), ( "snapshot", - warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?, + warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, ), ( "deferred_relations", @@ -70,10 +70,10 @@ pub(crate) fn check_selected_scope_buildability<'py>( check_selected_source_tables_exist( &project, ( - &runtime_field(&runtime, |runtime| &runtime.adapter)?, - &runtime_field(&runtime, |runtime| &runtime.connection)?, + &runtime_field(runtime, |runtime| &runtime.adapter)?, + &runtime_field(runtime, |runtime| &runtime.connection)?, ), &selected_scope, - &warehouse_field(&warehouse, |warehouse| &warehouse.inspection_relations)?, + &warehouse_field(warehouse, |warehouse| &warehouse.inspection_relations)?, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 0a26626150..460fcb822d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -70,16 +70,16 @@ pub(crate) fn build_planner_entry_results<'py>( source_freshness: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let project = runtime_field(&runtime, |runtime| &runtime.project)?; - let config = runtime_field(&runtime, |runtime| &runtime.project_config)?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; + let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let relations = planner_relations_context(RelationsRequest { project: project.clone(), - adapter: runtime_field(&runtime, |runtime| &runtime.adapter)?, - connection: runtime_field(&runtime, |runtime| &runtime.connection)?, + adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, + connection: runtime_field(runtime, |runtime| &runtime.connection)?, scope: execution_scope.clone(), deferral: deferral.clone(), project_config: config.clone(), - local_config: runtime_field(&runtime, |runtime| &runtime.local_config)?, + local_config: runtime_field(runtime, |runtime| &runtime.local_config)?, known_source_columns: warehouse .getattr("inspection_relations")? .getattr("source_warehouse_columns")?, @@ -132,7 +132,7 @@ pub(crate) fn build_planner_entry_results<'py>( ), ( "invocation_time", - runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + runtime_field(runtime, |runtime| &runtime.invocation_time)?, ), ("max_microbatches", max_microbatches), ( @@ -147,9 +147,9 @@ pub(crate) fn build_planner_entry_results<'py>( )?; plan_entries(PlanEntriesRequest { project, - adapter: runtime_field(&runtime, |runtime| &runtime.adapter)?, + adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, scope: execution_scope.clone(), - snapshot: warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?, + snapshot: warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, relations, resolved_models: reconciliation.resolved_actions.getattr("models")?, cursor_overrides: overrides.getattr("cursor_overrides")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index b83327bd44..e310d807c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -93,11 +93,11 @@ pub(crate) fn base_plan_output<'py>( )?; let mut output = PlanOutputFields { fields: plan_output_fields(PlanOutputInputs { - project: runtime_field(&runtime, |runtime| &runtime.project)?, - adapter: runtime_field(&runtime, |runtime| &runtime.adapter)?, + project: runtime_field(runtime, |runtime| &runtime.project)?, + adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, scope: execution_scope.clone(), - snapshot: warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?, - relations: warehouse_field(&warehouse, |warehouse| &warehouse.inspection_relations)?, + snapshot: warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, + relations: warehouse_field(warehouse, |warehouse| &warehouse.inspection_relations)?, changes: reconciliation.changes.clone(), model_entries: entries.entries, model_warnings: entries.warnings, @@ -163,7 +163,7 @@ pub(crate) fn add_storage_policies<'py>( .is_truthy()? { let models = PyDict::new(py); - for model in runtime_field(&runtime, |runtime| &runtime.project)? + for model in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { @@ -178,7 +178,7 @@ pub(crate) fn add_storage_policies<'py>( ("models_by_name", models.into_any()), ( "fingerprints", - warehouse_field(&warehouse, |warehouse| &warehouse.migration_fingerprints)?, + warehouse_field(warehouse, |warehouse| &warehouse.migration_fingerprints)?, ), ( "dialect", @@ -191,18 +191,18 @@ pub(crate) fn add_storage_policies<'py>( } let entries = with_column_rename_hints( &entries, - &warehouse_field(&warehouse, |warehouse| &warehouse.column_rename_hints)?, + &warehouse_field(warehouse, |warehouse| &warehouse.column_rename_hints)?, )?; - let migration_entries = warehouse_field(&warehouse, |warehouse| &warehouse.migration_entries)?; + let migration_entries = warehouse_field(warehouse, |warehouse| &warehouse.migration_entries)?; let column_entries = - warehouse_field(&warehouse, |warehouse| &warehouse.column_migration_entries)?; + warehouse_field(warehouse, |warehouse| &warehouse.column_migration_entries)?; let old_name_entries = - warehouse_field(&warehouse, |warehouse| &warehouse.old_name_view_entries)?; - let old_name_views = warehouse_field(&warehouse, |warehouse| &warehouse.old_name_views)?; + warehouse_field(warehouse, |warehouse| &warehouse.old_name_view_entries)?; + let old_name_views = warehouse_field(warehouse, |warehouse| &warehouse.old_name_views)?; let warnings = concat( py, &[ - warehouse_field(&warehouse, |warehouse| &warehouse.migration_warnings)?, + warehouse_field(warehouse, |warehouse| &warehouse.migration_warnings)?, output.field("warnings")?, ], )?; @@ -254,7 +254,7 @@ pub(crate) fn add_plan_warnings<'py>( ("changes", stale_warning_changes.clone()), ( "snapshot", - warehouse_field(&warehouse, |warehouse| &warehouse.snapshot)?, + warehouse_field(warehouse, |warehouse| &warehouse.snapshot)?, ), ( "version_identities", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs index 2c8eec692f..e58fa52e5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs @@ -31,8 +31,8 @@ pub(crate) fn effective_target_config<'py>( runtime: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); - let project_config = runtime_field(&runtime, |runtime| &runtime.project_config)?; - let local_config = runtime_field(&runtime, |runtime| &runtime.local_config)?; + let project_config = runtime_field(runtime, |runtime| &runtime.project_config)?; + let local_config = runtime_field(runtime, |runtime| &runtime.local_config)?; let target_name = runtime .getattr("project")? .getattr("effective_target_name")?; @@ -65,7 +65,7 @@ fn selected_models<'py>( .into_iter() .collect(); let mut models: Vec<_> = Vec::new(); - for model in runtime_field(&runtime, |runtime| &runtime.project)? + for model in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { @@ -123,7 +123,7 @@ pub(crate) fn plan_table_types<'py>( let relations = warehouse .getattr("snapshot")? .getattr("existing_relations")?; - let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let materialization = registry::class(py, RecordId::MaterializationType)?; let mut entries: Vec<_> = Vec::new(); for model in models { @@ -286,11 +286,11 @@ fn inspect<'py>( request: &Bound<'py, PyAny>, ) -> PyResult> { call_with( - &runtime_field(&runtime, |runtime| &runtime.adapter)?.getattr("inspect_retention")?, + &runtime_field(runtime, |runtime| &runtime.adapter)?.getattr("inspect_retention")?, &[ ( "connection", - runtime_field(&runtime, |runtime| &runtime.connection)?, + runtime_field(runtime, |runtime| &runtime.connection)?, ), ("request", request.clone()), ], @@ -470,7 +470,7 @@ fn bigquery_retention<'py>( } type Scope = (Option, String); let mut managed: BTreeMap>> = BTreeMap::new(); - for model in runtime_field(&runtime, |runtime| &runtime.project)? + for model in runtime_field(runtime, |runtime| &runtime.project)? .getattr("models")? .try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs index ee8e78bf1f..8627463617 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/warehouse_state.rs @@ -21,7 +21,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use std::time::Instant; fn progress(runtime: &Bound<'_, PyAny>, message: &str) -> PyResult<()> { - let on_progress = runtime_field(&runtime, |runtime| &runtime.on_progress)?; + let on_progress = runtime_field(runtime, |runtime| &runtime.on_progress)?; if !on_progress.is_none() { on_progress.call1((message,))?; } @@ -38,7 +38,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ) -> PyResult> { let py = runtime.py(); let started = Instant::now(); - let project = runtime_field(&runtime, |runtime| &runtime.project)?; + let project = runtime_field(runtime, |runtime| &runtime.project)?; let fingerprints = Py::new( py, MigrationFingerprintCache::open(py, Some(project.getattr("compile_cache_dir")?))?, @@ -48,7 +48,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( progress(runtime, "Inspecting warehouse state...")?; let selected_scope = scopes.getattr("selected_scope")?; let full_refresh: bool = overrides.getattr("full_refresh")?.extract()?; - let config = runtime_field(&runtime, |runtime| &runtime.project_config)?; + let config = runtime_field(runtime, |runtime| &runtime.project_config)?; let cursor_scope = native( py, RecordId::CursorSnapshotScope, @@ -60,7 +60,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ), ( "invocation_time", - runtime_field(&runtime, |runtime| &runtime.invocation_time)?, + runtime_field(runtime, |runtime| &runtime.invocation_time)?, ), ( "start_cursor_config", @@ -73,7 +73,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( ("cursor_overrides", overrides.getattr("cursor_overrides")?), ], )?; - let adapter = runtime_field(&runtime, |runtime| &runtime.adapter)?; + let adapter = runtime_field(runtime, |runtime| &runtime.adapter)?; let stale_keys = pyo3::types::PyFrozenSet::new( py, scopes @@ -87,12 +87,12 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let snapshot = warehouse_snapshot(&SnapshotRequest { project: project.clone(), adapter: adapter.clone(), - connection: runtime_field(&runtime, |runtime| &runtime.connection)?, + connection: runtime_field(runtime, |runtime| &runtime.connection)?, execute: adapter.getattr("execute")?, selected_keys: stale_keys, full_refresh: false, full_refresh_model_names: effectively_full_refreshed_model_names(&project, full_refresh)?, - on_progress: runtime_field(&runtime, |runtime| &runtime.on_progress)?, + on_progress: runtime_field(runtime, |runtime| &runtime.on_progress)?, deferred_locations: deferral.getattr("deferred_locations")?, cursor_scope, })?; @@ -108,11 +108,11 @@ pub(crate) fn gather_planner_warehouse_state<'py>( let inspection_relations = planner_relations_context(RelationsRequest { project: project.clone(), adapter: adapter.clone(), - connection: runtime_field(&runtime, |runtime| &runtime.connection)?, + connection: runtime_field(runtime, |runtime| &runtime.connection)?, scope: scopes.getattr("inspection_scope")?, deferral: deferral.clone(), project_config: config, - local_config: runtime_field(&runtime, |runtime| &runtime.local_config)?, + local_config: runtime_field(runtime, |runtime| &runtime.local_config)?, known_source_columns: py.None().into_bound(py), relation_keys: py.None().into_bound(py), })?; @@ -161,7 +161,7 @@ pub(crate) fn gather_planner_warehouse_state<'py>( }, (&migration_entries, &column_entries), )?; - if !runtime_field(&runtime, |runtime| &runtime.on_progress)?.is_none() { + if !runtime_field(runtime, |runtime| &runtime.on_progress)?.is_none() { progress( runtime, &format!( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 0927a97207..fa20b90490 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -24,6 +24,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_attachment::sql_expansion_context::{ SqlExpansionInputs, present, }; +use crate::bindings::_helpers::native_threads::interruptible_join::{Joined, join_interruptibly}; use crate::bindings::_helpers::rules::lint_expansion::{ LintExpansionSources, lint_expansion_inputs, }; @@ -157,6 +158,7 @@ pub(crate) struct PreparedSqlLint { project_dir: PathBuf, thread: Mutex>>>, result: OnceLock>, + stop: Arc, } impl PreparedSqlLint { @@ -170,7 +172,7 @@ impl PreparedSqlLint { same_codes(&self.codes, codes) } - /// The lint's result, waiting without the GIL until it finishes. + /// The lint's result, waiting without the GIL until it finishes or a signal handler raises. pub(crate) fn result(&self, py: Python<'_>) -> &Result { self.result.get_or_init(|| { let thread = self @@ -178,11 +180,14 @@ impl PreparedSqlLint { .lock() .unwrap_or_else(PoisonError::into_inner) .take(); - match thread { - Some(thread) => py.detach(|| thread.join()).unwrap_or_else(|_| { + let Some(thread) = thread else { + return Err(PyRuntimeError::new_err("early SQL lint was already joined")); + }; + match join_interruptibly(py, thread, || self.stop.store(true, Ordering::Release)) { + Joined::Finished(joined) => joined.unwrap_or_else(|_| { Err(PyRuntimeError::new_err("early SQL lint thread panicked")) }), - None => Err(PyRuntimeError::new_err("early SQL lint was already joined")), + Joined::Interrupted(signal) => Err(signal), } }) } @@ -224,6 +229,7 @@ fn start_early_lint( project_dir: plan.project_dir, thread: Mutex::new(Some(thread)), result: OnceLock::new(), + stop: Arc::clone(stop), })) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index a392e5efcf..e4a14efaad 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -1,7 +1,6 @@ //! The lint pipeline over native data: header rules, expanded bodies, native SQL lint and suppressions. use std::collections::{BTreeSet, HashMap, HashSet}; -use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -30,6 +29,7 @@ use sqlbuild_rules::lint_text::models::{ AuthoredDiagnostic, ExpansionSpan, InterpolationSite, LintSqlBody, LintSqlOptions, Neutralized, RetainedViolation, SuppressibleViolation, SuppressionFinding, }; +use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use sqlbuild_rules::sql_lint::main::preparation::prepare; use sqlbuild_rules::sql_lint::types::PreparedSql; @@ -412,7 +412,7 @@ pub(crate) struct LintRequest<'a> { /// `path` resolved through the filesystem, or unchanged when it cannot be. pub(crate) fn resolved(path: &Path) -> PathBuf { - fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) + project_real_path(path) } /// `given` filtered to the selection, or the project's lint files read from disk. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 038fbec56e..a98ccc8576 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::Instant; +use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, @@ -320,6 +321,9 @@ fn evaluated_sql_rules( } let files: Vec = match prepared_result(run, py) { Some(Ok(result)) => result.outcome.files.clone(), + Some(Err(error)) if !error.is_instance_of::(py) => { + return Err(error.clone_ref(py)); + } Some(Err(_)) | None => match prepared_files { Some(files) => files, None => lint_sources(project_dir, None, None)?, diff --git a/crates/sqlbuild-rules/src/project_tree/main/mod.rs b/crates/sqlbuild-rules/src/project_tree/main/mod.rs index 4ef4911b9a..db27d63ca8 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/mod.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/mod.rs @@ -1,4 +1,5 @@ pub mod project_path_escapes; +pub mod project_real_path; pub mod project_relative_parts; pub mod project_tree_children; pub mod project_tree_descendants; diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_real_path.rs b/crates/sqlbuild-rules/src/project_tree/main/project_real_path.rs new file mode 100644 index 0000000000..46e2db8ea7 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/project_real_path.rs @@ -0,0 +1,8 @@ +use std::path::{Path, PathBuf}; + +use crate::project_tree::_helpers::files::real_path; + +/// `path.resolve()` of an existing path without Windows' verbatim prefix, or `path` when it cannot be resolved. +pub fn project_real_path(path: &Path) -> PathBuf { + real_path(path).unwrap_or_else(|_| path.to_path_buf()) +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index dfa12101fa..f618ca5b23 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -575,6 +575,16 @@ class EarlyLintCompileTestCase: expected_diagnostics: tuple[tuple[str, str, int, int], ...] +@dataclass(frozen=True) +class EarlyLintInterruptTestCase: + """A rules-enabled project whose lint-thread macro expansions are slow enough to interrupt.""" + + description: str + model_count: int + macro_seconds: float + expected_max_stop_seconds: float + + @dataclass(frozen=True) class CompileReuseInvalidationTestCase: """One compile input change that must make the stored compile miss.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index fd248cbac8..f8898863c0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4970,3 +4970,46 @@ def compile_with_binding_error(project_dir: Path) -> dict[str, Any]: assert result.returncode == 1, result.stdout + result.stderr payload: dict[str, Any] = json.loads(result.stdout) return payload + + +_SLOW_LINT_MACRO: str = ( + "import os\n" + "import threading\n" + "import time\n\n\n" + "def slow_total(amount: str) -> str:\n" + ' """Return the amount, pausing when the background SQL lint expands it."""\n' + " if threading.current_thread() is not threading.main_thread():\n" + " with open(os.environ['SLOW_MACRO_MARKER'], 'a', encoding='utf-8') as marker:\n" + " marker.write('expanding\\n')\n" + " time.sleep(float(os.environ['SLOW_MACRO_SECONDS']))\n" + " return amount\n" +) + + +def slow_lint_model_files(*, model_count: int) -> dict[str, str]: + """A macro used by a model, a function, and macro tests that only the background SQL lint expands.""" + + return { + "macros/slow.py": _SLOW_LINT_MACRO, + "functions/sql/slow_amount.sql": ( + 'FUNCTION (description "Slow amount", arguments (p_amount INTEGER), returns INTEGER);\n' + 'SELECT @slow_total("p_amount")\n' + ), + "models/slow_total.sql": ( + 'MODEL (description "Slow total");\n' + 'SELECT @slow_total("amount") AS amount FROM (SELECT 5 AS amount) AS input_values\n' + ), + **{ + f"tests/slow_total_{index}.sql": ( + f'TEST (mode macro, name "slow_total_{index}");\n\n' + "WITH\n" + "input_values AS (SELECT 5 AS amount),\n" + "__macro_actual__ AS (\n" + ' SELECT @slow_total("amount") AS amount FROM input_values\n' + "),\n" + "__macro_expected__ AS (SELECT 5 AS amount)\n" + "SELECT 1\n" + ) + for index in range(model_count) + }, + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py index e350906c26..8ca339f922 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_lint.py @@ -2,6 +2,11 @@ from __future__ import annotations +import os +import signal +import subprocess +import sys +import time from pathlib import Path from typing import Any @@ -9,10 +14,12 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( EarlyLintCompileTestCase, + EarlyLintInterruptTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( compile_json_payload, diagnostic_location_keys, + slow_lint_model_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project @@ -87,3 +94,53 @@ def test_given_rules_enabled_project_when_compiling_cold_and_warm_then_lint_wait wait_ms: object = payload["compile_timings"]["early_lint_wait_ms"] assert isinstance(wait_ms, int) assert wait_ms >= 0 + + +@pytest.mark.parametrize( + "test_case", + ( + EarlyLintInterruptTestCase( + description="ctrl-c stops after the file being expanded", + model_count=8, + macro_seconds=3.0, + expected_max_stop_seconds=8.0, + ), + ), + ids=lambda case: case.description, +) +def test_given_slow_background_lint_when_interrupted_then_compile_stops_promptly( + test_case: EarlyLintInterruptTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="commerce", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + **slow_lint_model_files(model_count=test_case.model_count), + }, + ) + marker: Path = tmp_path / "slow_macro_marker.txt" + process: subprocess.Popen[str] = subprocess.Popen( + [str(Path(sys.executable).with_name("sqb")), "--project-dir", str(project_dir), "compile"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + env={ + **os.environ, + "SLOW_MACRO_MARKER": str(marker), + "SLOW_MACRO_SECONDS": str(test_case.macro_seconds), + }, + ) + deadline: float = time.monotonic() + 60.0 + while not marker.exists() and time.monotonic() < deadline and process.poll() is None: + time.sleep(0.05) + interrupted_at: float = time.monotonic() + process.send_signal(signal.SIGINT) + output: str = process.communicate(timeout=120)[0] + stop_seconds: float = time.monotonic() - interrupted_at + + assert process.returncode in (-signal.SIGINT, 130), output + assert stop_seconds < test_case.expected_max_stop_seconds, output + assert marker.read_text(encoding="utf-8").count("expanding") < test_case.model_count From d197166d5dd4bebc1ce937f3a7f277d38149b52a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:57:55 +0100 Subject: [PATCH 11/12] fix: raise Ctrl-C caught while stopping early SQL lint --- .../_helpers/compile_driver/analysis.rs | 4 ++-- .../bindings/_helpers/rules/early_sql_lint.rs | 24 ++++++++++++++----- .../cli/commands/main/compile/helpers.py | 2 +- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index b3fd172a9a..cb6fbaf5bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -182,7 +182,7 @@ fn analyzed<'py>( let graph_failed: bool = any_error(&project.getattr("diagnostics")?)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { - early_lint_wait_ms = early_lint.borrow().stop(py).into_pyobject(py)?.into_any(); + early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, &dialect)?; @@ -210,7 +210,7 @@ fn analyzed<'py>( (&core_diagnostics, &selected_keys), &lint_dialect, )?; - early_lint_wait_ms = early_lint.borrow().stop(py).into_pyobject(py)?.into_any(); + early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); result }; let diagnostics: Bound<'py, PyTuple> = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index fa20b90490..6ea9a18e7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -9,7 +9,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, OnceLock, PoisonError}; use std::thread::JoinHandle; -use pyo3::exceptions::PyRuntimeError; +use pyo3::exceptions::{PyException, PyRuntimeError}; use pyo3::prelude::{ Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRefMut, PyResult, Python, @@ -172,6 +172,11 @@ impl PreparedSqlLint { same_codes(&self.codes, codes) } + /// Whether the lint's result is already known. + fn finished(&self) -> bool { + self.result.get().is_some() + } + /// The lint's result, waiting without the GIL until it finishes or a signal handler raises. pub(crate) fn result(&self, py: Python<'_>) -> &Result { self.result.get_or_init(|| { @@ -307,14 +312,21 @@ impl EarlySqlLint { self.expansion_reuse.as_ref() } - /// Abandon unconsumed speculative work and return the milliseconds spent joining it. - pub(crate) fn stop(&self, py: Python<'_>) -> u64 { + /// Abandon speculative work, return the milliseconds spent joining it, and raise Ctrl+C caught while waiting. + pub(crate) fn stop(&self, py: Python<'_>) -> PyResult { let started: Instant = Instant::now(); self.stop_requested.store(true, Ordering::Release); if let Some(preparation) = &self.preparation { - let _ = preparation.get().result(py); + let preparation: &PreparedSqlLint = preparation.get(); + let waiting: bool = !preparation.finished(); + if let Err(error) = preparation.result(py) + && waiting + && !error.is_instance_of::(py) + { + return Err(error.clone_ref(py)); + } } - u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX) + Ok(u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX)) } } @@ -364,7 +376,7 @@ impl EarlySqlLint { /// Abandon unconsumed speculative work and return the milliseconds spent joining it. #[pyo3(name = "stop")] - fn py_stop(&self, py: Python<'_>) -> u64 { + fn py_stop(&self, py: Python<'_>) -> PyResult { self.stop(py) } } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index f8898863c0..2379aba549 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4987,7 +4987,7 @@ def compile_with_binding_error(project_dir: Path) -> dict[str, Any]: def slow_lint_model_files(*, model_count: int) -> dict[str, str]: - """A macro used by a model, a function, and macro tests that only the background SQL lint expands.""" + """A macro used by a model, a function, and macro tests only background SQL lint expands.""" return { "macros/slow.py": _SLOW_LINT_MACRO, From c387b459eed5b05752acf5458de96a36bb1da818 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 16 Oct 2026 16:52:23 +0100 Subject: [PATCH 12/12] test: stop spying on project resource assembly, now called natively --- tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py | 1 - .../cli/commands/main/compile/test_native_analysis_seams.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 2379aba549..2fd96350e6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4806,7 +4806,6 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( - (native_module, "assemble_native_project_resources"), (native_module, "infer_native_expression_source_shapes"), (native_module, "analyze_native_model_sql"), (native_module, "assemble_native_sql_tests"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 4986554e52..b9b48cb311 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -11,7 +11,6 @@ from sqlbuild._native import ( ContractValidationResult, NativeModelAnalyses, - NativeProjectResources, NativeSqlTestArtifact, NativeSqlTestAssembly, ProjectColumnLineage, @@ -31,7 +30,6 @@ NativeAnalysisSeamTestCase( description="models_contracts_lineage_and_sql_tests", expected_native_returns={ - "assemble_native_project_resources": [ANY], "infer_native_expression_source_shapes": [ANY, ANY, ANY, ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [ANY], @@ -65,7 +63,6 @@ def test_given_project_when_compiling_then_native_seams_answer( planned_counts: list[int] = [] for seams in (native_seams,): assert isinstance(seams["build_project_column_lineage"][0], ProjectColumnLineage) - assert isinstance(seams["assemble_native_project_resources"][0], NativeProjectResources) assert isinstance(seams["analyze_native_model_sql"][0], NativeModelAnalyses) assert seams["analyze_native_model_sql"][0].session is not None assert isinstance(seams["infer_native_expression_source_shapes"][0], tuple)