From 0e654eb6b4c1f7c16aeb72006fbc08f48b843626 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 07:59:58 +0100 Subject: [PATCH 01/28] refactor: answer native string enum value and name reads from their declared members --- .../src/bindings/_helpers/records/fields.rs | 66 +++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index 3539a0e634..a3001a18c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -5,8 +5,9 @@ use std::sync::{Mutex, OnceLock}; use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyAttributeError, PyRuntimeError, PyTypeError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; use pyo3::pyclass::CompareOp; +use pyo3::sync::PyOnceLock; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyStringMethods, PyTuple, PyTupleMethods, PyType, PyTypeMethods, @@ -17,17 +18,26 @@ use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::constants::{ - CACHED_PROPERTIES, CLASS_PROPERTIES, FROZEN_INSTANCE_ERROR, NATIVE_MODULE, ORDERED_RECORDS, - RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, + CACHED_PROPERTIES, CLASS_PROPERTIES, ENUMS, FROZEN_INSTANCE_ERROR, NATIVE_MODULE, + ORDERED_RECORDS, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, }; use crate::bindings::_helpers::records::construct::{enum_member, new_record}; use crate::bindings::_helpers::records::getters::getter_field; -use crate::bindings::_helpers::records::models::{FieldDefault, NativeRecord, RecordSpec}; +use crate::bindings::_helpers::records::models::{ + EnumSpec, FieldDefault, NativeRecord, RecordSpec, +}; use crate::bindings::_helpers::records::types::ClassProperty; /// How a test run treats `field` falling back to `getattr` on a native class: unset outside tests. pub(crate) const NATIVE_FIELD_STRICT_ENV_VAR: &str = "SQLBUILD_NATIVE_FIELD_STRICT"; +/// The string enum member attributes answered natively. +const ENUM_VALUE_ATTRIBUTE: &str = "value"; +const ENUM_NAME_ATTRIBUTE: &str = "name"; + +/// Native string enums by class object, for `.value` and `.name` reads answered from their members. +static ENUM_CLASSES: PyOnceLock> = PyOnceLock::new(); + /// Every registered record spec, in registration order. pub(crate) fn all_specs() -> impl Iterator { RECORDS.iter().flat_map(|group| group.iter()) @@ -67,6 +77,8 @@ pub(crate) fn field<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult( + value: &Bound<'py, PyAny>, + name: &str, +) -> Option>> { + let wants_value: bool = match name { + ENUM_VALUE_ATTRIBUTE => true, + ENUM_NAME_ATTRIBUTE => false, + _ => return None, + }; + let py: Python<'py> = value.py(); + let classes: &HashMap = + match ENUM_CLASSES.get_or_try_init(py, || enum_classes(py)) { + Ok(classes) => classes, + Err(error) => return Some(Err(error)), + }; + let spec: &EnumSpec = classes.get(&(value.get_type().as_ptr() as usize))?; + let Ok(member) = value.cast::() else { + return None; + }; + let text: String = match member.to_str() { + Ok(text) => text.to_owned(), + Err(error) => return Some(Err(error)), + }; + let (member_name, member_value) = spec + .members + .iter() + .find(|(_, declared)| *declared == text)?; + let answer: &str = if wants_value { + member_value + } else { + member_name + }; + Some(Ok(PyString::new(py, answer).into_any())) +} + +fn enum_classes(py: Python<'_>) -> PyResult> { + let module: Bound<'_, PyModule> = py.import(NATIVE_MODULE)?; + let mut classes: HashMap = HashMap::new(); + for spec in ENUMS.iter().flat_map(|group| group.iter()) { + let class: Bound<'_, PyAny> = module.getattr(spec.name)?; + let _ = classes.insert(class.as_ptr() as usize, spec); + } + Ok(classes) +} + /// `mapping.get(key)`: a dict is read through the C API. pub(crate) fn mapping_get<'py, K: pyo3::IntoPyObject<'py>>( mapping: &Bound<'py, PyAny>, From d71aac412221a8a02fc9da33d857dec18cd30dd7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 10:27:58 +0100 Subject: [PATCH 02/28] perf: hash SQL rule cache keys in parallel without the GIL, answer Python enum reads natively and read record fields from the native spec --- .../_helpers/compiled_project/records.rs | 13 +-- .../src/bindings/_helpers/records/fields.rs | 104 +++++++++++++++++- .../src/bindings/_helpers/rules/rule_cache.rs | 26 +++-- .../src/bindings/_helpers/rules/sql_rules.rs | 3 +- .../sqlbuild-rules/src/rule_cache/main/mod.rs | 1 + .../src/rule_cache/main/model_rule_keys.rs | 19 ++++ crates/sqlbuild-rules/src/rule_cache/mod.rs | 3 + crates/sqlbuild-rules/src/rule_cache/tests.rs | 4 + .../src/rule_cache/tests/test_rule_cache.rs | 50 +++++++++ .../src/rule_cache/tests/test_types.rs | 4 + 10 files changed, 208 insertions(+), 19 deletions(-) create mode 100644 crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/tests.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs create mode 100644 crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index 08671c4219..503d7026a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -3,7 +3,9 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple, PyType, +}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; @@ -16,6 +18,7 @@ use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView use crate::bindings::_helpers::compiled_project::schema_facts::entry_schema_fact; use crate::bindings::_helpers::records::construct::{enum_member, new_record, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::record_spec; use crate::bindings::_helpers::records::ids::RecordId; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -44,12 +47,8 @@ fn record<'py>(value: &Bound<'py, PyAny>, class: RecordId) -> PyResult = PyDict::new(py); - for field in imported(py, DATACLASSES_MODULE, "fields")? - .call1((&class,))? - .try_iter()? - { - let name: Bound<'py, PyAny> = native_field(&field?, "name")?; - fields.set_item(&name, native_field(value, &name.extract::()?)?)?; + for field in record_spec(class.cast::()?)?.fields { + fields.set_item(field.name, native_field(value, field.name)?)?; } class.call((), Some(&fields)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index a3001a18c4..ea6b8f3b3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -38,6 +38,25 @@ const ENUM_NAME_ATTRIBUTE: &str = "name"; /// Native string enums by class object, for `.value` and `.name` reads answered from their members. static ENUM_CLASSES: PyOnceLock> = PyOnceLock::new(); +/// Each class seen by `enum_member_field`, held with whether it is a Python enum. +type PythonEnumClasses = HashMap, bool)>; + +/// Whether each class seen by `enum_member_field` is a Python enum, keyed by class object. +static PYTHON_ENUM_CLASSES: Mutex> = Mutex::new(None); + +/// The name and value of each Python enum member read so far, keyed by member object. +static PYTHON_ENUM_MEMBERS: Mutex>> = Mutex::new(None); + +/// The module and class every Python enum derives from. +const ENUM_BASE: (&str, &str) = ("enum", "Enum"); + +/// A Python enum member with its name and value, held so the member identity stays valid. +struct PythonEnumMember { + _member: Py, + name: Py, + value: Py, +} + /// Every registered record spec, in registration order. pub(crate) fn all_specs() -> impl Iterator { RECORDS.iter().flat_map(|group| group.iter()) @@ -216,7 +235,9 @@ fn enum_member_field<'py>( Ok(classes) => classes, Err(error) => return Some(Err(error)), }; - let spec: &EnumSpec = classes.get(&(value.get_type().as_ptr() as usize))?; + let Some(spec) = classes.get(&(value.get_type().as_ptr() as usize)) else { + return python_enum_member_field(value, wants_value); + }; let Ok(member) = value.cast::() else { return None; }; @@ -236,6 +257,87 @@ fn enum_member_field<'py>( Some(Ok(PyString::new(py, answer).into_any())) } +/// `member.value` or `member.name` of a Python enum, read once per member and then answered natively. +fn python_enum_member_field<'py>( + value: &Bound<'py, PyAny>, + wants_value: bool, +) -> Option>> { + let py: Python<'py> = value.py(); + let identity: usize = value.as_ptr() as usize; + if let Some(answer) = cached_python_enum_member(py, identity, wants_value) { + return Some(Ok(answer)); + } + match python_enum_class_check(value) { + Ok(true) => {} + Ok(false) => return None, + Err(error) => return Some(Err(error)), + } + let member: PythonEnumMember = match read_python_enum_member(value) { + Ok(member) => member, + Err(error) => return Some(Err(error)), + }; + let answer: Bound<'py, PyAny> = if wants_value { + member.value.bind(py).clone() + } else { + member.name.bind(py).clone() + }; + let mut members = PYTHON_ENUM_MEMBERS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _ = members + .get_or_insert_with(HashMap::new) + .entry(identity) + .or_insert(member); + Some(Ok(answer)) +} + +fn cached_python_enum_member<'py>( + py: Python<'py>, + identity: usize, + wants_value: bool, +) -> Option> { + let members = PYTHON_ENUM_MEMBERS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let member: &PythonEnumMember = members.as_ref()?.get(&identity)?; + let answer: &Py = if wants_value { + &member.value + } else { + &member.name + }; + Some(answer.bind(py).clone()) +} + +/// Whether `value`'s class derives from `enum.Enum`, classified once per class. +fn python_enum_class_check(value: &Bound<'_, PyAny>) -> PyResult { + let class: Bound<'_, PyType> = value.get_type(); + let identity: usize = class.as_ptr() as usize; + let known: Option = PYTHON_ENUM_CLASSES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + .and_then(|classes| classes.get(&identity).map(|(_, is_enum)| *is_enum)); + if let Some(is_enum) = known { + return Ok(is_enum); + } + let is_enum: bool = class.is_subclass(&imported(value.py(), ENUM_BASE.0, ENUM_BASE.1)?)?; + let _ = PYTHON_ENUM_CLASSES + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get_or_insert_with(HashMap::new) + .entry(identity) + .or_insert((class.into_any().unbind(), is_enum)); + Ok(is_enum) +} + +fn read_python_enum_member(value: &Bound<'_, PyAny>) -> PyResult { + Ok(PythonEnumMember { + _member: value.clone().unbind(), + name: value.getattr(ENUM_NAME_ATTRIBUTE)?.unbind(), + value: value.getattr(ENUM_VALUE_ATTRIBUTE)?.unbind(), + }) +} + fn enum_classes(py: Python<'_>) -> PyResult> { let module: Bound<'_, PyModule> = py.import(NATIVE_MODULE)?; let mut classes: HashMap = HashMap::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index ac9dcc3ecf..c764e1d800 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::rules::findings::FindingClasses; use sqlbuild_rules::rule_cache::main::lint_key::lint_key; use sqlbuild_rules::rule_cache::main::lookup_rule_cache::lookup_rule_cache; -use sqlbuild_rules::rule_cache::main::model_rule_key::model_rule_key; +use sqlbuild_rules::rule_cache::main::model_rule_keys::model_rule_keys as parallel_model_rule_keys; use sqlbuild_rules::rule_cache::main::parse_rule_cache::parse_rule_cache; use sqlbuild_rules::rule_cache::main::rule_cache_text::rule_cache_text; use sqlbuild_rules::rule_cache::models::{CacheLookup, CachedEntry, LintKeyInput, ModelKeyInput}; @@ -56,29 +56,35 @@ fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { #[pyfunction] #[pyo3(signature = (*, lint_key, codes, models))] fn sql_model_rule_keys( + py: Python<'_>, lint_key: &str, codes: Vec, models: Vec<(String, String, String, Option)>, ) -> BTreeMap { - model_rule_keys(lint_key, &codes, &models) + model_rule_keys(py, lint_key, &codes, &models) } /// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. +/// Hashed in parallel with the GIL released: the inputs are owned Rust strings. pub(crate) fn model_rule_keys( + py: Python<'_>, lint_key: &str, codes: &[String], models: &[(String, String, String, Option)], ) -> BTreeMap { - models + let inputs: Vec> = models .iter() - .map(|(path, authored_sql, query_sql, contract)| { - let model = ModelKeyInput { - authored_sql, - query_sql, - contract: contract.as_deref(), - }; - (path.clone(), model_rule_key(lint_key, codes, &model)) + .map(|(_, authored_sql, query_sql, contract)| ModelKeyInput { + authored_sql, + query_sql, + contract: contract.as_deref(), }) + .collect(); + let keys: Vec = py.detach(|| parallel_model_rule_keys(lint_key, codes, &inputs)); + models + .iter() + .map(|(path, ..)| path.clone()) + .zip(keys) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 370ad14585..6a3c3f27c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -292,7 +292,8 @@ fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult = model_rule_keys(lint_key, run.codes, &models); + let keys: BTreeMap = + model_rule_keys(run.project.py(), lint_key, run.codes, &models); Ok(models .into_iter() .map(|(path, ..)| { diff --git a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs index 20a4012180..3fd43bd89e 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs @@ -2,6 +2,7 @@ pub mod file_rule_key; pub mod lint_key; pub mod lookup_rule_cache; pub mod model_rule_key; +pub mod model_rule_keys; pub mod parse_rule_cache; pub mod rule_cache_text; pub mod rules_bulk_cache_path; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs new file mode 100644 index 0000000000..f31271f9de --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs @@ -0,0 +1,19 @@ +//! `model_rule_keys`: the SQL rule cache keys of many models, hashed in parallel. + +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +use crate::rule_cache::main::model_rule_key::model_rule_key; +use crate::rule_cache::models::ModelKeyInput; + +/// Each model's `model_rule_key`, in input order. +#[must_use] +pub fn model_rule_keys( + lint_key: &str, + codes: &[String], + models: &[ModelKeyInput<'_>], +) -> Vec { + models + .par_iter() + .map(|model| model_rule_key(lint_key, codes, model)) + .collect() +} diff --git a/crates/sqlbuild-rules/src/rule_cache/mod.rs b/crates/sqlbuild-rules/src/rule_cache/mod.rs index 76c842f679..1644773e62 100644 --- a/crates/sqlbuild-rules/src/rule_cache/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/mod.rs @@ -3,3 +3,6 @@ pub(crate) mod constants; pub mod main; pub mod models; pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/rule_cache/tests.rs b/crates/sqlbuild-rules/src/rule_cache/tests.rs new file mode 100644 index 0000000000..18a0eb0074 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_rule_cache.rs"] +mod rule_cache; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs b/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs new file mode 100644 index 0000000000..ce98fd39c9 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs @@ -0,0 +1,50 @@ +use crate::rule_cache::main::model_rule_key::model_rule_key; +use crate::rule_cache::main::model_rule_keys::model_rule_keys; +use crate::rule_cache::models::ModelKeyInput; +use crate::rule_cache::tests::test_types::ModelRuleKeysTestCase; + +#[test] +fn given_models_when_hashing_rule_keys_together_then_each_key_matches_its_single_model_key_in_order() + { + let test_cases = [ + ModelRuleKeysTestCase { + description: "no models give no keys", + models: vec![], + }, + ModelRuleKeysTestCase { + description: "many models keep their order and single-model keys", + models: (0..64) + .map(|index| { + let sql: &'static str = + Box::leak(format!("select {index} as id").into_boxed_str()); + let contract: Option<&'static str> = (index % 3 == 0).then_some("id:integer"); + (sql, sql, contract) + }) + .collect(), + }, + ]; + let codes: Vec = vec!["SQBRSQL001".to_owned(), "SQBRSQL033".to_owned()]; + + for test_case in test_cases { + let inputs: Vec> = test_case + .models + .iter() + .map(|(authored_sql, query_sql, contract)| ModelKeyInput { + authored_sql, + query_sql, + contract: *contract, + }) + .collect(); + let expected: Vec = inputs + .iter() + .map(|model| model_rule_key("lint-key", &codes, model)) + .collect(); + + assert_eq!( + model_rule_keys("lint-key", &codes, &inputs), + expected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs b/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs new file mode 100644 index 0000000000..9abdde79a7 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs @@ -0,0 +1,4 @@ +pub(crate) struct ModelRuleKeysTestCase { + pub(crate) description: &'static str, + pub(crate) models: Vec<(&'static str, &'static str, Option<&'static str>)>, +} From 23d8879592282ad58f042fa8f4963eb894b6797a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 10:59:05 +0100 Subject: [PATCH 03/28] perf: make written artifact paths absolute natively and tidy the parallel rule cache keys --- .../sqlbuild-core/src/posix_path/constants.rs | 2 + .../sqlbuild-core/src/posix_path/main/mod.rs | 1 + .../src/posix_path/main/posix_abspath.rs | 49 +++++++ .../src/posix_path/tests/test_posix_path.rs | 120 +++++++++++++++++- .../src/posix_path/tests/test_types.rs | 7 + .../compile_command/reuse_recording.rs | 20 ++- .../src/bindings/_helpers/records/fields.rs | 104 +-------------- .../src/bindings/_helpers/rules/rule_cache.rs | 3 +- .../src/rule_cache/_helpers/mod.rs | 1 + .../{main => _helpers}/model_rule_key.rs | 6 +- .../sqlbuild-rules/src/rule_cache/main/mod.rs | 1 - .../src/rule_cache/main/model_rule_keys.rs | 2 +- .../src/rule_cache/tests/test_rule_cache.rs | 11 +- .../src/rule_cache/tests/test_types.rs | 1 + fensu.toml | 2 +- 15 files changed, 216 insertions(+), 114 deletions(-) create mode 100644 crates/sqlbuild-core/src/posix_path/main/posix_abspath.rs rename crates/sqlbuild-rules/src/rule_cache/{main => _helpers}/model_rule_key.rs (86%) diff --git a/crates/sqlbuild-core/src/posix_path/constants.rs b/crates/sqlbuild-core/src/posix_path/constants.rs index 9216305dc5..20101ec03f 100644 --- a/crates/sqlbuild-core/src/posix_path/constants.rs +++ b/crates/sqlbuild-core/src/posix_path/constants.rs @@ -6,3 +6,5 @@ pub(crate) const ROOT: &str = "/"; pub(crate) const DOUBLE_ROOT: &str = "//"; pub(crate) const CURRENT: &str = "."; pub(crate) const SUFFIX_MARK: char = '.'; +/// The parent-directory component `normpath` removes lexically. +pub(crate) const PARENT: &str = ".."; diff --git a/crates/sqlbuild-core/src/posix_path/main/mod.rs b/crates/sqlbuild-core/src/posix_path/main/mod.rs index e4ab185549..d135530cc5 100644 --- a/crates/sqlbuild-core/src/posix_path/main/mod.rs +++ b/crates/sqlbuild-core/src/posix_path/main/mod.rs @@ -1,3 +1,4 @@ +pub mod posix_abspath; pub mod posix_join; pub mod posix_name; pub mod posix_parent; diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_abspath.rs b/crates/sqlbuild-core/src/posix_path/main/posix_abspath.rs new file mode 100644 index 0000000000..8647b2a3ee --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/posix_abspath.rs @@ -0,0 +1,49 @@ +//! `posixpath.abspath(path)` for a process working directory `cwd`. + +use crate::posix_path::constants::{CURRENT, DOUBLE_ROOT, PARENT, ROOT, SEPARATOR}; + +/// `path` joined to `cwd` when relative, then normalised exactly as `posixpath.normpath` does. +#[must_use] +pub fn posix_abspath(cwd: &str, path: &str) -> String { + if path.starts_with(SEPARATOR) { + return posix_normpath(path); + } + let joined: String = if cwd.is_empty() || cwd.ends_with(SEPARATOR) { + format!("{cwd}{path}") + } else { + format!("{cwd}{ROOT}{path}") + }; + posix_normpath(&joined) +} + +fn posix_normpath(path: &str) -> String { + if path.is_empty() { + return CURRENT.to_owned(); + } + let root: &str = if path.starts_with(DOUBLE_ROOT) && !path.starts_with("///") { + DOUBLE_ROOT + } else if path.starts_with(SEPARATOR) { + ROOT + } else { + "" + }; + let mut kept: Vec<&str> = Vec::new(); + for part in path.split(SEPARATOR) { + if part.is_empty() || part == CURRENT { + continue; + } + let keeps_parent: bool = + (root.is_empty() && kept.is_empty()) || kept.last() == Some(&PARENT); + if part != PARENT || keeps_parent { + kept.push(part); + } else if !kept.is_empty() { + let _ = kept.pop(); + } + } + let normalised: String = format!("{root}{}", kept.join(ROOT)); + if normalised.is_empty() { + CURRENT.to_owned() + } else { + normalised + } +} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs index 4722f0737b..93fcae39f3 100644 --- a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs +++ b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs @@ -1,9 +1,10 @@ +use crate::posix_path::main::posix_abspath::posix_abspath; use crate::posix_path::main::posix_join::posix_join; use crate::posix_path::main::posix_name::posix_name; use crate::posix_path::main::posix_parent::posix_parent; use crate::posix_path::main::posix_parts::posix_parts; use crate::posix_path::main::posix_suffix::posix_suffix; -use crate::posix_path::tests::test_types::PosixPathTestCase; +use crate::posix_path::tests::test_types::{PosixAbspathTestCase, PosixPathTestCase}; #[test] fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { @@ -140,3 +141,120 @@ fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { ); } } + +#[test] +fn given_relative_and_absolute_paths_when_making_them_absolute_then_they_match_posixpath_abspath() { + let test_cases = [ + PosixAbspathTestCase { + description: "a relative artifact path", + cwd: "/home/u/proj", + path: "target/compiled/a.sql", + expected_path: "/home/u/proj/target/compiled/a.sql", + }, + PosixAbspathTestCase { + description: "an absolute path ignores the working directory", + cwd: "/home/u/proj", + path: "/abs/x.sql", + expected_path: "/abs/x.sql", + }, + PosixAbspathTestCase { + description: "current and empty components are dropped", + cwd: "/home/u/proj", + path: "./a//b/", + expected_path: "/home/u/proj/a/b", + }, + PosixAbspathTestCase { + description: "a parent component climbs out of the working directory", + cwd: "/home/u/proj", + path: "../up/x", + expected_path: "/home/u/up/x", + }, + PosixAbspathTestCase { + description: "parents stop at the root", + cwd: "/home/u/proj", + path: "a/../../../../x", + expected_path: "/x", + }, + PosixAbspathTestCase { + description: "a root working directory", + cwd: "/", + path: "rel", + expected_path: "/rel", + }, + PosixAbspathTestCase { + description: "a parent of the root stays at the root", + cwd: "/", + path: "../x", + expected_path: "/x", + }, + PosixAbspathTestCase { + description: "a working directory with a trailing slash", + cwd: "/home/u/", + path: "x", + expected_path: "/home/u/x", + }, + PosixAbspathTestCase { + description: "exactly two leading slashes are kept", + cwd: "/home/u/proj", + path: "//net/x", + expected_path: "//net/x", + }, + PosixAbspathTestCase { + description: "three leading slashes collapse to one", + cwd: "/home/u/proj", + path: "///three/x", + expected_path: "/three/x", + }, + PosixAbspathTestCase { + description: "a double-slash working directory keeps its root", + cwd: "//net/share", + path: "x/../y", + expected_path: "//net/share/y", + }, + PosixAbspathTestCase { + description: "an empty path is the working directory", + cwd: "/home/u/proj", + path: "", + expected_path: "/home/u/proj", + }, + PosixAbspathTestCase { + description: "a dot is the working directory", + cwd: "/home/u/proj", + path: ".", + expected_path: "/home/u/proj", + }, + PosixAbspathTestCase { + description: "dots between components are dropped", + cwd: "/home/u/proj", + path: "a/./b/./", + expected_path: "/home/u/proj/a/b", + }, + PosixAbspathTestCase { + description: "climbing past the root gives the root", + cwd: "/home/u/proj", + path: "/a/b/../../..", + expected_path: "/", + }, + PosixAbspathTestCase { + description: "climbing past a double-slash root keeps it", + cwd: "/home/u/proj", + path: "//a/../..", + expected_path: "//", + }, + PosixAbspathTestCase { + description: "three dots are an ordinary name", + cwd: "/home/u/proj", + path: "x/.../y", + expected_path: "/home/u/proj/x/.../y", + }, + ]; + + for test_case in test_cases { + assert_eq!( + posix_abspath(test_case.cwd, test_case.path), + test_case.expected_path, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs index 126c6e31d9..69f579487f 100644 --- a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs +++ b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs @@ -7,3 +7,10 @@ pub(super) struct PosixPathTestCase { /// `suffix` under Python 3.12 and under Python 3.14. pub(super) expected_suffix: &'static str, } + +pub(super) struct PosixAbspathTestCase { + pub(super) description: &'static str, + pub(super) cwd: &'static str, + pub(super) path: &'static str, + pub(super) expected_path: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 00f2e51c8d..1b80e2cfef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -1,6 +1,7 @@ //! What one full compile printed and left in each artifact, and storing it for compile reuse. use std::collections::HashMap; +use std::path::PathBuf; use std::sync::{Mutex, PoisonError}; use pyo3::exceptions::{PyException, PyOSError}; @@ -8,6 +9,8 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::posix_path::main::posix_abspath::posix_abspath; + use crate::bindings::_helpers::compile_command::constants::{ BYTES_PER_MEBIBYTE, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_HIT_TIMING, REUSE_LOGGER_NAME, REUSE_MISS_TIMING, REUSE_STORE_DONE_PREFIX, REUSE_STORE_NOTICE_BYTES, @@ -101,13 +104,28 @@ impl CompileArtifactWrites { /// Where a compile notes the artifacts it writes; `None` outside a recorded compile. pub(crate) type ArtifactRecorder<'a> = Option<&'a Py>; -/// `os.path.abspath(path)`. +/// `os.path.abspath(path)`, computed natively on POSIX for UTF-8 paths and working directories. fn absolute(path: &Bound<'_, PyAny>) -> PyResult { + if cfg!(unix) + && let Some(found) = native_absolute(path) + { + return Ok(found); + } imported(path.py(), "os.path", "abspath")? .call1((path,))? .extract() } +fn native_absolute(path: &Bound<'_, PyAny>) -> Option { + let Ok(path) = path.extract::() else { + return None; + }; + let Ok(cwd) = std::env::current_dir() else { + return None; + }; + Some(posix_abspath(cwd.to_str()?, path.to_str()?)) +} + /// A new, empty recorder of one compile's artifact writes. pub(crate) fn new_artifact_recorder(py: Python<'_>) -> PyResult> { Py::new( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index ea6b8f3b3c..a3001a18c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -38,25 +38,6 @@ const ENUM_NAME_ATTRIBUTE: &str = "name"; /// Native string enums by class object, for `.value` and `.name` reads answered from their members. static ENUM_CLASSES: PyOnceLock> = PyOnceLock::new(); -/// Each class seen by `enum_member_field`, held with whether it is a Python enum. -type PythonEnumClasses = HashMap, bool)>; - -/// Whether each class seen by `enum_member_field` is a Python enum, keyed by class object. -static PYTHON_ENUM_CLASSES: Mutex> = Mutex::new(None); - -/// The name and value of each Python enum member read so far, keyed by member object. -static PYTHON_ENUM_MEMBERS: Mutex>> = Mutex::new(None); - -/// The module and class every Python enum derives from. -const ENUM_BASE: (&str, &str) = ("enum", "Enum"); - -/// A Python enum member with its name and value, held so the member identity stays valid. -struct PythonEnumMember { - _member: Py, - name: Py, - value: Py, -} - /// Every registered record spec, in registration order. pub(crate) fn all_specs() -> impl Iterator { RECORDS.iter().flat_map(|group| group.iter()) @@ -235,9 +216,7 @@ fn enum_member_field<'py>( Ok(classes) => classes, Err(error) => return Some(Err(error)), }; - let Some(spec) = classes.get(&(value.get_type().as_ptr() as usize)) else { - return python_enum_member_field(value, wants_value); - }; + let spec: &EnumSpec = classes.get(&(value.get_type().as_ptr() as usize))?; let Ok(member) = value.cast::() else { return None; }; @@ -257,87 +236,6 @@ fn enum_member_field<'py>( Some(Ok(PyString::new(py, answer).into_any())) } -/// `member.value` or `member.name` of a Python enum, read once per member and then answered natively. -fn python_enum_member_field<'py>( - value: &Bound<'py, PyAny>, - wants_value: bool, -) -> Option>> { - let py: Python<'py> = value.py(); - let identity: usize = value.as_ptr() as usize; - if let Some(answer) = cached_python_enum_member(py, identity, wants_value) { - return Some(Ok(answer)); - } - match python_enum_class_check(value) { - Ok(true) => {} - Ok(false) => return None, - Err(error) => return Some(Err(error)), - } - let member: PythonEnumMember = match read_python_enum_member(value) { - Ok(member) => member, - Err(error) => return Some(Err(error)), - }; - let answer: Bound<'py, PyAny> = if wants_value { - member.value.bind(py).clone() - } else { - member.name.bind(py).clone() - }; - let mut members = PYTHON_ENUM_MEMBERS - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let _ = members - .get_or_insert_with(HashMap::new) - .entry(identity) - .or_insert(member); - Some(Ok(answer)) -} - -fn cached_python_enum_member<'py>( - py: Python<'py>, - identity: usize, - wants_value: bool, -) -> Option> { - let members = PYTHON_ENUM_MEMBERS - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let member: &PythonEnumMember = members.as_ref()?.get(&identity)?; - let answer: &Py = if wants_value { - &member.value - } else { - &member.name - }; - Some(answer.bind(py).clone()) -} - -/// Whether `value`'s class derives from `enum.Enum`, classified once per class. -fn python_enum_class_check(value: &Bound<'_, PyAny>) -> PyResult { - let class: Bound<'_, PyType> = value.get_type(); - let identity: usize = class.as_ptr() as usize; - let known: Option = PYTHON_ENUM_CLASSES - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .as_ref() - .and_then(|classes| classes.get(&identity).map(|(_, is_enum)| *is_enum)); - if let Some(is_enum) = known { - return Ok(is_enum); - } - let is_enum: bool = class.is_subclass(&imported(value.py(), ENUM_BASE.0, ENUM_BASE.1)?)?; - let _ = PYTHON_ENUM_CLASSES - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .get_or_insert_with(HashMap::new) - .entry(identity) - .or_insert((class.into_any().unbind(), is_enum)); - Ok(is_enum) -} - -fn read_python_enum_member(value: &Bound<'_, PyAny>) -> PyResult { - Ok(PythonEnumMember { - _member: value.clone().unbind(), - name: value.getattr(ENUM_NAME_ATTRIBUTE)?.unbind(), - value: value.getattr(ENUM_VALUE_ATTRIBUTE)?.unbind(), - }) -} - fn enum_classes(py: Python<'_>) -> PyResult> { let module: Bound<'_, PyModule> = py.import(NATIVE_MODULE)?; let mut classes: HashMap = HashMap::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index c764e1d800..7334eb244e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -64,8 +64,7 @@ fn sql_model_rule_keys( model_rule_keys(py, lint_key, &codes, &models) } -/// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. -/// Hashed in parallel with the GIL released: the inputs are owned Rust strings. +/// Each model path's SQL rule cache key, hashed in parallel with the GIL released. pub(crate) fn model_rule_keys( py: Python<'_>, lint_key: &str, diff --git a/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs index 71d61e1a07..0898810f52 100644 --- a/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod digest; +pub(crate) mod model_rule_key; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/model_rule_key.rs similarity index 86% rename from crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs rename to crates/sqlbuild-rules/src/rule_cache/_helpers/model_rule_key.rs index 99df4d0c9e..ccec4a557f 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/model_rule_key.rs @@ -6,7 +6,11 @@ use crate::rule_cache::models::ModelKeyInput; /// A digest of the lint key, selected codes, authored and compiled SQL and dynamic contract. #[must_use] -pub fn model_rule_key(lint_key: &str, codes: &[String], model: &ModelKeyInput<'_>) -> String { +pub(crate) fn model_rule_key( + lint_key: &str, + codes: &[String], + model: &ModelKeyInput<'_>, +) -> String { let codes: String = codes.join("\0"); let contract: &str = model.contract.unwrap_or(NO_CONTRACT); let has_contract: &str = if model.contract.is_some() { "1" } else { "0" }; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs index 3fd43bd89e..3a70ebde30 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs @@ -1,7 +1,6 @@ pub mod file_rule_key; pub mod lint_key; pub mod lookup_rule_cache; -pub mod model_rule_key; pub mod model_rule_keys; pub mod parse_rule_cache; pub mod rule_cache_text; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs index f31271f9de..91093063ad 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs +++ b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs @@ -2,7 +2,7 @@ use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use crate::rule_cache::main::model_rule_key::model_rule_key; +use crate::rule_cache::_helpers::model_rule_key::model_rule_key; use crate::rule_cache::models::ModelKeyInput; /// Each model's `model_rule_key`, in input order. diff --git a/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs b/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs index ce98fd39c9..4d2c72069a 100644 --- a/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs +++ b/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs @@ -1,4 +1,4 @@ -use crate::rule_cache::main::model_rule_key::model_rule_key; +use crate::rule_cache::_helpers::model_rule_key::model_rule_key; use crate::rule_cache::main::model_rule_keys::model_rule_keys; use crate::rule_cache::models::ModelKeyInput; use crate::rule_cache::tests::test_types::ModelRuleKeysTestCase; @@ -10,6 +10,7 @@ fn given_models_when_hashing_rule_keys_together_then_each_key_matches_its_single ModelRuleKeysTestCase { description: "no models give no keys", models: vec![], + expected_key_count: 0, }, ModelRuleKeysTestCase { description: "many models keep their order and single-model keys", @@ -21,6 +22,7 @@ fn given_models_when_hashing_rule_keys_together_then_each_key_matches_its_single (sql, sql, contract) }) .collect(), + expected_key_count: 64, }, ]; let codes: Vec = vec!["SQBRSQL001".to_owned(), "SQBRSQL033".to_owned()]; @@ -40,11 +42,14 @@ fn given_models_when_hashing_rule_keys_together_then_each_key_matches_its_single .map(|model| model_rule_key("lint-key", &codes, model)) .collect(); + let keys: Vec = model_rule_keys("lint-key", &codes, &inputs); + assert_eq!( - model_rule_keys("lint-key", &codes, &inputs), - expected, + keys.len(), + test_case.expected_key_count, "{}", test_case.description ); + assert_eq!(keys, expected, "{}", test_case.description); } } diff --git a/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs b/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs index 9abdde79a7..e267ac3f3d 100644 --- a/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs @@ -1,4 +1,5 @@ pub(crate) struct ModelRuleKeysTestCase { pub(crate) description: &'static str, pub(crate) models: Vec<(&'static str, &'static str, Option<&'static str>)>, + pub(crate) expected_key_count: usize, } diff --git a/fensu.toml b/fensu.toml index 220546fd6c..b25b4a3735 100644 --- a/fensu.toml +++ b/fensu.toml @@ -625,7 +625,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs=3", From 2a96d74171b4cf9321d6e4cf9443452dc4d75fa1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 11:12:15 +0100 Subject: [PATCH 04/28] perf: build header contract records positionally and skip per-model enum and factory lookups Header columns, audit instances and measurement thresholds are built with positional arguments in their declared field order, verified once against the registered record specs, instead of a per-object keyword dict. The model loop keeps header columns and audits as vectors rather than round-tripping through temporary tuples, derives direct audit identities from the parsed nodes, skips enum column resolution when no enums are visible, and lists audit factories only for headers that reference them. --- .../compiled_project/model_loop_contracts.rs | 10 +- .../compiled_project/model_loop_header.rs | 34 ++-- .../_helpers/model_config/header_metadata.rs | 185 +++++++++++++----- 3 files changed, 166 insertions(+), 63 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs index b82fe8f7bf..b2a45bfde3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs @@ -115,7 +115,7 @@ pub(crate) fn factory_audits<'py>( } /// `audit.name or audit.definition_name`. -pub(crate) fn audit_identity(audit: &Bound<'_, PyAny>) -> PyResult { +fn audit_identity(audit: &Bound<'_, PyAny>) -> PyResult { let name: Bound<'_, PyAny> = native_field(audit, NAME_FIELD)?; if name.is_truthy()? { return name.extract(); @@ -171,6 +171,14 @@ pub(crate) fn enum_columns<'py>( ) -> PyResult> { let py: Python<'py> = values.py(); let (enums, (declarations, consumer)) = visible; + if enums.is_empty() { + return Ok(EnumColumns { + columns: PyTuple::new(py, columns.specs)?, + facts: columns.facts, + declarations: PyDict::new(py), + usages: Vec::new(), + }); + } let enforced: bool = match values.get_item(CONTRACT_KEY)? { Some(value) => value.eq(&context.classes.contract_enforced)?, None => false, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index 75fbcd9740..d9db4b41f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, compile_error, config_error, }; use crate::bindings::_helpers::compiled_project::model_loop_contracts::{ - EnumColumns, audit_identity, dynamic_family_objects, enum_columns, factory_audits, + EnumColumns, dynamic_family_objects, enum_columns, factory_audits, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; use crate::bindings::_helpers::compiled_project::model_loop_schema::{ @@ -27,7 +27,9 @@ use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::header_metadata::{audit_tuple, column_tuple}; +use crate::bindings::_helpers::model_config::header_metadata::{ + audit_instances, parsed_audit_identity, schema_columns, +}; use crate::bindings::_helpers::records::fields::field as native_field; const DESCRIPTION_KEY: &str = "description"; @@ -103,9 +105,7 @@ pub(crate) fn header_schema<'py>( let locations: Bound<'py, PyDict> = source.model_file.header_locations(py)?; let model_path: &Bound<'py, PyAny> = source.model_file.paths(py).1; let local_columns: DeclaredColumns<'py> = DeclaredColumns { - specs: column_tuple(py, context.contracts, &parsed_columns, &locations)? - .iter() - .collect(), + specs: schema_columns(py, context.contracts, &parsed_columns, &locations)?, facts: local_column_facts(&parsed_columns)?, }; let columns: DeclaredColumns<'py> = match source.schema { @@ -121,14 +121,20 @@ pub(crate) fn header_schema<'py>( let parsed_audits = metadata .audits .map_err(|error| raised_config_error(py, error))?; - let available: Vec = context.classes.audit_factories.keys().extract()?; - let references: Vec = audit_factory_references(&node(factories), model, &available) - .map_err(|error| raised_config_error(py, error))?; - let direct_audits: Bound<'py, PyTuple> = audit_tuple(py, context.contracts, &parsed_audits)?; + let references: Vec = match factories { + Some(factories) => { + let available: Vec = context.classes.audit_factories.keys().extract()?; + audit_factory_references(&PyNode(factories), model, &available) + .map_err(|error| raised_config_error(py, error))? + } + None => Vec::new(), + }; + let mut direct_audits: Vec> = + audit_instances(py, context.contracts, &parsed_audits)?; let (generated, generated_identities) = factory_audits(py, context, &references)?; - let direct_identities: Vec = direct_audits + let direct_identities: Vec = parsed_audits .iter() - .map(|audit| audit_identity(&audit)) + .map(parsed_audit_identity) .collect::>()?; merged_audit_identities(&direct_identities, &generated_identities, model) .map_err(|error| raised_config_error(py, error))?; @@ -150,10 +156,8 @@ pub(crate) fn header_schema<'py>( .iter() .any(|column| column.data_type.is_some()); let family_facts: Vec = families.iter().map(family_fact).collect(); - let audits: Bound<'py, PyTuple> = PyTuple::new( - py, - direct_audits.iter().chain(generated).collect::>(), - )?; + direct_audits.extend(generated); + let audits: Bound<'py, PyTuple> = PyTuple::new(py, direct_audits)?; let families: Vec> = dynamic_family_objects(py, context, &families)?; let enums: Bound<'py, PyDict> = native_field(source.visible.0, ENUMS_FIELD)?.downcast_into()?; let enums: Bound<'py, PyDict> = match source.schema { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index 2d91542cb0..bcc3dcbbdc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -1,5 +1,8 @@ //! Parsed MODEL header columns and audits built into the Python contract objects. +use std::sync::OnceLock; + +use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{FromPyObject, IntoPyObject}; @@ -8,6 +11,44 @@ use sqlbuild_model_config::header_metadata::models::{ }; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::records::fields::all_specs; + +/// `SchemaColumn` fields in declaration order, the order its objects are built positionally in. +const SCHEMA_COLUMN_FIELDS: [&str; 8] = [ + "name", + "type", + "nullable", + "description", + "meta", + "audits", + "location", + "migrate_from", +]; +/// `SchemaAuditInstance` fields in declaration order. +const AUDIT_INSTANCE_FIELDS: [&str; 11] = [ + "definition_name", + "arguments", + "name", + "description", + "severity", + "run_scope", + "always_run", + "thresholds", + "minimum_samples", + "evidence_limit", + "location", +]; +/// `MeasurementThresholds` fields in declaration order. +const THRESHOLDS_FIELDS: [&str; 2] = ["warn", "error"]; +/// `MeasurementThresholdBound` fields in declaration order. +const THRESHOLD_BOUND_FIELDS: [&str; 4] = ["operator", "limit", "lower", "upper"]; +/// Each positionally built record with the field order its values are given in. +const POSITIONAL_RECORDS: [(&str, &[&str]); 4] = [ + ("SchemaColumn", &SCHEMA_COLUMN_FIELDS), + ("SchemaAuditInstance", &AUDIT_INSTANCE_FIELDS), + ("MeasurementThresholds", &THRESHOLDS_FIELDS), + ("MeasurementThresholdBound", &THRESHOLD_BOUND_FIELDS), +]; /// The Python classes the parsed metadata becomes, read from a Python mapping. #[derive(FromPyObject)] @@ -29,11 +70,21 @@ pub(crate) fn column_tuple<'py>( parsed: &[ParsedColumn>], locations: &Bound<'py, PyDict>, ) -> PyResult> { + PyTuple::new(py, schema_columns(py, classes, parsed, locations)?) +} + +/// The parsed columns as `SchemaColumn` objects, in declaration order. +pub(crate) fn schema_columns<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + parsed: &[ParsedColumn>], + locations: &Bound<'py, PyDict>, +) -> PyResult>> { let mut columns: Vec> = Vec::with_capacity(parsed.len()); for column in parsed { columns.push(schema_column(py, classes, column, locations)?); } - PyTuple::new(py, columns) + Ok(columns) } pub(crate) fn audit_tuple<'py>( @@ -41,24 +92,73 @@ pub(crate) fn audit_tuple<'py>( classes: &ContractClasses<'py>, parsed: &[ParsedAudit>], ) -> PyResult> { + PyTuple::new(py, audit_instances(py, classes, parsed)?) +} + +/// The parsed audits as `SchemaAuditInstance` objects without a location, in order. +pub(crate) fn audit_instances<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + parsed: &[ParsedAudit>], +) -> PyResult>> { let no_location: Bound<'py, PyAny> = py.None().into_bound(py); let mut audits: Vec> = Vec::with_capacity(parsed.len()); for audit in parsed { audits.push(audit_instance(py, classes, audit, &no_location)?); } - PyTuple::new(py, audits) + Ok(audits) +} + +/// `audit.name or audit.definition_name` of the instance `audit` builds. +pub(crate) fn parsed_audit_identity(audit: &ParsedAudit>) -> PyResult { + if let Some(name) = &audit.name + && name.0.is_truthy()? + { + return name.0.extract(); + } + audit.definition_name.0.extract() } -/// `class(**fields)`. -fn contract_object<'py>( +/// `class(*values)`: the same record as keywords bind, given in `POSITIONAL_RECORDS` order. +fn contract_object<'py, const N: usize>( class: &Bound<'py, PyAny>, - fields: &[(&str, Bound<'py, PyAny>)], + values: [Bound<'py, PyAny>; N], ) -> PyResult> { - let keywords: Bound<'py, PyDict> = PyDict::new(class.py()); - for (name, value) in fields { - keywords.set_item(name, value)?; + checked_field_orders()?; + class.call1(PyTuple::new(class.py(), values)?) +} + +/// Fail when a positionally built record no longer declares its fields in the expected order. +fn checked_field_orders() -> PyResult<()> { + static MISMATCH: OnceLock> = OnceLock::new(); + match MISMATCH.get_or_init(mismatched_record) { + Some(record) => Err(PyRuntimeError::new_err(format!( + "native record {record} does not declare the fields header metadata builds it with" + ))), + None => Ok(()), + } +} + +/// The first positionally built record whose registered fields differ from its expected order. +fn mismatched_record() -> Option<&'static str> { + for (record, fields) in POSITIONAL_RECORDS { + let mut declared: bool = false; + for spec in all_specs() { + if spec.name == record { + declared = spec.fields.len() == fields.len() + && spec + .fields + .iter() + .zip(fields) + .all(|(field, expected)| field.name == *expected); + break; + } + } + if !declared { + return Some(record); + } } - class.call((), Some(&keywords)) + None } fn optional<'py>(py: Python<'py>, value: Option<&PyNode<'py>>) -> Bound<'py, PyAny> { @@ -81,15 +181,15 @@ fn schema_column<'py>( .collect::>>()?; contract_object( &classes.schema_column, - &[ - ("name", column.name.0.clone()), - ("type", optional(py, column.column_type.as_ref())), - ("nullable", optional(py, column.nullable.as_ref())), - ("description", optional(py, column.description.as_ref())), - ("meta", PyDict::new(py).into_any()), - ("audits", PyTuple::new(py, audits)?.into_any()), - ("location", location), - ("migrate_from", optional(py, column.migrate_from.as_ref())), + [ + column.name.0.clone(), + optional(py, column.column_type.as_ref()), + optional(py, column.nullable.as_ref()), + optional(py, column.description.as_ref()), + PyDict::new(py).into_any(), + PyTuple::new(py, audits)?.into_any(), + location, + optional(py, column.migrate_from.as_ref()), ], ) } @@ -117,27 +217,18 @@ fn audit_instance<'py>( ); contract_object( &classes.schema_audit_instance, - &[ - ("definition_name", audit.definition_name.0.clone()), - ("arguments", arguments.into_any()), - ("name", optional(py, audit.name.as_ref())), - ("description", optional(py, audit.description.as_ref())), - ("severity", severity), - ("run_scope", optional(py, audit.run_scope.as_ref())), - ("always_run", always_run), - ( - "thresholds", - thresholds(py, classes, audit.thresholds.as_ref())?, - ), - ( - "minimum_samples", - optional(py, audit.minimum_samples.as_ref()), - ), - ( - "evidence_limit", - optional(py, audit.evidence_limit.as_ref()), - ), - ("location", location.clone()), + [ + audit.definition_name.0.clone(), + arguments.into_any(), + optional(py, audit.name.as_ref()), + optional(py, audit.description.as_ref()), + severity, + optional(py, audit.run_scope.as_ref()), + always_run, + thresholds(py, classes, audit.thresholds.as_ref())?, + optional(py, audit.minimum_samples.as_ref()), + optional(py, audit.evidence_limit.as_ref()), + location.clone(), ], ) } @@ -153,9 +244,9 @@ pub(crate) fn thresholds<'py>( }; contract_object( &classes.measurement_thresholds, - &[ - ("warn", threshold_bound(py, classes, parsed.warn)?), - ("error", threshold_bound(py, classes, parsed.error)?), + [ + threshold_bound(py, classes, parsed.warn)?, + threshold_bound(py, classes, parsed.error)?, ], ) } @@ -179,11 +270,11 @@ fn threshold_bound<'py>( .unwrap_or_else(|| py.None().into_bound(py)); contract_object( &classes.measurement_threshold_bound, - &[ - ("operator", operator), - ("limit", limit.into_pyobject(py)?.into_any()), - ("lower", lower.into_pyobject(py)?.into_any()), - ("upper", upper.into_pyobject(py)?.into_any()), + [ + operator, + limit.into_pyobject(py)?.into_any(), + lower.into_pyobject(py)?.into_any(), + upper.into_pyobject(py)?.into_any(), ], ) } From 8e5a396fb0649c8fb0c65f5339af3244dfdb6b6b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 11:08:33 +0100 Subject: [PATCH 05/28] perf: skip built-in rule evaluation when no rules are selected A run selecting no rules now validates its rules configuration through a model-free native request on the calling thread instead of spawning the native rules thread, rebuilding the scope index and every model's rules facts, and replaying the project's models to count unevaluated ones. --- .../src/bindings/_helpers/rules/rows.rs | 78 +++++++++++++---- .../bindings/_helpers/rules/rule_dispatch.rs | 84 ++++++++++++++----- .../src/bindings/_helpers/rules/rule_run.rs | 6 +- .../sqlbuild/cli/commands/main/_test_types.py | 10 +++ .../sqlbuild/cli/commands/main/test_rules.py | 70 ++++++++++++++++ 5 files changed, 209 insertions(+), 39 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 7ab29e953d..1dbdcf35cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -110,26 +110,76 @@ pub(crate) fn build_rules_request( as_rules_error(py, built_request(&request)) } -fn built_request(options: &Request<'_, '_>) -> PyResult { - let py: Python<'_> = options.py(); - let project: Bound<'_, PyAny> = options.get("project")?; - let config: Bound<'_, PyAny> = options.get("config")?; - let project_dir: Bound<'_, PyAny> = options.get("project_dir")?; - let dialect: Bound<'_, PyAny> = options.get("dialect")?; +/// The model-free request of a run selecting no rules, which only validates its configuration. +pub(crate) fn selectionless_rules_request( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + dialect: &str, + custom_payloads: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = config.py(); + let header: PyResult = dialect + .into_bound_py_any(py) + .and_then(|dialect| request_header(config, project_dir, &dialect, (None, custom_payloads))); + as_rules_error( + py, + header.and_then(|header| { + compiler_guard(|| { + let request: EvaluateRequest = + serde_json::from_value(header).map_err(value_error)?; + Ok(NativeRulesRequest { + request: Mutex::new(Some(RowsRequest { + request, + memo: None, + })), + }) + }) + }), + ) +} + +/// The request's version, paths, dialect, config, custom rules and, when given, scope index. +fn request_header( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, + (project, custom_payloads): (Option<&Bound<'_, PyAny>>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = config.py(); let resolved: Bound<'_, PyAny> = project_dir.call_method0("resolve")?; let resolved_path: PathBuf = resolved.extract()?; - let header: Value = json!({ + let mut header: Value = json!({ "version": API_VERSION, "project_dir": resolved.str()?.to_str()?, "dialect": dialect.extract::()?, - "config": config_value(&config)?, - "scope_index": plain_value( - metadata_projection(py, &project_attribute(&project, "scope_index")?)?.bind(py), - )?, - "defer_suppressions": true, - "custom_rules": plain_value(&options.get("custom_payloads")?)?, - "rules_cache_path": path_text(py, rules_bulk_cache_path(&resolved_path, NATIVE_RULES_CACHE_FILE))?, + "config": config_value(config)?, }); + if let Some(project) = project { + header["scope_index"] = plain_value( + metadata_projection(py, &project_attribute(project, "scope_index")?)?.bind(py), + )?; + } + header["defer_suppressions"] = Value::Bool(true); + header["custom_rules"] = plain_value(custom_payloads)?; + header["rules_cache_path"] = Value::String(path_text( + py, + rules_bulk_cache_path(&resolved_path, NATIVE_RULES_CACHE_FILE), + )?); + Ok(header) +} + +fn built_request(options: &Request<'_, '_>) -> PyResult { + let py: Python<'_> = options.py(); + let project: Bound<'_, PyAny> = options.get("project")?; + let config: Bound<'_, PyAny> = options.get("config")?; + let project_dir: Bound<'_, PyAny> = options.get("project_dir")?; + let dialect: Bound<'_, PyAny> = options.get("dialect")?; + let header: Value = request_header( + &config, + &project_dir, + &dialect, + (Some(&project), &options.get("custom_payloads")?), + )?; let (public_enums, public_constants) = declaration_rows( PyTuple::new( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index aa7b557440..c1a4cf807c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -36,7 +36,7 @@ use crate::bindings::_helpers::rules::custom_hosts::HostCancel; use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules; use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; use crate::bindings::_helpers::rules::rows::{ - NativeRulesRequest, build_rules_request, evaluate_rules_request, + NativeRulesRequest, build_rules_request, evaluate_rules_request, selectionless_rules_request, }; use crate::bindings::_helpers::rules::rule_plan::plan_rule_run; use crate::bindings::_helpers::rules::rule_run::finish_rule_run; @@ -268,27 +268,32 @@ pub(crate) fn evaluate_rules<'py>( ]; let native_dialect: String = run.dialect.clone(); let native_cancel: HostCancel = run.cancel.clone(); - let native_thread: PhaseThread = PhaseThread::spawn(RULES_THREAD, move || { - Python::attach(|py| { - let [project, config, project_dir, catalogue, payloads] = native_run; - let run: NativeRun<'_> = NativeRun { - project: project.into_bound(py), - config: config.into_bound(py), - project_dir: project_dir.into_bound(py), - catalogue: catalogue.into_bound(py), - dialect: native_dialect, - verify_determinism: false, - initial_findings: None, - cancel: native_cancel, - }; - evaluate_native( - &run, - (true, Some(payloads.into_bound(py))), - CustomSource::Started(custom), - ) - .map(Bound::unbind) - }) - })?; + let selectionless: bool = custom.is_none() && field(&plan, "selected")?.len()? == 0; + let native_thread: Option = if selectionless { + None + } else { + Some(PhaseThread::spawn(RULES_THREAD, move || { + Python::attach(|py| { + let [project, config, project_dir, catalogue, payloads] = native_run; + let run: NativeRun<'_> = NativeRun { + project: project.into_bound(py), + config: config.into_bound(py), + project_dir: project_dir.into_bound(py), + catalogue: catalogue.into_bound(py), + dialect: native_dialect, + verify_determinism: false, + initial_findings: None, + cancel: native_cancel, + }; + evaluate_native( + &run, + (true, Some(payloads.into_bound(py))), + CustomSource::Started(custom), + ) + .map(Bound::unbind) + }) + })?) + }; let sql_result: PyResult> = run_planned_sql_rules(Some(&keywords( py, &[ @@ -300,7 +305,10 @@ pub(crate) fn evaluate_rules<'py>( ], )?)); cancel_on_interrupt(py, &sql_result, &run.cancel); - let result: PyResult> = joined(py, native_thread, &run.cancel); + let result: PyResult> = match native_thread { + Some(native_thread) => joined(py, native_thread, &run.cancel), + None => selectionless_result(&run, &payloads), + }; let sql_result: Bound<'py, PyAny> = sql_result?; let result: Bound<'py, PyAny> = result?; finish_rule_run(&plan, &sql_result, &result) @@ -333,6 +341,36 @@ fn evaluate_native<'py>( Some(handle) => joined(py, handle, &run.cancel), None => outcome(py, (Vec::new(), 0, 0), (0, 0)), }; + rules_result(run, defer_suppressions, evaluated, custom) +} + +/// The built-in rules result of a run that selects no rules, without building its models. +fn selectionless_result<'py>( + run: &NativeRun<'py>, + payloads: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = run.project.py(); + let request: Bound<'py, NativeRulesRequest> = Bound::new( + py, + selectionless_rules_request(&run.config, &run.project_dir, &run.dialect, payloads)?, + )?; + let evaluated: PyResult> = evaluate_rules_request(py, &request); + rules_result( + run, + true, + evaluated, + outcome(py, (Vec::new(), 0, 0), (0, 0)), + ) +} + +/// Combine one native evaluation with its custom-rule outcome into a `RulesResult`. +fn rules_result<'py>( + run: &NativeRun<'py>, + defer_suppressions: bool, + evaluated: PyResult>, + custom: PyResult>, +) -> PyResult> { + let py: Python<'py> = run.project.py(); let native_result: Bound<'py, PyAny> = evaluated?; let custom: Bound<'py, PyAny> = custom?; let combined: Vec> = field(&native_result, "findings")? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index 5f53eaf5ca..56f766520f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -88,8 +88,10 @@ pub(crate) fn finish_rule_run<'py>( .map(|finding| finding.path.clone()) .collect(); let mut unevaluated_models: i64 = 0; - for model in project_models(&record_field(plan, "project")?)? { - unevaluated_models += i64::from(unevaluated.contains(&model.posix)); + if !unevaluated.is_empty() { + for model in project_models(&record_field(plan, "project")?)? { + unevaluated_models += i64::from(unevaluated.contains(&model.posix)); + } } let findings: Vec> = grouped .into_iter() diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 0b688f3b54..22a1024976 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -164,6 +164,16 @@ class RulesIntegrationTestCase: expected_code: str +@dataclass(frozen=True) +class EmptyRuleSelectionTestCase: + """A compile that selects no rules, with the Rules configuration it still validates.""" + + description: str + configuration: str + expected_exit_code: int + expected_messages: tuple[str, ...] + + @dataclass(frozen=True) class CustomHostSplitIntegrationTestCase: """A compile whose custom rules run on several hosts and must match one host.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d0d62ffc72..2ea7a60274 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -29,6 +29,7 @@ CustomRuleCacheSequenceTestCase, DynamicPivotRulesIntegrationTestCase, EarlyLintInputsTestCase, + EmptyRuleSelectionTestCase, ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, NumericRangeDecisionIntegrationTestCase, @@ -2845,6 +2846,75 @@ def required_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]: assert payload["diagnostics"] == [] +@pytest.mark.parametrize( + "test_case", + [ + EmptyRuleSelectionTestCase( + "valid exception for an unevaluated rule compiles", + "[[rules.rule_exceptions]]\n" + 'rule = "SQBRCONTRACT101"\npath = "models/orders.sql"\nreason = "Legacy."\n', + 0, + (), + ), + EmptyRuleSelectionTestCase( + "unknown exception rule is rejected", + '[[rules.rule_exceptions]]\nrule = "SQBRNOPE999"\npath = "models/orders.sql"\n' + 'reason = "Legacy."\n', + 1, + ("rule exceptions target unknown rules: SQBRNOPE999",), + ), + EmptyRuleSelectionTestCase( + "missing exception path is rejected", + '[[rules.rule_exceptions]]\nrule = "SQBRCONTRACT101"\npath = "models/missing.sql"\n' + 'reason = "Legacy."\n', + 1, + ("rule exception path does not exist: models/missing.sql",), + ), + EmptyRuleSelectionTestCase( + "ignore selector matching no rule is rejected", + '[[rules.rule_ignores]]\nrules = ["SQBRZZZ"]\npaths = ["models/orders.sql"]\n' + 'reason = "Legacy."\n', + 1, + ("rule-ignore selector matches no rules: SQBRZZZ",), + ), + EmptyRuleSelectionTestCase( + "unknown built-in rule option is rejected", + "[rules.rule_options.SQBRSQL004]\nnope = 3\n", + 1, + ("rule SQBRSQL004 has unknown options: nope",), + ), + ], + ids=lambda case: case.description, +) +def test_given_empty_rule_selection_when_compiling_then_only_configuration_is_validated( + test_case: EmptyRuleSelectionTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = []\n\n{test_case.configuration}', + encoding="utf-8", + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + timings: dict[str, int] = payload.get("compile_timings", {}) + + assert exit_code == test_case.expected_exit_code + assert [diagnostic["message"] for diagnostic in payload["diagnostics"]] == list( + test_case.expected_messages + ) + assert not rules_bulk_cache_path( + project_dir=tmp_path, file_name="native-response.json" + ).exists() + assert (timings.get("rule_cache_hits", 0), timings.get("rule_cache_misses", 0)) == (0, 0) + + @pytest.mark.parametrize( "test_case", [ From 9f83773e117332c728d7edb33b11e74d1b2ef0b1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 11:29:25 +0100 Subject: [PATCH 06/28] perf: hand the rules request its scope projection without a Python round trip The built-in rules request converted the native scope metadata projection into Python dictionaries only to read them straight back into JSON values. Convert the projection to the same sorted JSON value natively instead. --- .../src/bindings/_helpers/rules/rows.rs | 7 ++- .../_helpers/scopes/scope_report_objects.rs | 43 +++++++++++++++++++ .../bindings/_helpers/scopes/scope_reports.rs | 18 +++++--- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 1dbdcf35cd..613df85f12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -42,7 +42,7 @@ use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_m use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; -use crate::bindings::_helpers::scopes::scope_reports::metadata_projection; +use crate::bindings::_helpers::scopes::scope_reports::metadata_projection_value; use crate::bindings::_helpers::type_system::equality::types_equal; use crate::bindings::types::CompilerDetach; @@ -155,9 +155,8 @@ fn request_header( "config": config_value(config)?, }); if let Some(project) = project { - header["scope_index"] = plain_value( - metadata_projection(py, &project_attribute(project, "scope_index")?)?.bind(py), - )?; + header["scope_index"] = + metadata_projection_value(&project_attribute(project, "scope_index")?)?; } header["defer_suppressions"] = Value::Bool(true); header["custom_rules"] = plain_value(custom_payloads)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs index 68014b23f0..6f50a0d376 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs @@ -24,6 +24,8 @@ use crate::bindings::_helpers::scopes::scope_objects::{ type PyObject = Py; const PATHLIB_MODULE: &str = "pathlib"; +/// The error `plain_value` raises for an integer outside 64 bits. +const WIDE_INTEGER: &str = "Integer exceeds 64-bit range"; fn texts(item: &Bound<'_, PyAny>, name: &str) -> PyResult> { native_field(item, name)? @@ -130,6 +132,47 @@ pub(crate) fn json_to_python(py: Python<'_>, value: JsonValue, tuples: bool) -> } } +/// `plain_value(int(integer))`: a 64-bit signed or unsigned number, else the wide-integer error. +fn plain_integer(integer: &JsonInteger) -> PyResult { + let Some(number) = integer.to_i128() else { + return Err(PyValueError::new_err(WIDE_INTEGER)); + }; + if let Ok(signed) = i64::try_from(number) { + return Ok(serde_json::Value::from(signed)); + } + match u64::try_from(number) { + Ok(unsigned) => Ok(serde_json::Value::from(unsigned)), + Err(_) => Err(PyValueError::new_err(WIDE_INTEGER)), + } +} + +/// `plain_value(json_to_python(value, False))` computed without building the Python value. +pub(crate) fn json_to_plain(value: JsonValue) -> PyResult { + Ok(match value { + JsonValue::Null => serde_json::Value::Null, + JsonValue::Bool(flag) => serde_json::Value::Bool(flag), + JsonValue::Integer(integer) => plain_integer(&integer)?, + JsonValue::Float(number) => serde_json::Number::from_f64(number) + .map_or(serde_json::Value::Null, serde_json::Value::Number), + JsonValue::String(text) => serde_json::Value::String(text), + JsonValue::Utf16(units) => serde_json::Value::String(String::from_utf16_lossy(&units)), + JsonValue::Array(items) => serde_json::Value::Array( + items + .into_iter() + .map(json_to_plain) + .collect::>()?, + ), + JsonValue::Object(entries) => { + let mut object: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for (key, item) in entries { + let _replaced = object.insert(key, json_to_plain(item)?); + } + serde_json::Value::Object(object.into_iter().collect()) + } + }) +} + /// `_value` of one metadata value; an unsupported type is the error Python's serializer raises. pub(crate) fn python_to_json(value: &Bound<'_, PyAny>) -> PyResult { if value.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index 94fdc75d68..9583d93bff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -32,9 +32,9 @@ use crate::bindings::_helpers::scopes::scope_objects::{ read_declaration, read_index, }; use crate::bindings::_helpers::scopes::scope_report_objects::{ - browse_object, diagnostics_object, explanation_object, json_to_python, list_object, - move_object, read_declaration_target, read_filters, read_optional_target, read_result, - read_target, report_object, + browse_object, diagnostics_object, explanation_object, json_to_plain, json_to_python, + list_object, move_object, read_declaration_target, read_filters, read_optional_target, + read_result, read_target, report_object, }; type PyObject = Py; @@ -318,9 +318,17 @@ pub(crate) fn serialize_scope_report( #[pyfunction] #[pyo3(name = "scope_metadata_projection", signature = (*, index))] pub(crate) fn metadata_projection(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult { + json_to_python(py, projection_json(index)?, false) +} + +/// `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. +pub(crate) fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult { + json_to_plain(projection_json(index)?) +} + +fn projection_json(index: &Bound<'_, PyAny>) -> PyResult { let index: ProjectionIndex = projection_index(read_index(index)?); - let projection: JsonValue = scope_metadata_projection(&index); - json_to_python(py, projection, false) + Ok(scope_metadata_projection(&index)) } fn optional<'py>( From 57f01f7925f0ccdcdef6ab98d33f2b31a34d497a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:15:10 +0100 Subject: [PATCH 07/28] perf: index SQL rule models by path instead of rescanning every earlier model --- .../src/bindings/_helpers/rules/sql_rules.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 6a3c3f27c3..d3124c629b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -1,5 +1,6 @@ //! Run the planned SQL rules through native lint, reusing cached and early-lint results. +use std::collections::hash_map::Entry; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::Instant; @@ -213,14 +214,17 @@ fn sql_rule_models<'py>( if !field(&project_attribute(project, "settings")?, "sql_analysis")?.is_truthy()? { return Ok(models); } + let mut positions: HashMap = HashMap::new(); for model in project_attribute(project, "models")?.try_iter()? { let Some(entry) = sql_rule_model(model?, keyed)? else { continue; }; - if let Some(index) = models.iter().position(|known| known.path == entry.path) { - models[index] = entry; - } else { - models.push(entry); + match positions.entry(entry.path.clone()) { + Entry::Occupied(known) => models[*known.get()] = entry, + Entry::Vacant(slot) => { + slot.insert(models.len()); + models.push(entry); + } } } Ok(models) From c23049c3647eb3d623e897a96e475f0cbeccbbb4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:15:10 +0100 Subject: [PATCH 08/28] perf: keep discovered model file paths native so lint inputs skip pathlib round trips --- .../_helpers/discovery/model_file_record.rs | 17 ++++++++++++- .../_helpers/discovery/project_files.rs | 25 +++++++++++-------- .../bindings/_helpers/rules/lint_inputs.rs | 11 ++++---- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index 8106673ec7..fe4758982d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -1,5 +1,7 @@ //! A discovered SQL model file as a native record whose Python attributes are built on demand. +use std::path::PathBuf; + use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::ids::RecordId; use pyo3::exceptions::PyTypeError; @@ -89,6 +91,8 @@ impl Locations { #[pyclass(module = "sqlbuild._native", name = "DiscoveredSqlModelFile", frozen)] pub(crate) struct NativeModelFile { file_path: Py, + /// `file_path` as a native path, read once so hot paths need not call back into `pathlib`. + native_file_path: Option, relative_path: Py, contents: Py, header_values: Py, @@ -133,9 +137,10 @@ struct ModelFileFields { } impl NativeModelFile { - pub(crate) fn discovered(parts: ModelFileParts) -> Self { + pub(crate) fn discovered(py: Python<'_>, parts: ModelFileParts) -> Self { let (enum_declarations, constant_declarations) = parts.declarations; Self { + native_file_path: parts.file_path.bind(py).extract().ok(), file_path: parts.file_path, relative_path: parts.relative_path, contents: parts.contents, @@ -167,6 +172,14 @@ impl NativeModelFile { ) } + /// The file path as a native path, reading `file_path` only when it was not a path at construction. + pub(crate) fn file_path_buf(&self, py: Python<'_>) -> PyResult { + match &self.native_file_path { + Some(path) => Ok(path.clone()), + None => self.file_path.bind(py).extract(), + } + } + pub(crate) fn paths<'py>(&self, py: Python<'py>) -> (&Bound<'py, PyAny>, &Bound<'py, PyAny>) { (self.file_path.bind(py), self.relative_path.bind(py)) } @@ -218,6 +231,7 @@ impl NativeModelFile { } let fields: ModelFileFields = fields.extract()?; Ok(Self { + native_file_path: fields.file_path.bind(py).extract().ok(), file_path: fields.file_path, relative_path: fields.relative_path, contents: fields.contents, @@ -243,6 +257,7 @@ impl NativeModelFile { query_sql: Py, ) -> Self { Self { + native_file_path: self.native_file_path.clone(), file_path: self.file_path.clone_ref(py), relative_path: self.relative_path.clone_ref(py), contents, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index 2042228633..c339645445 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -76,17 +76,20 @@ pub(crate) fn model_record<'py>( Err(error) => return Err(error), }; let (enum_declarations, constant_declarations) = declarations; - let record: NativeModelFile = NativeModelFile::discovered(ModelFileParts { - file_path: file_path.unbind(), - relative_path: relative_path.unbind(), - contents: PyString::new(py, &model.contents).unbind(), - header_values: header_values.unbind(), - header_column_locations: model.header_column_locations, - output_column_locations: model.output_column_locations, - query_sql: PyString::new(py, &model.query_sql).unbind(), - declarations: (enum_declarations.unbind(), constant_declarations.unbind()), - extract_implicit_alias_columns, - }); + let record: NativeModelFile = NativeModelFile::discovered( + py, + ModelFileParts { + file_path: file_path.unbind(), + relative_path: relative_path.unbind(), + contents: PyString::new(py, &model.contents).unbind(), + header_values: header_values.unbind(), + header_column_locations: model.header_column_locations, + output_column_locations: model.output_column_locations, + query_sql: PyString::new(py, &model.query_sql).unbind(), + declarations: (enum_declarations.unbind(), constant_declarations.unbind()), + extract_implicit_alias_columns, + }, + ); Ok(Ok(record.into_py_any(py)?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 0252f0f7e6..fb309d237b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -10,6 +10,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ ExpansionRecord, ModelInputRecord, NativeModelInput, model_relation_key_row, }; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, @@ -34,11 +35,11 @@ pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - record - .model_file - .bind(py) - .getattr(FILE_PATH_ATTRIBUTE)? - .extract() + let model_file: &Bound<'_, PyAny> = record.model_file.bind(py); + match model_file.downcast::() { + Ok(native) => native.get().file_path_buf(py), + Err(_) => model_file.getattr(FILE_PATH_ATTRIBUTE)?.extract(), + } } /// The record's shared texts with the given passes, read only when their file is linted. From 2981340e18b19301627b473e09e2e2572df81d12 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:25:03 +0100 Subject: [PATCH 09/28] refactor: read model file paths strictly at construction --- .../_helpers/discovery/model_file_record.rs | 21 ++++++++----------- .../_helpers/discovery/project_files.rs | 2 +- .../bindings/_helpers/rules/lint_inputs.rs | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index fe4758982d..9bf2858958 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -92,7 +92,7 @@ impl Locations { pub(crate) struct NativeModelFile { file_path: Py, /// `file_path` as a native path, read once so hot paths need not call back into `pathlib`. - native_file_path: Option, + native_file_path: PathBuf, relative_path: Py, contents: Py, header_values: Py, @@ -137,10 +137,10 @@ struct ModelFileFields { } impl NativeModelFile { - pub(crate) fn discovered(py: Python<'_>, parts: ModelFileParts) -> Self { + pub(crate) fn discovered(py: Python<'_>, parts: ModelFileParts) -> PyResult { let (enum_declarations, constant_declarations) = parts.declarations; - Self { - native_file_path: parts.file_path.bind(py).extract().ok(), + Ok(Self { + native_file_path: parts.file_path.bind(py).extract()?, file_path: parts.file_path, relative_path: parts.relative_path, contents: parts.contents, @@ -151,7 +151,7 @@ impl NativeModelFile { enum_declarations, constant_declarations, extract_implicit_alias_columns: parts.extract_implicit_alias_columns, - } + }) } pub(crate) fn header_locations<'py>(&self, py: Python<'py>) -> PyResult> { @@ -172,12 +172,9 @@ impl NativeModelFile { ) } - /// The file path as a native path, reading `file_path` only when it was not a path at construction. - pub(crate) fn file_path_buf(&self, py: Python<'_>) -> PyResult { - match &self.native_file_path { - Some(path) => Ok(path.clone()), - None => self.file_path.bind(py).extract(), - } + /// The file path as a native path. + pub(crate) fn file_path_buf(&self) -> PathBuf { + self.native_file_path.clone() } pub(crate) fn paths<'py>(&self, py: Python<'py>) -> (&Bound<'py, PyAny>, &Bound<'py, PyAny>) { @@ -231,7 +228,7 @@ impl NativeModelFile { } let fields: ModelFileFields = fields.extract()?; Ok(Self { - native_file_path: fields.file_path.bind(py).extract().ok(), + native_file_path: fields.file_path.bind(py).extract()?, file_path: fields.file_path, relative_path: fields.relative_path, contents: fields.contents, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index c339645445..ab49541661 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -89,7 +89,7 @@ pub(crate) fn model_record<'py>( declarations: (enum_declarations.unbind(), constant_declarations.unbind()), extract_implicit_alias_columns, }, - ); + )?; Ok(Ok(record.into_py_any(py)?)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index fb309d237b..e66134584b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -37,7 +37,7 @@ pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { let model_file: &Bound<'_, PyAny> = record.model_file.bind(py); match model_file.downcast::() { - Ok(native) => native.get().file_path_buf(py), + Ok(native) => Ok(native.get().file_path_buf()), Err(_) => model_file.getattr(FILE_PATH_ATTRIBUTE)?.extract(), } } From 0a68f0e940b52400b3eb0fabc63fa1e1a42dec4e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:23:21 +0100 Subject: [PATCH 10/28] perf: retain native header columns until public projection --- .../_helpers/compiled_project/audit_loop.rs | 11 +- .../compiled_project/compiled_model_view.rs | 16 ++- .../compiled_project/model_input_view.rs | 7 +- .../compiled_project/model_loop_contracts.rs | 46 +++---- .../compiled_project/model_loop_header.rs | 15 ++- .../compiled_project/model_loop_host.rs | 2 +- .../compiled_project/model_loop_schema.rs | 68 +++++----- .../_helpers/compiled_project/project.rs | 2 +- .../_helpers/model_config/header_metadata.rs | 123 +++++++++++++++++- .../_helpers/semantic_checks/metadata.rs | 3 +- fensu.toml | 16 +-- .../compiler/compiled_project/_test_types.py | 10 ++ .../test_native_compiled_project.py | 48 ++++++- 13 files changed, 279 insertions(+), 88 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs index d02cd2cdc6..78eecf3de5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, HashMap}; +use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use pyo3::FromPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; @@ -671,10 +672,7 @@ fn header_text(header: &Bound<'_, PyDict>, key: &str) -> PyResult pub(crate) fn model_owner<'py>( name: String, path: String, - schema: ( - &Bound<'py, PyTuple>, - (&Bound<'py, PyTuple>, &[SchemaColumnFact]), - ), + schema: (&Bound<'py, PyTuple>, (&HeaderColumns, &[SchemaColumnFact])), ) -> PyResult> { let (audits, (columns, facts)) = schema; let mut instances: Vec<(String, Option, Bound<'py, PyAny>)> = Vec::new(); @@ -686,7 +684,10 @@ pub(crate) fn model_owner<'py>( continue; } let column_owner: &str = fact.location_path.as_deref().unwrap_or(&path); - for audit in native_field(&columns.get_item(index)?, "audits")?.try_iter()? { + for audit in columns.specs[index] + .field(audits.py(), "audits")? + .try_iter()? + { let audit: Bound<'py, PyAny> = audit?; let owner_file: String = location_path(&audit)?.unwrap_or_else(|| column_owner.to_owned()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index bfece36153..8a60cd1169 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -1,5 +1,7 @@ //! A compiled model, read from the native compiled project that owns it. +use std::sync::Arc; + use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; @@ -22,6 +24,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::graph::project_graph::ResourceRow; +use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::relation_keys::RelationKeyRow; @@ -52,7 +55,7 @@ pub(crate) struct SqlRuleKey { } /// A schema's entry audits, column specs and each audited column's index and name. -pub(crate) type AuditedSchema = (Py, Py, Vec<(usize, String)>); +pub(crate) type AuditedSchema = (Py, Arc, Vec<(usize, String)>); /// `(name, type, nullability)` of one inferred output column. pub(crate) type InferredRow = (String, Option, String); @@ -355,10 +358,11 @@ impl NativeCompiledModel { /// The attached schema's declared `SchemaColumn`s, or `None` without a schema. pub(crate) fn declared_columns(&self, py: Python<'_>) -> PyResult>> { self.read(py, |record, py| { - Ok(record + record .schema .as_ref() - .map(|schema| schema.columns.clone_ref(py))) + .map(|schema| schema.columns.project(py).map(Bound::unbind)) + .transpose() }) } @@ -381,7 +385,7 @@ impl NativeCompiledModel { .collect(); Ok(Some(( schema.audits.clone_ref(py), - schema.columns.clone_ref(py), + schema.columns.clone(), audited, ))) }) @@ -639,14 +643,14 @@ impl NativeCompiledModel { /// Inferred columns, else the attached schema's declared columns, else zero. #[getter] pub(crate) fn output_column_count(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, py| { + self.read(py, |record, _py| { if let Some(rows) = &compiled_of(record)?.analysis.inferred { return Ok(rows.len()); } Ok(record .schema .as_ref() - .map_or(0, |schema| schema.columns.bind(py).len())) + .map_or(0, |schema| schema.columns.specs.len())) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index 0de3a2a3d4..a7ddf57ddf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -1,5 +1,7 @@ //! A model's compile input, read from the native compiled project that owns it. +use std::sync::Arc; + use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; @@ -12,6 +14,7 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; +use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::SpanRow; use crate::bindings::_helpers::rules::relation_keys::{RelationKeyRow, relation_key_row}; @@ -33,7 +36,7 @@ pub(crate) struct InputViewClasses<'py> { /// A model's attached header schema; its specs carry user-authored audit arguments. pub(crate) struct SchemaRecord { pub(crate) description: Option, - pub(crate) columns: Py, + pub(crate) columns: Arc, pub(crate) audits: Py, pub(crate) dynamic_columns: Py, pub(crate) type_enforcement: bool, @@ -417,7 +420,7 @@ pub(crate) fn schema_entry( fields.set_item("model_schema", &schema.model_schema)?; fields.set_item("description", &schema.description)?; fields.set_item("type_enforcement", schema.type_enforcement.then_some(true))?; - fields.set_item("columns", schema.columns.bind(py))?; + fields.set_item("columns", schema.columns.project(py)?)?; fields.set_item("dynamic_columns", schema.dynamic_columns.bind(py))?; fields.set_item("audits", schema.audits.bind(py))?; Ok(classes.schema_model_entry.call((), Some(&fields))?.unbind()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs index b2a45bfde3..843076350c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs @@ -1,5 +1,7 @@ //! Header contract objects the model loop builds: factory audits, dynamic families, enum columns. +use std::sync::Arc; + use pyo3::BoundObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -8,6 +10,7 @@ use sqlbuild_model_config::header_metadata::models::DynamicFamily; use crate::bindings::_helpers::compiled_project::model_loop::LoopContext; use crate::bindings::_helpers::compiled_project::model_loop_schema::DeclaredColumns; +use crate::bindings::_helpers::model_config::header_metadata::{HeaderColumn, HeaderColumns}; use crate::bindings::_helpers::records::fields::field as native_field; const CASES_ATTRIBUTE: &str = "cases"; @@ -44,15 +47,6 @@ const COPIED_CASE_FIELDS: [&str; 8] = [ "minimum_samples", "evidence_limit", ]; -/// `SchemaColumn` fields `dataclasses.replace` carries over unchanged. -const KEPT_COLUMN_FIELDS: [&str; 6] = [ - "name", - "nullable", - "description", - "meta", - "location", - "migrate_from", -]; const PIVOT_COLUMN_FIELD: &str = "pivot_column"; const VALUE_COLUMN_FIELD: &str = "value_column"; const AGGREGATE_FIELD: &str = "aggregate"; @@ -60,7 +54,7 @@ const NAME_PATTERN_FIELD: &str = "name_pattern"; /// A model's enum-typed columns resolved to scalar types, and the usages they generate. pub(crate) struct EnumColumns<'py> { - pub(crate) columns: Bound<'py, PyTuple>, + pub(crate) columns: Arc, pub(crate) facts: Vec, /// Column name to its enum declaration, as `CompileModelInput.enum_columns` holds them. pub(crate) declarations: Bound<'py, PyDict>, @@ -167,13 +161,13 @@ pub(crate) fn enum_columns<'py>( (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ), values: &Bound<'py, PyDict>, - columns: DeclaredColumns<'py>, + columns: DeclaredColumns, ) -> PyResult> { let py: Python<'py> = values.py(); let (enums, (declarations, consumer)) = visible; if enums.is_empty() { return Ok(EnumColumns { - columns: PyTuple::new(py, columns.specs)?, + columns: Arc::new(HeaderColumns::new(columns.specs)), facts: columns.facts, declarations: PyDict::new(py), usages: Vec::new(), @@ -184,7 +178,7 @@ pub(crate) fn enum_columns<'py>( None => false, }; let resolved: Bound<'py, PyDict> = PyDict::new(py); - let mut resolved_columns: Vec> = Vec::with_capacity(columns.specs.len()); + let mut resolved_columns: Vec = Vec::with_capacity(columns.specs.len()); let mut facts: Vec = Vec::with_capacity(columns.facts.len()); for (column, mut fact) in columns.specs.into_iter().zip(columns.facts) { let key: &str = fact.data_type.as_deref().unwrap_or_default(); @@ -194,7 +188,8 @@ pub(crate) fn enum_columns<'py>( continue; }; resolved.set_item(&fact.name, &declaration)?; - let mut audits: Vec> = native_field(&column, AUDITS_FIELD)? + let mut audits: Vec> = column + .field(py, AUDITS_FIELD)? .try_iter()? .collect::>()?; if enforced { @@ -214,19 +209,26 @@ pub(crate) fn enum_columns<'py>( } let scalar_type: Bound<'py, PyAny> = native_field(&declaration, SCALAR_TYPE_ATTRIBUTE)?; fact.data_type = scalar_type.extract()?; - let fields: Bound<'py, PyDict> = PyDict::new(py); - for field in KEPT_COLUMN_FIELDS { - fields.set_item(field, native_field(&column, field)?)?; - } - fields.set_item(TYPE_FIELD, scalar_type)?; - fields.set_item(AUDITS_FIELD, PyTuple::new(py, audits)?)?; - resolved_columns.push(context.contracts.schema_column.call((), Some(&fields))?); + resolved_columns.push(HeaderColumn::replaced( + &context.contracts.schema_column, + [ + column.field(py, "name")?, + scalar_type, + column.field(py, "nullable")?, + column.field(py, "description")?, + column.field(py, "meta")?, + PyTuple::new(py, audits)?.into_any(), + column.field(py, "location")?, + column.field(py, "migrate_from")?, + ], + [0, 2, 3, 4, 6, 7, 1, 5], + )?); facts.push(fact); } let usages: Vec> = generated_usages(context, declarations, consumer, &resolved)?; Ok(EnumColumns { - columns: PyTuple::new(py, resolved_columns)?, + columns: Arc::new(HeaderColumns::new(resolved_columns)), facts, declarations: resolved, usages, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index d9db4b41f1..95f14eb56b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -1,5 +1,7 @@ //! A model's MODEL header attached as its schema entry, by the native header parser. +use std::sync::Arc; + use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods}; use sqlbuild_analysis::compiled_project::models::{ @@ -28,7 +30,7 @@ use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::header_metadata::{ - audit_instances, parsed_audit_identity, schema_columns, + HeaderColumns, audit_instances, native_schema_column, parsed_audit_identity, }; use crate::bindings::_helpers::records::fields::field as native_field; @@ -59,7 +61,7 @@ pub(crate) struct AttachedHeader<'py> { /// A header schema entry's Python contract objects, enum columns and retained fact. pub(crate) struct HeaderSchema<'py> { - pub(crate) columns: Bound<'py, PyTuple>, + pub(crate) columns: Arc, pub(crate) audits: Bound<'py, PyTuple>, pub(crate) dynamic_columns: Bound<'py, PyTuple>, pub(crate) type_enforcement: bool, @@ -104,11 +106,14 @@ pub(crate) fn header_schema<'py>( .map_err(|error| raised_config_error(py, error))?; let locations: Bound<'py, PyDict> = source.model_file.header_locations(py)?; let model_path: &Bound<'py, PyAny> = source.model_file.paths(py).1; - let local_columns: DeclaredColumns<'py> = DeclaredColumns { - specs: schema_columns(py, context.contracts, &parsed_columns, &locations)?, + let local_columns: DeclaredColumns = DeclaredColumns { + specs: parsed_columns + .iter() + .map(|column| native_schema_column(py, context.contracts, column, &locations)) + .collect::>()?, facts: local_column_facts(&parsed_columns)?, }; - let columns: DeclaredColumns<'py> = match source.schema { + let columns: DeclaredColumns = match source.schema { Some(schema) => merged_columns( context, schema, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 478af8e829..1f01666301 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -218,7 +218,7 @@ pub(crate) fn finished_record<'py>( }; let schema: Option = finished.header.schema.as_ref().map(|schema| SchemaRecord { description: finished.header.description.clone(), - columns: schema.columns.clone().unbind(), + columns: schema.columns.clone(), audits: schema.audits.clone().unbind(), dynamic_columns: schema.dynamic_columns.clone().unbind(), type_enforcement: schema.type_enforcement, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs index 0bcfd48851..8931d52454 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs @@ -1,5 +1,6 @@ //! A model's `model_schema`: resolved, checked for visibility, and merged into its header columns. +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; @@ -7,6 +8,7 @@ use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compile_error}; use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fact; +use crate::bindings::_helpers::model_config::header_metadata::HeaderColumn; use crate::bindings::_helpers::records::fields::field as native_field; const MODEL_SCHEMA_KEY: &str = "model_schema"; @@ -15,21 +17,11 @@ const TYPE_FIELD: &str = "type"; const NULLABLE_FIELD: &str = "nullable"; const DESCRIPTION_FIELD: &str = "description"; const AUDITS_FIELD: &str = "audits"; -const MIGRATE_FROM_FIELD: &str = "migrate_from"; const LOCATION_FIELD: &str = "location"; const PATH_FIELD: &str = "path"; const LINE_FIELD: &str = "line"; const COLUMNS_FIELD: &str = "columns"; const STRIP_METHOD: &str = "strip"; -/// `SchemaColumn` fields a named-schema column keeps when a header column augments it. -const KEPT_NAMED_FIELDS: [&str; 6] = [ - "name", - "type", - "nullable", - "description", - "meta", - "location", -]; /// The reusable schema a model names: its columns, name, description and enum column types. pub(crate) struct ModelSchema<'py> { @@ -101,8 +93,8 @@ pub(crate) fn schema_enums<'py>( } /// Schema columns as Python specs paired with their native facts, in declaration order. -pub(crate) struct DeclaredColumns<'py> { - pub(crate) specs: Vec>, +pub(crate) struct DeclaredColumns { + pub(crate) specs: Vec, pub(crate) facts: Vec, } @@ -110,11 +102,11 @@ pub(crate) struct DeclaredColumns<'py> { pub(crate) fn merged_columns<'py>( context: &LoopContext<'_, 'py>, schema: &ModelSchema<'py>, - local: DeclaredColumns<'py>, + local: DeclaredColumns, model: (&str, &str, &Bound<'py, PyAny>), -) -> PyResult> { +) -> PyResult { let (model_name, relative_path, model_path) = model; - let mut merged: DeclaredColumns<'py> = DeclaredColumns { + let mut merged: DeclaredColumns = DeclaredColumns { specs: Vec::new(), facts: Vec::new(), }; @@ -123,14 +115,14 @@ pub(crate) fn merged_columns<'py>( merged .facts .push(python_column_fact(&column, Some(model_path))?); - merged.specs.push(column); + merged.specs.push(HeaderColumn::from_python(column)?); } let keys: Vec = merged .facts .iter() .map(|fact| fact.name.to_lowercase()) .collect(); - let mut additional: DeclaredColumns<'py> = DeclaredColumns { + let mut additional: DeclaredColumns = DeclaredColumns { specs: Vec::new(), facts: Vec::new(), }; @@ -162,15 +154,17 @@ pub(crate) fn merged_columns<'py>( fn augmented<'py>( context: &LoopContext<'_, 'py>, - (named, named_fact): (&Bound<'py, PyAny>, &SchemaColumnFact), - (local, local_fact): (&Bound<'py, PyAny>, SchemaColumnFact), -) -> PyResult<(Bound<'py, PyAny>, SchemaColumnFact)> { - let py: Python<'py> = named.py(); - let mut audits: Vec> = native_field(named, AUDITS_FIELD)? + (named, named_fact): (&HeaderColumn, &SchemaColumnFact), + (local, local_fact): (&HeaderColumn, SchemaColumnFact), +) -> PyResult<(HeaderColumn, SchemaColumnFact)> { + let py: Python<'py> = context.contracts.schema_column.py(); + let mut audits: Vec> = named + .field(py, AUDITS_FIELD)? .try_iter()? .collect::>()?; let mut fact: SchemaColumnFact = named_fact.clone(); - for (audit, definition) in native_field(local, AUDITS_FIELD)? + for (audit, definition) in local + .field(py, AUDITS_FIELD)? .try_iter()? .zip(local_fact.audit_definitions) { @@ -187,15 +181,23 @@ fn augmented<'py>( fact.audit_definitions.push(definition); } } - let fields: Bound<'py, PyDict> = PyDict::new(py); - for field in KEPT_NAMED_FIELDS { - fields.set_item(field, native_field(named, field)?)?; - } - fields.set_item(AUDITS_FIELD, PyTuple::new(py, audits)?)?; - fields.set_item(MIGRATE_FROM_FIELD, &local_fact.migrate_from)?; + let migrate_from = local_fact.migrate_from.clone().into_bound_py_any(py)?; fact.migrate_from = local_fact.migrate_from; Ok(( - context.contracts.schema_column.call((), Some(&fields))?, + HeaderColumn::replaced( + &context.contracts.schema_column, + [ + named.field(py, NAME_FIELD)?, + named.field(py, TYPE_FIELD)?, + named.field(py, NULLABLE_FIELD)?, + named.field(py, DESCRIPTION_FIELD)?, + named.field(py, "meta")?, + PyTuple::new(py, audits)?.into_any(), + named.field(py, LOCATION_FIELD)?, + migrate_from, + ], + [0, 1, 2, 3, 4, 6, 5, 7], + )?, fact, )) } @@ -204,7 +206,7 @@ fn augmented<'py>( fn check_augmentation( context: &LoopContext<'_, '_>, local: &SchemaColumnFact, - named: &Bound<'_, PyAny>, + named: &HeaderColumn, model: (&str, &str), ) -> PyResult<()> { let (model_name, relative_path) = model; @@ -224,7 +226,7 @@ fn check_augmentation( "model '{model_name}' in {relative_path} cannot override {} for named-schema column \ '{name}' from {}; only audit augmentation is supported", overridden.join(", "), - origin(named)? + origin(&named.project(context.contracts.schema_column.py())?)? ); return Err(compile_error(context.classes, &message, None)); } @@ -233,7 +235,7 @@ fn check_augmentation( "model '{model_name}' in {relative_path} redeclares named-schema column '{name}' from \ {} without audits or migrate_from; only audit augmentation and column \ migrations are supported", - origin(named)? + origin(&named.project(context.contracts.schema_column.py())?)? ); return Err(compile_error(context.classes, &message, None)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index c0422ad9ca..ff757ea087 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -167,7 +167,7 @@ impl NativeCompiledProject { .to_string(), ( schema.audits.bind(py), - (schema.columns.bind(py), &fact.columns), + (schema.columns.as_ref(), &fact.columns), ), )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index bcc3dcbbdc..2940dc1a98 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -3,7 +3,8 @@ use std::sync::OnceLock; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{FromPyObject, IntoPyObject}; use sqlbuild_model_config::header_metadata::models::{ @@ -11,7 +12,116 @@ use sqlbuild_model_config::header_metadata::models::{ }; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::all_specs; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::RecordId; + +/// Native column fields; a Python record is constructed only for public consumers. +pub(crate) struct HeaderColumn { + values: Option<[Py; 8]>, + class: Py, + projection: PyOnceLock>, +} + +impl HeaderColumn { + pub(crate) fn replaced<'py>( + class: &Bound<'py, PyAny>, + values: [Bound<'py, PyAny>; 8], + keyword_order: [usize; 8], + ) -> PyResult { + if class.is(&record_class(class.py(), RecordId::SchemaColumn)?) { + return Self::new(class, values); + } + let fields = PyDict::new(class.py()); + for index in keyword_order { + fields.set_item(SCHEMA_COLUMN_FIELDS[index], &values[index])?; + } + Self::from_python(class.call((), Some(&fields))?) + } + + pub(crate) fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let Some(values) = &self.values else { + return field(&self.project(py)?, name); + }; + let index = SCHEMA_COLUMN_FIELDS + .iter() + .position(|field| *field == name) + .ok_or_else(|| PyRuntimeError::new_err("unknown column field"))?; + Ok(values[index].bind(py).clone()) + } + + pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.projection + .get_or_try_init(py, || { + let values = self + .values + .as_ref() + .ok_or_else(|| PyRuntimeError::new_err("missing column projection"))?; + contract_object( + self.class.bind(py), + values.each_ref().map(|value| value.bind(py).clone()), + ) + .map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } + + pub(crate) fn from_python(value: Bound<'_, PyAny>) -> PyResult { + let py = value.py(); + let projection = PyOnceLock::new(); + let class = value.get_type().into_any().unbind(); + let _ = projection.set(py, value.unbind()); + Ok(Self { + values: None, + class, + projection, + }) + } + + pub(crate) fn new<'py>( + class: &Bound<'py, PyAny>, + values: [Bound<'py, PyAny>; 8], + ) -> PyResult { + if !class.is(&record_class(class.py(), RecordId::SchemaColumn)?) { + return Self::from_python(contract_object(class, values)?); + } + checked_field_orders()?; + Ok(Self { + values: Some(values.map(Bound::unbind)), + class: class.clone().unbind(), + projection: PyOnceLock::new(), + }) + } +} + +/// Retained native columns and their shared, lazy public tuple. +pub(crate) struct HeaderColumns { + pub(crate) specs: Vec, + projection: PyOnceLock>, +} + +impl HeaderColumns { + pub(crate) fn new(specs: Vec) -> Self { + Self { + specs, + projection: PyOnceLock::new(), + } + } + + pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.projection + .get_or_try_init(py, || { + let columns = self + .specs + .iter() + .map(|column| column.project(py)) + .collect::>>()?; + PyTuple::new(py, columns).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } +} /// `SchemaColumn` fields in declaration order, the order its objects are built positionally in. const SCHEMA_COLUMN_FIELDS: [&str; 8] = [ @@ -171,6 +281,15 @@ fn schema_column<'py>( column: &ParsedColumn>, locations: &Bound<'py, PyDict>, ) -> PyResult> { + native_schema_column(py, classes, column, locations)?.project(py) +} + +pub(crate) fn native_schema_column<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + column: &ParsedColumn>, + locations: &Bound<'py, PyDict>, +) -> PyResult { let location = locations .get_item(&column.name.0)? .unwrap_or_else(|| py.None().into_bound(py)); @@ -179,7 +298,7 @@ fn schema_column<'py>( .iter() .map(|audit| audit_instance(py, classes, audit, &location)) .collect::>>()?; - contract_object( + HeaderColumn::new( &classes.schema_column, [ column.name.0.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 02aeae8369..51db6c40a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -480,8 +480,7 @@ fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { attached.push((None, audit)); } for (index, name) in audited { - let column: Bound<'_, PyAny> = columns.bind(py).get_item(index)?; - for audit in native_field(&column, "audits")?.try_iter()? { + for audit in columns.specs[index].field(py, "audits")?.try_iter()? { attached.push((Some(name.clone()), audit?)); } } diff --git a/fensu.toml b/fensu.toml index b25b4a3735..1b60010ba2 100644 --- a/fensu.toml +++ b/fensu.toml @@ -612,7 +612,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=6", @@ -621,7 +621,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_named.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs=1", @@ -810,7 +810,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", @@ -1145,17 +1145,17 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=25", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs=27", @@ -1230,7 +1230,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 4055ecad6e..681d5d2618 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -5,6 +5,16 @@ from dataclasses import dataclass, field +@dataclass(frozen=True) +class HeaderProjectionTestCase: + """Native header columns projected after real compilation and audit attachment.""" + + description: str + files: dict[str, str] + model_name: str + expected_columns: tuple[str, ...] + + @dataclass(frozen=True) class RetainedRulesRowsTestCase: """A real compile whose rules request reads model facts the compile-input stage retained.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 0dd0ca946b..7239f6f3ad 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -13,14 +13,17 @@ from sqlbuild._native import ( InferredColumn, InvalidQualifiedIdentityError, + SchemaColumn, + SchemaModelEntry, build_compiled_project, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + HeaderProjectionTestCase, ImmediateSyntaxErrorTestCase, ModelLoopErrorTestCase, ModelLoopParityTestCase, @@ -54,6 +57,49 @@ from sqlbuild._native import CompileInputsView +@pytest.mark.parametrize( + "test_case", + [ + HeaderProjectionTestCase( + "enum and factory audits", HEADER_CONTRACT_FILES, "order_statuses", ("status", "total") + ) + ], + ids=lambda case: case.description, +) +def test_given_header_contracts_when_compiling_then_columns_project_only_on_public_access( + test_case: HeaderProjectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_files(project_dir=tmp_path, files=test_case.files) + constructed: list[SchemaColumn] = [] + + def counted(self: SchemaColumn, *args: Any, **kwargs: Any) -> None: + # Native records initialize their fields in __new__. + object.__init__(self) + constructed.append(self) + + monkeypatch.setattr(SchemaColumn, "__init__", counted) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert constructed == [] + models: dict[str, CompiledModelView] = {model.name: model for model in project.models} + model: CompiledModelView = models[test_case.model_name] + entry: SchemaModelEntry | None = model.schema_entry + assert entry is not None + columns: tuple[SchemaColumn, ...] = entry.columns + assert tuple(column.name for column in columns) == test_case.expected_columns + assert columns[0].type == "VARCHAR" + assert [audit.definition_name for audit in columns[0].audits] == ["not_null", "accepted_values"] + assert columns[0].audits[1].arguments == {"values": ("PLACED", "SHIPPED")} + assert entry.audits[0].name == "positive_total" + assert len(constructed) == 2 + assert model.schema_entry is entry + assert model.schema_entry.columns is columns + assert pickle.loads(pickle.dumps(columns)) == columns + + @pytest.mark.parametrize( "test_case", [SharedInferredColumnsTestCase("repeated output schemas share pickle entries", 24, 3)], From 9c037cce88e6ecd7a1131e4ca08c245a71ab505a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:34:48 +0100 Subject: [PATCH 11/28] perf: reuse rule parser tokens for dependency classification --- .../src/engine/tests/test_evaluation.rs | 100 ++++++++++++++++++ .../src/engine/tests/test_types.rs | 7 ++ .../src/rules/_helpers/evaluation.rs | 57 +++++++--- 3 files changed, 150 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs index 959acb40da..7e0025814d 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs @@ -7,6 +7,106 @@ use crate::engine::tests::helpers; use crate::engine::tests::test_types; use crate::rules::_helpers::evaluation::normalize_rules_sql; +#[test] +fn given_dependency_tokens_when_classifying_imports_then_preserves_dialect_and_original_sql() +-> Result<(), String> { + let test_cases = [ + test_types::DependencyTokenTestCase { + description: "comments and escaped literals retain canonical arguments", + expected_columns: &[1], + dialect: "duckdb", + query_sql: "WITH imported AS (SELECT * FROM __ReF /* __ref('ignored') */ ('Order''s')) SELECT '__source(''ignored'')' AS label FROM imported", + expected_messages: &[ + "each __ref/__source must appear once in its own top-level import CTE", + ], + }, + test_types::DependencyTokenTestCase { + description: "quoted function names are not dependency calls", + expected_columns: &[], + dialect: "duckdb", + query_sql: "SELECT * FROM \"__ref\"('orders')", + expected_messages: &[], + }, + test_types::DependencyTokenTestCase { + description: "backtick quoting and comment-separated source arguments", + expected_columns: &[], + dialect: "bigquery", + query_sql: "WITH imported AS (SELECT * FROM __source /* ignored */ (`Orders`)) SELECT order_id FROM imported", + expected_messages: &[], + }, + test_types::DependencyTokenTestCase { + description: "dollar strings do not contribute spurious calls", + expected_columns: &[], + dialect: "postgres", + query_sql: "WITH imported AS (SELECT * FROM __ref('orders')) SELECT $$__ref('ignored')$$ AS label FROM imported", + expected_messages: &[], + }, + test_types::DependencyTokenTestCase { + description: "generic fallback retains original dialect dependencies", + expected_columns: &[], + dialect: "postgres", + query_sql: "WITH imported AS (SELECT * FROM __ref('orders')) SELECT payload:order_id FROM imported", + expected_messages: &[], + }, + test_types::DependencyTokenTestCase { + description: "normalization must not erase dependency calls", + expected_columns: &[1], + dialect: "snowflake", + query_sql: "SELECT * EXCLUDE (__ref('orders')) FROM orders", + expected_messages: &[ + "each __ref/__source must appear once in its own top-level import CTE", + ], + }, + test_types::DependencyTokenTestCase { + description: "duplicate dependencies remain ordered findings", + expected_columns: &[1, 55], + dialect: "duckdb", + query_sql: "WITH first_import AS (SELECT * FROM __ref('orders')), second_import AS (SELECT * FROM __REF('orders') WHERE order_id > 0) SELECT order_id FROM first_import", + expected_messages: &[ + "each __ref/__source must appear once in its own top-level import CTE", + "import CTE \"second_import\" contains transformation logic", + ], + }, + ]; + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let config = json!({"select": ["SQBRMODEL101"], "cache": {"enabled": false}}); + for test_case in test_cases { + let mut request: Value = serde_json::from_str(&helpers::request(&project_dir, &config)) + .map_err(|error| error.to_string())?; + request["dialect"] = json!(test_case.dialect); + request["models"][0]["query_sql"] = json!(test_case.query_sql); + // Authored SQL can differ after expansion: dependency classification has + // always used query_sql, while literal decisions use authored_sql. + request["models"][0]["authored_sql"] = json!("SELECT @orders()"); + let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + let faults = result["faults"].as_array().ok_or("missing faults")?; + let messages: Vec<&str> = faults + .iter() + .filter_map(|fault| fault["message"].as_str()) + .collect(); + assert_eq!( + messages, test_case.expected_messages, + "{}: {result}", + test_case.description + ); + assert!( + faults.iter().all(|fault| fault["unevaluated"] != true), + "{}: {result}", + test_case.description + ); + for (fault, expected_column) in faults.iter().zip(test_case.expected_columns) { + assert_eq!(fault["line"], 1, "{}", test_case.description); + assert_eq!( + fault["column"], *expected_column, + "{}", + test_case.description + ); + } + } + Ok(()) +} + #[test] fn given_conflicting_model_identity_when_evaluating_layer_rules_then_reports_independent_findings() -> Result<(), String> { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index c3476b1906..a4e58fe74e 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -252,3 +252,10 @@ pub(crate) struct CachePruningTestCase { pub(crate) description: &'static str, pub(crate) expected_paths: &'static [&'static str], } +pub(crate) struct DependencyTokenTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub query_sql: &'static str, + pub expected_messages: &'static [&'static str], + pub expected_columns: &'static [usize], +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 339a58906f..621c051d3d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -155,8 +155,13 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result) -> Result dependencies, + None => dependency_calls(&model.query_sql, &request.dialect)?, + }; + let classification = classify_model(&query, model, dependencies); let parsed = ParsedModel { query, model, @@ -284,13 +293,21 @@ fn requires_model_query(rule: &RuleMetadata) -> bool { pub(super) fn parse_rule_statements( sql: &str, dialect_name: &str, +) -> Result, ParserError> { + parse_rule_statements_with_dependencies(sql, dialect_name, None) +} + +fn parse_rule_statements_with_dependencies( + sql: &str, + dialect_name: &str, + dependencies: Option<&mut Option>>, ) -> Result, ParserError> { let dialect = rules_dialect(dialect_name); - match parse_with_dialect(sql, dialect.as_ref()) { + match parse_with_dialect(sql, dialect.as_ref(), dependencies) { Ok(statements) => Ok(statements), Err(primary) if !dialect_name.eq_ignore_ascii_case("generic") => { let fallback_sql = normalize_generic_fallback(sql); - parse_with_dialect(&fallback_sql, &GenericDialect {}).map_err(|fallback| { + parse_with_dialect(&fallback_sql, &GenericDialect {}, None).map_err(|fallback| { ParserError::ParserError(format!( "{primary}; generic fallback also failed: {fallback}" )) @@ -331,11 +348,21 @@ fn normalize_generic_fallback(sql: &str) -> String { String::from_utf8(bytes).unwrap_or_else(|_| sql.to_owned()) } -fn parse_with_dialect(sql: &str, dialect: &dyn Dialect) -> Result, ParserError> { +fn parse_with_dialect( + sql: &str, + dialect: &dyn Dialect, + dependencies: Option<&mut Option>>, +) -> Result, ParserError> { + let tokens = Tokenizer::new(dialect, sql).tokenize_with_location()?; + if let Some(dependencies) = dependencies { + *dependencies = Some(dependency_calls_from_tokens( + tokens.iter().map(|token| &token.token), + )); + } let mut parser = Parser::new(dialect) .with_options(ParserOptions::new().with_trailing_commas(true)) .with_recursion_limit(RULES_PARSER_RECURSION_LIMIT) - .try_with_sql(sql)?; + .with_tokens_with_locations(tokens); parser.parse_statements() } @@ -692,9 +719,8 @@ pub(super) fn sole_table_name(query: &Query) -> Option { fn classify_model( query: &Query, model: &Model, - dialect_name: &str, -) -> Result { - let authored_dependencies = dependency_calls(&model.query_sql, dialect_name)?; + authored_dependencies: Vec, +) -> ModelClassification { let mut logical_seen = false; let ctes = top_ctes(query) .iter() @@ -738,11 +764,11 @@ fn classify_model( && root_select(query).is_some_and(|select| plain_select(select, true)) }); - Ok(ModelClassification { + ModelClassification { authored_dependencies, ctes, passthrough, - }) + } } fn dependency_calls(source: &str, dialect_name: &str) -> Result, String> { @@ -750,8 +776,11 @@ fn dependency_calls(source: &str, dialect_name: &str) -> Result, Str let tokens = Tokenizer::new(dialect.as_ref(), source) .tokenize() .map_err(|error| format!("could not tokenize model dependencies: {error}"))?; + Ok(dependency_calls_from_tokens(tokens.iter())) +} + +fn dependency_calls_from_tokens<'a>(tokens: impl Iterator) -> Vec { let tokens = tokens - .iter() .filter(|token| !matches!(token, Token::Whitespace(_))) .collect::>(); let mut calls: Vec = Vec::new(); @@ -794,7 +823,7 @@ fn dependency_calls(source: &str, dialect_name: &str) -> Result, Str } index = cursor; } - Ok(calls) + calls } fn query_dependency_calls(query: &Query) -> Vec { From b14af9f48f94158e65c1af2835985fb0d769f22b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 15:57:05 +0100 Subject: [PATCH 12/28] perf: reduce SQL test planning copies and marker scans --- .../compiler/_helpers/sql_tests/planning.rs | 90 ++++++++++--------- .../_helpers/sql_tests/relation_markers.rs | 54 ++++++----- .../tests/test_sql_test_relation_markers.rs | 12 +++ .../main/test/test_commented_fixtures.py | 32 +++++++ 4 files changed, 118 insertions(+), 70 deletions(-) diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index 12a44c5c3e..f2a63e76f5 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -146,7 +146,7 @@ struct AnalysisResolvedSql { #[derive(Clone)] pub(crate) struct AnalysisTemplate { existing_cte_names: HashSet, - marker_calls: Vec<(String, String)>, + marker_calls: Arc<[(String, String)]>, } type AnalysisTemplateCell = Arc>>; @@ -1341,10 +1341,10 @@ fn analyze_and_resolve_sql( for (name, sql) in request.fixture_ctes { generated_state.insert_helper(name, sql); } - let mut replacements: HashMap<(String, String), String> = HashMap::new(); - for (function_name, referenced_name) in template.marker_calls { + let mut replacements: HashMap<(&str, &str), String> = HashMap::new(); + for (function_name, referenced_name) in template.marker_calls.iter() { let target = if function_name == REF_FUNCTION { - if let Some(chain_sql) = request.resolved_chain.get(&referenced_name) { + if let Some(chain_sql) = request.resolved_chain.get(referenced_name) { for (name, sql) in &chain_sql.generated_ctes { if request.fixtures.scope.is_top_level_helper(name) || name.starts_with(TABLE_FUNCTION_PREFIX) @@ -1364,7 +1364,7 @@ fn analyze_and_resolve_sql( } else { generated_state.mock_target(MockTargetRequest { prefix: REF_PREFIX, - referenced_name: &referenced_name, + referenced_name, mocks: &request.fixtures.mock_refs, fixtures: request.fixtures, file_label: request.file_label, @@ -1373,7 +1373,7 @@ fn analyze_and_resolve_sql( } else if function_name == SOURCE_FUNCTION { generated_state.mock_target(MockTargetRequest { prefix: SOURCE_PREFIX, - referenced_name: &referenced_name, + referenced_name, mocks: &request.fixtures.mock_sources, fixtures: request.fixtures, file_label: request.file_label, @@ -1381,7 +1381,7 @@ fn analyze_and_resolve_sql( } else if function_name == SEED_FUNCTION { generated_state.mock_target(MockTargetRequest { prefix: SEED_PREFIX, - referenced_name: &referenced_name, + referenced_name, mocks: &request.fixtures.mock_seeds, fixtures: request.fixtures, file_label: request.file_label, @@ -1389,7 +1389,7 @@ fn analyze_and_resolve_sql( } else if function_name == DBT_REF_FUNCTION { generated_state.mock_target(MockTargetRequest { prefix: DBT_REF_PREFIX, - referenced_name: &referenced_name, + referenced_name, mocks: &request.fixtures.mock_dbt_refs, fixtures: request.fixtures, file_label: request.file_label, @@ -1397,13 +1397,13 @@ fn analyze_and_resolve_sql( } else if function_name == UDF_FUNCTION || function_name == TABLE_FUNCTION { request .functions - .get(&referenced_name) + .get(referenced_name) .map(|_| referenced_name.clone()) } else { None }; if let Some(target) = target { - replacements.insert((function_name, referenced_name), target); + replacements.insert((function_name.as_str(), referenced_name.as_str()), target); } } let cte_body_sql = strip_statement_terminators( @@ -1437,7 +1437,7 @@ fn analysis_template( let expression = statements.remove(0); Some(AnalysisTemplate { existing_cte_names: top_level_cte_names(&expression), - marker_calls: relation_marker_calls(&expression), + marker_calls: relation_marker_calls(&expression).into(), }) }) }) @@ -1453,11 +1453,13 @@ where { let cached = { let mut cache = templates.lock().map_err(|error| error.to_string())?; - Arc::clone( - cache - .entry(query_sql.to_string()) - .or_insert_with(|| Arc::new(OnceLock::new())), - ) + if let Some(cached) = cache.get(query_sql) { + Arc::clone(cached) + } else { + let cached = Arc::new(OnceLock::new()); + cache.insert(query_sql.to_owned(), Arc::clone(&cached)); + cached + } }; Ok(cached.get_or_init(initialize).clone()) } @@ -1532,7 +1534,7 @@ fn resolve_textual_sql(request: TextualResolutionRequest<'_>) -> Result) -> Result { + sql: std::borrow::Cow<'a, str>, reached: HashSet, ctes: Vec<(String, String)>, } -fn resolve_table_function_fixtures( - sql: &str, +fn resolve_table_function_fixtures<'a>( + sql: &'a str, fixtures: &TestFixtures, patterns: &SqlTestPatterns, -) -> Result { +) -> Result, String> { let mut resolved = TableFunctionFixtures { - sql: sql.to_string(), + sql: std::borrow::Cow::Borrowed(sql), reached: HashSet::new(), ctes: Vec::new(), }; @@ -1603,7 +1605,7 @@ fn resolve_table_function_fixtures( .then(|| (format!("{TABLE_FUNCTION_PREFIX}{name}"), true)) }, )?; - resolved.sql = replaced; + resolved.sql = std::borrow::Cow::Owned(replaced); for (name, body) in &fixtures.mock_table_functions { if !reached.contains(name) { continue; @@ -1662,33 +1664,39 @@ fn top_level_cte_names(expression: &Expression) -> HashSet { with.ctes.iter().map(|cte| cte.alias.name.clone()).collect() } -fn replace_relation_markers( - sql: &str, - replacements: &HashMap<(String, String), String>, +fn replace_relation_markers<'a>( + sql: &'a str, + replacements: &HashMap<(&str, &str), String>, patterns: &SqlTestPatterns, -) -> String { - let mut result = sql.to_string(); +) -> std::borrow::Cow<'a, str> { + let mut result = std::borrow::Cow::Borrowed(sql); for (function_name, pattern) in [ (REF_FUNCTION, &patterns.reference), (SOURCE_FUNCTION, &patterns.source), (SEED_FUNCTION, &patterns.seed), ] { - result = replace_named_markers(&result, pattern, &patterns.lexical, |name| { - replacements - .get(&(function_name.to_string(), name.to_string())) - .cloned() - }); + if !replacements.keys().any(|(name, _)| *name == function_name) { + continue; + } + result = std::borrow::Cow::Owned(replace_named_markers( + &result, + pattern, + &patterns.lexical, + |name| replacements.get(&(function_name, name)).cloned(), + )); + } + if !replacements + .keys() + .any(|(name, _)| *name == DBT_REF_FUNCTION) + { + return result; } - replace_dbt_ref_markers( + std::borrow::Cow::Owned(replace_dbt_ref_markers( &result, &patterns.dbt_reference, &patterns.lexical, - |name| { - replacements - .get(&(DBT_REF_FUNCTION.to_string(), name.to_string())) - .cloned() - }, - ) + |name| replacements.get(&(DBT_REF_FUNCTION, name)).cloned(), + )) } struct UnresolvedReferenceRequest<'a> { diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs index 9c36ee4206..959d41a9fc 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs @@ -9,24 +9,28 @@ pub(crate) type MarkerCall = (String, String); /// Return marker calls in node order, where Polyglot children follow declared field order. pub(crate) fn relation_marker_calls(expression: &Expression) -> Vec { - expression.dfs().flat_map(node_marker_calls).collect() + expression.dfs().fold(Vec::new(), node_marker_calls) } /// FROM then JOIN marker calls of one node, plus the serialized `on` that traversal skips. -fn node_marker_calls(node: &Expression) -> Vec { +fn node_marker_calls(mut calls: Vec, node: &Expression) -> Vec { match node { - Expression::Select(select) => select - .from - .iter() - .flat_map(from_markers) - .chain(join_markers(&select.joins)) - .collect(), - Expression::From(from) => from_markers(from).collect(), - Expression::Delete(delete) => join_markers(&delete.joins).collect(), - Expression::JoinedTable(table) => join_markers(&table.joins).collect(), - Expression::DataDeletionProperty(property) => relation_marker_calls(&property.on), - _ => Vec::new(), + Expression::Select(select) => calls.extend( + select + .from + .iter() + .flat_map(from_markers) + .chain(join_markers(&select.joins)), + ), + Expression::From(from) => calls.extend(from_markers(from)), + Expression::Delete(delete) => calls.extend(join_markers(&delete.joins)), + Expression::JoinedTable(table) => calls.extend(join_markers(&table.joins)), + Expression::DataDeletionProperty(property) => { + return property.on.dfs().fold(calls, node_marker_calls); + } + _ => {} } + calls } fn from_markers(from: &From) -> impl Iterator + '_ { @@ -40,34 +44,26 @@ fn join_markers(joins: &[Join]) -> impl Iterator + '_ { fn marker(expression: &Expression) -> Option { match expression { Expression::Alias(alias) => marker(&alias.this), - Expression::Function(function) => { - let names: Vec> = function - .args - .iter() - .map(|argument| match argument { - Expression::Column(column) => Some(column.name.name.as_str()), - _ => None, - }) - .collect(); - function_marker(&function.name, &names) - } + Expression::Function(function) => function_marker(&function.name, &function.args), _ => None, } } -fn function_marker(name: &str, args: &[Option<&str>]) -> Option { +fn function_marker(name: &str, args: &[Expression]) -> Option { let function_name = name.to_ascii_lowercase(); let referenced_name = if function_name == DBT_REF_FUNCTION { match args { - [Some(first)] => (*first).to_string(), - [Some(first), Some(second)] => format!("{first}__{second}"), + [Expression::Column(first)] => first.name.name.clone(), + [Expression::Column(first), Expression::Column(second)] => { + format!("{}__{}", first.name.name, second.name.name) + } _ => return None, } } else { - let [Some(first)] = args else { + let [Expression::Column(first)] = args else { return None; }; - (*first).to_string() + first.name.name.clone() }; Some((function_name, referenced_name)) } diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs index 9a9c4a061d..f054efe809 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs @@ -6,6 +6,18 @@ use crate::compiler::tests::test_types::{RelationMarkerOracleTestCase, RelationM #[test] fn given_parsed_statements_when_collecting_relation_markers_then_json_walk_order_is_reproduced() { let test_cases = [ + RelationMarkersTestCase { + description: "nested CTEs preserve relation order and ignore escaped marker lookalikes", + dialect: "postgres", + sql: r#"WITH "recent" AS (WITH nested AS (SELECT * FROM __source("orders")) SELECT * FROM nested) SELECT '__ref("ignored")''s' AS note, $$__seed("ignored")$$ AS body FROM recent JOIN __ref("customers") ON TRUE /* __ref("ignored") */"#, + expected_calls: &[("__ref", "customers"), ("__source", "orders")], + }, + RelationMarkersTestCase { + description: "function markers require column arguments even inside nested relations", + dialect: "duckdb", + sql: r#"SELECT __udf("projected") FROM __table_fn("orders") JOIN __udf("lookup") ON TRUE JOIN (SELECT * FROM __ref('literal') JOIN __ref(COALESCE("a", "b")) ON TRUE) t ON TRUE"#, + expected_calls: &[("__table_fn", "orders"), ("__udf", "lookup")], + }, RelationMarkersTestCase { description: "plain reference", dialect: "duckdb", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py index 5b50be7a74..9a71b45d43 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py @@ -86,6 +86,38 @@ @pytest.mark.parametrize( "test_case", [ + CommentedFixtureE2ETestCase( + description="nested model CTEs and marker lookalikes preserve mocked source results", + test_files={ + "macros/status.py": "", + "models/_macros/status.py": _PROJECT_FILES["macros/status.py"], + "models/orders.sql": ( + "MODEL (description 'Orders through nested CTEs.');\n" + 'WITH "recent" AS (\n' + " WITH nested AS (SELECT order_id, status, '__ref(\"ignored\")''s' AS note\n" + ' FROM __source("raw_orders"))\n' + " SELECT order_id, status FROM nested\n" + ")\n" + 'SELECT order_id, @normalize_status("status") AS status\n' + 'FROM "recent" /* __seed("ignored") */\n' + ), + "tests/unit/test_orders.sql": ( + 'TEST (name "nested_orders");\nWITH\n' + + _LINE_COMMENTED_SOURCE + + _LINE_COMMENTED_EXPECTED + ), + "tests/unit/test_order_counts.sql": ( + 'TEST (name "nested_order_counts");\nWITH\n' + + _LINE_COMMENTED_SOURCE + + "__expected__order_counts AS (\n" + " SELECT 'paid' AS status, CAST(1 AS BIGINT) AS order_count\n" + " UNION ALL SELECT 'void' AS status, CAST(1 AS BIGINT) AS order_count\n" + ") SELECT 1\n" + ), + }, + expected_exit_code=0, + expected_output_fragments=("PASS=2", "FAIL=0"), + ), CommentedFixtureE2ETestCase( description="line and block comments around fixture branches pass", test_files={ From 7f767e321c01ba90834b577a8220a558d0c4567f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:16:19 +0100 Subject: [PATCH 13/28] perf: recognize bounded SQL test templates without full ASTs --- .../compiler/_helpers/sql_tests/constants.rs | 2 + .../src/compiler/_helpers/sql_tests/mod.rs | 2 + .../compiler/_helpers/sql_tests/planning.rs | 55 ++-- .../_helpers/sql_tests/simple_template.rs | 254 ++++++++++++++++++ .../sqlbuild-analysis/src/compiler/tests.rs | 4 + .../compiler/tests/simple_template/helpers.rs | 77 ++++++ .../tests/test_sql_test_simple_template.rs | 235 ++++++++++++++++ .../src/compiler/tests/test_types.rs | 17 ++ .../main/test/test_commented_fixtures.py | 25 ++ 9 files changed, 656 insertions(+), 15 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs new file mode 100644 index 0000000000..586f6c747f --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs @@ -0,0 +1,2 @@ +pub(crate) const SIMPLE_TEMPLATE_DIALECT: &str = "duckdb"; +pub(crate) const MAX_TEMPLATE_DEPTH: usize = 32; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs index 7207851e88..eefb8cb50c 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs @@ -2,6 +2,7 @@ pub mod assembly; pub mod authored_macro_calls; pub mod case_identity; pub(crate) mod check_independence; +pub(crate) mod constants; pub mod cte_rename; pub mod cte_slices; pub mod cte_sql; @@ -19,3 +20,4 @@ pub mod reference_calls; pub mod relation_markers; pub mod rendering; pub mod scenario_extraction; +pub(crate) mod simple_template; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index f2a63e76f5..e0d564e389 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -143,10 +143,10 @@ struct AnalysisResolvedSql { reachable_mock_names: HashSet, } -#[derive(Clone)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct AnalysisTemplate { - existing_cte_names: HashSet, - marker_calls: Arc<[(String, String)]>, + pub(crate) existing_cte_names: HashSet, + pub(crate) marker_calls: Arc<[(String, String)]>, } type AnalysisTemplateCell = Arc>>; @@ -1420,29 +1420,54 @@ fn analyze_and_resolve_sql( })) } -fn analysis_template( +#[cfg(test)] +thread_local! { + static TEMPLATE_AST_PARSES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn take_template_ast_parse_count() -> usize { + TEMPLATE_AST_PARSES.with(|count| count.replace(0)) +} + +pub(crate) fn analysis_template( query_sql: &str, dialect_name: &str, templates: &AnalysisTemplateCache, ) -> Result, String> { cached_analysis_template(query_sql, templates, || { Dialect::get_by_name(dialect_name).and_then(|dialect| { - let mut statements = match dialect.parse(query_sql) { - Ok(statements) => statements, - Err(_) => return None, - }; - if statements.len() != 1 { - return None; + if let Some(template) = + super::simple_template::simple_analysis_template(query_sql, dialect_name, &dialect) + { + return Some(template); } - let expression = statements.remove(0); - Some(AnalysisTemplate { - existing_cte_names: top_level_cte_names(&expression), - marker_calls: relation_marker_calls(&expression).into(), - }) + parsed_analysis_template(query_sql, &dialect) }) }) } +/// The full-parser path also serves as the differential oracle for the bounded recognizer. +pub(crate) fn parsed_analysis_template( + query_sql: &str, + dialect: &Dialect, +) -> Option { + #[cfg(test)] + TEMPLATE_AST_PARSES.with(|count| count.set(count.get() + 1)); + let mut statements = match dialect.parse(query_sql) { + Ok(statements) => statements, + Err(_) => return None, + }; + if statements.len() != 1 { + return None; + } + let expression = statements.remove(0); + Some(AnalysisTemplate { + existing_cte_names: top_level_cte_names(&expression), + marker_calls: relation_marker_calls(&expression).into(), + }) +} + pub(crate) fn cached_analysis_template( query_sql: &str, templates: &AnalysisTemplateCache, diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs new file mode 100644 index 0000000000..e47f34c16e --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs @@ -0,0 +1,254 @@ +//! Closed token productions for test-planning facts; unrecognized input uses the full parser. + +use std::collections::HashSet; + +use polyglot_sql::Dialect; +use polyglot_sql::tokens::{Token, TokenType}; + +use crate::compiler::_helpers::sql_tests::constants::{ + MAX_TEMPLATE_DEPTH, SIMPLE_TEMPLATE_DIALECT, +}; +use crate::compiler::_helpers::sql_tests::planning::{AnalysisTemplate, DBT_REF_FUNCTION}; +use crate::compiler::_helpers::sql_tests::relation_markers::MarkerCall; + +/// Recognize the bounded DuckDB subset; every other dialect retains the existing parser. +pub(crate) fn simple_analysis_template( + sql: &str, + dialect_name: &str, + dialect: &Dialect, +) -> Option { + if dialect_name != SIMPLE_TEMPLATE_DIALECT || sql.contains("/*") { + return None; + } + let tokens = match dialect.tokenize(sql) { + Ok(tokens) => tokens, + Err(_) => return None, + }; + let mut reader = TemplateReader { + tokens: &tokens, + position: 0, + }; + let facts = reader.query(0)?; + reader.take(TokenType::Semicolon); + if reader.position != tokens.len() { + return None; + } + Some(AnalysisTemplate { + existing_cte_names: facts.names, + marker_calls: facts.calls.into(), + }) +} + +struct QueryFacts { + names: HashSet, + calls: Vec, +} + +struct TemplateReader<'a> { + tokens: &'a [Token], + position: usize, +} + +impl<'a> TemplateReader<'a> { + fn take(&mut self, kind: TokenType) -> bool { + if self + .tokens + .get(self.position) + .is_some_and(|token| token.token_type == kind) + { + self.position += 1; + true + } else { + false + } + } + + fn require(&mut self, kind: TokenType) -> Option<()> { + self.take(kind).then_some(()) + } + + fn identifier(&mut self) -> Option<&'a str> { + let token = self.tokens.get(self.position)?; + if !matches!( + token.token_type, + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Final + ) { + return None; + } + self.position += 1; + Some(&token.text) + } + + fn query(&mut self, depth: usize) -> Option { + if depth > MAX_TEMPLATE_DEPTH { + return None; + } + let mut names: HashSet = HashSet::new(); + let mut cte_calls: Vec = Vec::new(); + let has_with = self.take(TokenType::With); + if has_with { + loop { + names.insert(self.identifier()?.to_owned()); + self.require(TokenType::As)?; + self.require(TokenType::LParen)?; + cte_calls.extend(self.query(depth + 1)?.calls); + self.require(TokenType::RParen)?; + if !self.take(TokenType::Comma) { + break; + } + } + } + let mut calls = self.select(depth)?; + while self.take(TokenType::Union) { + if has_with { + return None; + } + self.require(TokenType::All)?; + if self.take(TokenType::By) { + let name = self.tokens.get(self.position)?; + if name.token_type != TokenType::Var || !name.text.eq_ignore_ascii_case("name") { + return None; + } + self.position += 1; + } + calls.extend(self.select(depth)?); + } + calls.extend(cte_calls); + Some(QueryFacts { names, calls }) + } + + fn select(&mut self, depth: usize) -> Option> { + self.require(TokenType::Select)?; + loop { + if !self.take(TokenType::Star) { + self.scalar(depth + 1)?; + if self.take(TokenType::As) { + self.identifier()?; + } + } + if !self.take(TokenType::Comma) { + break; + } + } + self.require(TokenType::From)?; + let mut calls: Vec = self.relation()?.into_iter().collect(); + loop { + let left = self.take(TokenType::Left); + if !self.take(TokenType::Join) { + if left { + return None; + } + break; + } + calls.extend(self.relation()?); + self.require(TokenType::On)?; + self.condition(depth + 1)?; + } + if self.take(TokenType::Where) { + self.condition(depth + 1)?; + } + Some(calls) + } + + fn relation(&mut self) -> Option> { + let name = self.identifier()?; + let call = if self.take(TokenType::LParen) { + let function = name.to_ascii_lowercase(); + if !matches!( + function.as_str(), + "__ref" | "__source" | "__seed" | "__dbt_ref" + ) { + return None; + } + let mut target = self.identifier()?.to_owned(); + if function == DBT_REF_FUNCTION && self.take(TokenType::Comma) { + target.push_str("__"); + target.push_str(self.identifier()?); + } + self.require(TokenType::RParen)?; + Some((function, target)) + } else { + None + }; + if self.take(TokenType::As) { + self.identifier()?; + } + Some(call) + } + + fn condition(&mut self, depth: usize) -> Option<()> { + self.scalar(depth)?; + self.require(TokenType::Eq)?; + self.scalar(depth) + } + + fn scalar(&mut self, depth: usize) -> Option<()> { + self.atom(depth)?; + while self.take(TokenType::Plus) + || self.take(TokenType::Dash) + || self.take(TokenType::Star) + || self.take(TokenType::Slash) + { + self.atom(depth)?; + } + Some(()) + } + + fn atom(&mut self, depth: usize) -> Option<()> { + if depth > MAX_TEMPLATE_DEPTH { + return None; + } + if self.take(TokenType::LParen) { + self.scalar(depth + 1)?; + return self.require(TokenType::RParen); + } + if self.take(TokenType::Cast) { + self.require(TokenType::LParen)?; + self.scalar(depth + 1)?; + self.require(TokenType::As)?; + let data_type = self.tokens.get(self.position)?; + if !matches!( + data_type.token_type, + TokenType::Var | TokenType::Int | TokenType::Double + ) || !["int", "integer", "double"] + .iter() + .any(|name| data_type.text.eq_ignore_ascii_case(name)) + { + return None; + } + self.position += 1; + return self.require(TokenType::RParen); + } + if let Some(token) = self.tokens.get(self.position) + && token.token_type == TokenType::Number + { + if !token.text.bytes().any(|byte| byte.is_ascii_digit()) + || !token + .text + .bytes() + .all(|byte| byte.is_ascii_digit() || byte == b'.') + || token.text.bytes().filter(|byte| *byte == b'.').count() > 1 + { + return None; + } + self.position += 1; + return Some(()); + } + let name = self.identifier()?; + if self.take(TokenType::LParen) { + if !name.eq_ignore_ascii_case("coalesce") { + return None; + } + self.scalar(depth + 1)?; + while self.take(TokenType::Comma) { + self.scalar(depth + 1)?; + } + self.require(TokenType::RParen) + } else { + if self.take(TokenType::Dot) { + self.identifier()?; + } + Some(()) + } + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index 602fb830b6..5700adc60d 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/simple_template/helpers.rs"] +mod simple_template_helpers; #[path = "tests/test_sql_test_cte_rename.rs"] mod sql_test_cte_rename; #[path = "tests/test_sql_test_cte_slices.rs"] @@ -22,5 +24,7 @@ mod sql_test_planning; mod sql_test_relation_markers; #[path = "tests/test_sql_test_rendering.rs"] mod sql_test_rendering; +#[path = "tests/test_sql_test_simple_template.rs"] +mod sql_test_simple_template; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs new file mode 100644 index 0000000000..968c9a404e --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs @@ -0,0 +1,77 @@ +use polyglot_sql::Dialect; + +use crate::compiler::_helpers::sql_tests::planning::{AnalysisTemplate, parsed_analysis_template}; +use crate::compiler::_helpers::sql_tests::simple_template::simple_analysis_template; +use crate::compiler::tests::test_types::SimpleTemplateCorpusOutcome; + +pub(crate) fn simple_template_corpus_outcome() -> SimpleTemplateCorpusOutcome { + let dialect = Dialect::get_by_name("duckdb").expect("DuckDB dialect"); + let mut queries: Vec = Vec::new(); + for projection in [ + "id", + "*", + "CAST(id AS INTEGER) AS id", + "COALESCE(id, 0) + id * 2 AS id", + "CAST((id + 1) / 2 AS DOUBLE) AS id", + "1.5 AS id", + "\"__ref(ignored)\" AS id", + "\"a\"\"b\" AS id", + ] { + for relation in [ + "__ref(\"orders\")", + "__source(\"orders\")", + "__seed(\"orders\")", + "__dbt_ref(\"shop\", \"orders\")", + "\"__ref\"(\"orders\")", + "__REF(orders)", + "\"__REF\"(\"orders\")", + "__ref(final)", + "__ref(\"a.b\")", + "__ref(\"a\"\"b\")", + ] { + let body = format!("SELECT {projection} FROM {relation}"); + for sql in [ + body.clone(), + format!("WITH recent AS ({body}) SELECT id FROM recent"), + format!("{body} UNION ALL BY NAME {body}"), + format!( + "WITH recent AS ({body}) SELECT id FROM __ref(\"customers\") AS c JOIN recent AS r ON c.id = r.id" + ), + ] { + let tokens = dialect.tokenize(&sql).expect("generated SQL tokenizes"); + for token in &tokens { + queries.push(format!( + "{} {}", + &sql[..token.span.start], + &sql[token.span.end..] + )); + for inserted in [".", ")", "(", "FROM", "'__ref(\"ignored\")'"] { + queries.push(format!( + "{} {inserted} {}", + &sql[..token.span.start], + &sql[token.span.start..] + )); + } + } + queries.push(sql); + } + } + } + let recognized: Vec<(String, AnalysisTemplate)> = queries + .into_iter() + .filter_map(|sql| { + simple_analysis_template(&sql, "duckdb", &dialect).map(|facts| (sql, facts)) + }) + .collect(); + SimpleTemplateCorpusOutcome { + recognized: recognized.len(), + mismatches: recognized + .into_iter() + .filter_map(|(sql, fast)| { + let parsed = parsed_analysis_template(&sql, &dialect); + (Some(&fast) != parsed.as_ref()) + .then(|| format!("{sql}: fast={fast:?}, parsed={parsed:?}")) + }) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs new file mode 100644 index 0000000000..8b2101a4dc --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs @@ -0,0 +1,235 @@ +use std::collections::HashMap; +use std::sync::Mutex; + +use polyglot_sql::Dialect; + +use crate::compiler::_helpers::sql_tests::planning::{ + analysis_template, parsed_analysis_template, take_template_ast_parse_count, +}; +use crate::compiler::_helpers::sql_tests::simple_template::simple_analysis_template; +use crate::compiler::tests::simple_template_helpers::simple_template_corpus_outcome; +use crate::compiler::tests::test_types::{SimpleTemplateCorpusTestCase, SimpleTemplateTestCase}; + +#[test] +fn given_bounded_and_fallback_queries_when_planning_then_facts_and_ast_counts_match() { + let test_cases = [ + SimpleTemplateTestCase { + description: "dense CTE projections joins and union remove one cold template AST", + dialect: "duckdb", + sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), +lookup_0 AS (SELECT * FROM __source("customers")), +projected AS (SELECT CAST(b.id AS INTEGER) AS id, +CAST(b.amount + COALESCE(l0.amount, 0) AS DOUBLE) AS amount, +CAST(COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE) AS DOUBLE) AS quantity +FROM imported AS b LEFT JOIN lookup_0 AS l0 ON l0.id = b.id), +combined AS (SELECT id, amount, quantity FROM projected +UNION ALL BY NAME SELECT id, amount, quantity FROM projected WHERE id = 0), +final AS (SELECT CAST(id AS INTEGER) AS id, CAST(amount AS DOUBLE) AS amount FROM combined) +SELECT id, amount FROM final"#, + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "nested CTEs follow outer relation then WITH traversal order", + dialect: "duckdb", + sql: "WITH recent AS (WITH nested AS (SELECT * FROM __source(\"orders\")) SELECT id FROM nested) SELECT id FROM recent JOIN __ref(\"customers\") AS c ON c.id = recent.id", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "root union has no Select CTE names", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") UNION ALL SELECT id FROM __source(\"orders\")", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "quoted names escapes and line-comment marker lookalikes", + dialect: "duckdb", + sql: "WITH \"recent\" AS (SELECT * FROM __ref(\"or\"\"ders\")) SELECT id FROM \"recent\" -- __seed(\"ignored\")", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "two-part dbt references precede joined seeds", + dialect: "duckdb", + sql: "SELECT id FROM __dbt_ref(\"shop\", \"orders\") AS o JOIN __seed(\"regions\") AS r ON o.id = r.id", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "scalar nested subquery retains full parser", + dialect: "duckdb", + sql: "SELECT (SELECT id FROM __ref(\"orders\")) AS id", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "string marker lookalike retains full parser", + dialect: "duckdb", + sql: "SELECT '__ref(\"ignored\")' AS note FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "table function call retains full parser", + dialect: "duckdb", + sql: "SELECT id FROM __table_fn(\"orders\")(1)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "scalar function marker retains full parser", + dialect: "duckdb", + sql: "SELECT __udf(\"amount\")(id) AS id FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "WITH over UNION retains full parser AST attachment", + dialect: "duckdb", + sql: "WITH recent AS (SELECT id FROM orders) SELECT id FROM recent UNION ALL SELECT id FROM recent", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unterminated block comment retains full parser policy", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") /* unclosed", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "nested block comments retain full parser policy", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") /* outer /* __ref(\"ignored\") */ inner */", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "missing projection retains parser failure", + dialect: "duckdb", + sql: "SELECT FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "incomplete cast retains parser failure", + dialect: "duckdb", + sql: "SELECT CAST(id AS INTEGER FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "incomplete WHERE retains parser failure", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") WHERE", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unterminated identifier retains parser failure", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unterminated table identifier retains parser failure", + dialect: "duckdb", + sql: "SELECT id FROM \"orders", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "incomplete join retains parser failure", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") LEFT", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "multiple statements retain textual fallback", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\"); SELECT id FROM other", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "literal marker argument is not a Column", + dialect: "duckdb", + sql: "SELECT id FROM __ref('orders')", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "postgres dollar quoting retains its parser", + dialect: "postgres", + sql: "SELECT $$__ref(\"ignored\")$$ AS note FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "snowflake quoted names retain its parser", + dialect: "snowflake", + sql: "SELECT id FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "bigquery backtick names retain its parser", + dialect: "bigquery", + sql: "SELECT id FROM __ref(`orders`)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "tsql brackets retain its parser", + dialect: "tsql", + sql: "SELECT id FROM __ref([orders])", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "databricks quoting retains its parser", + dialect: "databricks", + sql: "SELECT id FROM __ref(`orders`)", + expected_ast_parses: 1, + }, + ]; + for test_case in test_cases { + let dialect = Dialect::get_by_name(test_case.dialect).expect("supported test dialect"); + take_template_ast_parse_count(); + let expected = parsed_analysis_template(test_case.sql, &dialect); + assert_eq!( + take_template_ast_parse_count(), + 1, + "{}", + test_case.description + ); + let fast = simple_analysis_template(test_case.sql, test_case.dialect, &dialect); + assert_eq!( + usize::from(fast.is_none()), + test_case.expected_ast_parses, + "{}", + test_case.description + ); + assert_eq!( + fast.as_ref(), + expected + .as_ref() + .filter(|_| test_case.expected_ast_parses == 0), + "{}", + test_case.description + ); + let cache = Mutex::new(HashMap::new()); + let actual = + analysis_template(test_case.sql, test_case.dialect, &cache).expect("template cache"); + assert_eq!(actual, expected, "{}", test_case.description); + assert_eq!( + take_template_ast_parse_count(), + test_case.expected_ast_parses, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_generated_and_mutated_queries_when_recognized_then_every_fact_matches_the_ast() { + let test_cases = [SimpleTemplateCorpusTestCase { + description: "combinations and token deletions preserve parser acceptance and facts", + expected_min_recognized: 200, + expected_mismatches: &[], + }]; + for test_case in test_cases { + let outcome = simple_template_corpus_outcome(); + assert!( + outcome.recognized >= test_case.expected_min_recognized, + "{}: {} recognized", + test_case.description, + outcome.recognized + ); + assert_eq!( + outcome.mismatches, test_case.expected_mismatches, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index b94a1c4d0f..1b6964acab 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -186,3 +186,20 @@ pub(crate) struct SqlTestAssemblyTestCase { pub(crate) batch: crate::compiler::models::SqlTestAssemblyBatch, pub(crate) expected_outcomes: Vec, } +pub(crate) struct SimpleTemplateTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub sql: &'static str, + pub expected_ast_parses: usize, +} + +pub(crate) struct SimpleTemplateCorpusTestCase { + pub description: &'static str, + pub expected_min_recognized: usize, + pub expected_mismatches: &'static [&'static str], +} + +pub(crate) struct SimpleTemplateCorpusOutcome { + pub recognized: usize, + pub mismatches: Vec, +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py index 9a71b45d43..ded3f76be0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py @@ -86,6 +86,31 @@ @pytest.mark.parametrize( "test_case", [ + CommentedFixtureE2ETestCase( + description="bounded CTE casts joins and unions execute with mocked sources", + test_files={ + "macros/status.py": "", + "models/orders.sql": ( + "MODEL (description 'Orders through projected CTEs.');\n" + 'WITH imported AS (SELECT * FROM __source("raw_orders")),\n' + "projected AS (SELECT CAST(COALESCE(b.order_id, 0) + 0 AS INTEGER) AS order_id,\n" + "b.status AS status FROM imported AS b\n" + "LEFT JOIN imported AS c ON c.order_id = b.order_id),\n" + "combined AS (SELECT order_id, status FROM projected\n" + "UNION ALL BY NAME SELECT order_id, status FROM projected WHERE order_id = 0),\n" + "final AS (SELECT CAST(order_id AS INTEGER) AS order_id, status FROM combined)\n" + "SELECT order_id, status FROM final\n" + ), + "tests/unit/test_orders.sql": ( + 'TEST (name "projected_orders");\nWITH\n' + "__source__raw_orders AS (\n" + "SELECT 1 AS order_id, 'paid' AS status UNION ALL SELECT 2, 'void'\n" + "),\n" + _LINE_COMMENTED_EXPECTED + ), + }, + expected_exit_code=0, + expected_output_fragments=("PASS=1", "FAIL=0"), + ), CommentedFixtureE2ETestCase( description="nested model CTEs and marker lookalikes preserve mocked source results", test_files={ From a50d24a6f946d5e8c88b298cdeab8b66606cbd67 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:31:01 +0100 Subject: [PATCH 14/28] fix: recognize Snowflake test templates with parser-safe markers --- .../compiler/_helpers/sql_tests/constants.rs | 2 +- .../_helpers/sql_tests/simple_template.rs | 22 ++- .../compiler/tests/simple_template/helpers.rs | 45 ++++++- .../tests/test_sql_test_simple_template.rs | 125 ++++++++++++++++-- .../src/compiler/tests/test_types.rs | 1 + 5 files changed, 175 insertions(+), 20 deletions(-) diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs index 586f6c747f..5c74d8b071 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs @@ -1,2 +1,2 @@ -pub(crate) const SIMPLE_TEMPLATE_DIALECT: &str = "duckdb"; +pub(crate) const SIMPLE_TEMPLATE_DIALECTS: [&str; 2] = ["duckdb", "snowflake"]; pub(crate) const MAX_TEMPLATE_DEPTH: usize = 32; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs index e47f34c16e..6709e33764 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs @@ -3,21 +3,22 @@ use std::collections::HashSet; use polyglot_sql::Dialect; +use polyglot_sql::parser::NO_PAREN_FUNCTION_NAMES; use polyglot_sql::tokens::{Token, TokenType}; use crate::compiler::_helpers::sql_tests::constants::{ - MAX_TEMPLATE_DEPTH, SIMPLE_TEMPLATE_DIALECT, + MAX_TEMPLATE_DEPTH, SIMPLE_TEMPLATE_DIALECTS, }; use crate::compiler::_helpers::sql_tests::planning::{AnalysisTemplate, DBT_REF_FUNCTION}; use crate::compiler::_helpers::sql_tests::relation_markers::MarkerCall; -/// Recognize the bounded DuckDB subset; every other dialect retains the existing parser. +/// Recognize bounded DuckDB/Snowflake productions; other syntax retains the existing parser. pub(crate) fn simple_analysis_template( sql: &str, dialect_name: &str, dialect: &Dialect, ) -> Option { - if dialect_name != SIMPLE_TEMPLATE_DIALECT || sql.contains("/*") { + if !SIMPLE_TEMPLATE_DIALECTS.contains(&dialect_name) || sql.contains("/*") { return None; } let tokens = match dialect.tokenize(sql) { @@ -160,10 +161,10 @@ impl<'a> TemplateReader<'a> { ) { return None; } - let mut target = self.identifier()?.to_owned(); + let mut target = self.marker_argument()?.to_owned(); if function == DBT_REF_FUNCTION && self.take(TokenType::Comma) { target.push_str("__"); - target.push_str(self.identifier()?); + target.push_str(self.marker_argument()?); } self.require(TokenType::RParen)?; Some((function, target)) @@ -176,6 +177,17 @@ impl<'a> TemplateReader<'a> { Some(call) } + /// Require quoted Columns; unquoted arguments can invoke private dialect-specific function rules. + fn marker_argument(&mut self) -> Option<&'a str> { + let token = self.tokens.get(self.position)?; + if token.token_type != TokenType::QuotedIdentifier + || NO_PAREN_FUNCTION_NAMES.contains(token.text.to_ascii_uppercase().as_str()) + { + return None; + } + self.identifier() + } + fn condition(&mut self, depth: usize) -> Option<()> { self.scalar(depth)?; self.require(TokenType::Eq)?; diff --git a/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs index 968c9a404e..8fb1f0df7c 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs @@ -1,11 +1,48 @@ use polyglot_sql::Dialect; +use polyglot_sql::parser::NO_PAREN_FUNCTION_NAMES; +use std::collections::HashMap; +use std::sync::Mutex; -use crate::compiler::_helpers::sql_tests::planning::{AnalysisTemplate, parsed_analysis_template}; +use crate::compiler::_helpers::sql_tests::planning::{ + AnalysisTemplate, analysis_template, parsed_analysis_template, +}; use crate::compiler::_helpers::sql_tests::simple_template::simple_analysis_template; use crate::compiler::tests::test_types::SimpleTemplateCorpusOutcome; -pub(crate) fn simple_template_corpus_outcome() -> SimpleTemplateCorpusOutcome { - let dialect = Dialect::get_by_name("duckdb").expect("DuckDB dialect"); +pub(crate) fn no_paren_marker_mismatches(dialect_name: &str) -> Vec { + let dialect = Dialect::get_by_name(dialect_name).expect("supported test dialect"); + let cache = Mutex::new(HashMap::new()); + NO_PAREN_FUNCTION_NAMES + .iter() + .flat_map(|name| [name.to_string(), name.to_ascii_lowercase()]) + .flat_map(|name| [name.clone(), format!("\"{name}\"")]) + .flat_map(|argument| { + [ + format!("__ref({argument})"), + format!("__source({argument})"), + format!("__seed({argument})"), + format!("__dbt_ref({argument})"), + format!("__dbt_ref({argument}, \"orders\")"), + format!("__dbt_ref(\"orders\", {argument})"), + ] + }) + .filter_map(|relation| { + let sql = format!("SELECT id FROM {relation}"); + let fast = simple_analysis_template(&sql, dialect_name, &dialect); + let expected = parsed_analysis_template(&sql, &dialect); + let actual = analysis_template(&sql, dialect_name, &cache).expect("template cache"); + (fast.is_some() + || actual != expected + || expected + .as_ref() + .is_some_and(|facts| !facts.marker_calls.is_empty())) + .then(|| format!("{sql}: fast={fast:?}, actual={actual:?}, expected={expected:?}")) + }) + .collect() +} + +pub(crate) fn simple_template_corpus_outcome(dialect_name: &str) -> SimpleTemplateCorpusOutcome { + let dialect = Dialect::get_by_name(dialect_name).expect("supported test dialect"); let mut queries: Vec = Vec::new(); for projection in [ "id", @@ -60,7 +97,7 @@ pub(crate) fn simple_template_corpus_outcome() -> SimpleTemplateCorpusOutcome { let recognized: Vec<(String, AnalysisTemplate)> = queries .into_iter() .filter_map(|sql| { - simple_analysis_template(&sql, "duckdb", &dialect).map(|facts| (sql, facts)) + simple_analysis_template(&sql, dialect_name, &dialect).map(|facts| (sql, facts)) }) .collect(); SimpleTemplateCorpusOutcome { diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs index 8b2101a4dc..a11546bfe3 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs @@ -7,12 +7,108 @@ use crate::compiler::_helpers::sql_tests::planning::{ analysis_template, parsed_analysis_template, take_template_ast_parse_count, }; use crate::compiler::_helpers::sql_tests::simple_template::simple_analysis_template; -use crate::compiler::tests::simple_template_helpers::simple_template_corpus_outcome; -use crate::compiler::tests::test_types::{SimpleTemplateCorpusTestCase, SimpleTemplateTestCase}; +use crate::compiler::tests::simple_template_helpers::{ + no_paren_marker_mismatches, simple_template_corpus_outcome, +}; +use crate::compiler::tests::test_types::{ + RelationMarkerOracleTestCase, SimpleTemplateCorpusTestCase, SimpleTemplateTestCase, +}; + +#[test] +fn given_registered_no_paren_arguments_when_planning_then_all_marker_positions_use_the_parser() { + let test_cases = [ + RelationMarkerOracleTestCase { + description: "all registry names in quoted and unquoted marker argument positions", + dialect: "duckdb", + expected_mismatches: &[], + }, + RelationMarkerOracleTestCase { + description: "all Snowflake registry names in quoted and unquoted marker argument positions", + dialect: "snowflake", + expected_mismatches: &[], + }, + ]; + for test_case in test_cases { + assert_eq!( + no_paren_marker_mismatches(test_case.dialect), + test_case.expected_mismatches, + "{}", + test_case.description + ); + } +} #[test] fn given_bounded_and_fallback_queries_when_planning_then_facts_and_ast_counts_match() { let test_cases = [ + SimpleTemplateTestCase { + description: "Snowflake dense projections and union remove the template AST", + dialect: "snowflake", + sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), +projected AS (SELECT CAST(b.id AS INTEGER) AS id, +CAST(COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE) AS DOUBLE) AS amount FROM imported AS b), +combined AS (SELECT id, amount FROM projected UNION ALL BY NAME SELECT id, amount FROM projected WHERE id = 0), +final AS (SELECT CAST(id AS INTEGER) AS id, CAST(amount AS DOUBLE) AS amount FROM combined) +SELECT id, amount FROM final"#, + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "Snowflake dense UDF projection retains the template AST", + dialect: "snowflake", + sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), +projected AS (SELECT CAST(b.id AS INTEGER) AS id, +CAST(__udf("amount")(b.amount) AS DOUBLE) AS amount FROM imported AS b) +SELECT id, amount FROM projected"#, + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "Snowflake QUALIFY remains on the full parser", + dialect: "snowflake", + sql: "SELECT id FROM __ref(\"orders\") QUALIFY ROW_NUMBER() OVER (ORDER BY id) = 1", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "quoted no-paren source argument retains parser Function classification", + dialect: "duckdb", + sql: "SELECT id FROM __source(\"now\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unquoted no-paren reference argument retains parser Function classification", + dialect: "duckdb", + sql: "SELECT id FROM __ref(NOW)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "first dbt argument with no-paren name retains parser classification", + dialect: "duckdb", + sql: "SELECT id FROM __dbt_ref(\"pi\", \"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "second dbt argument with no-paren name retains parser classification", + dialect: "duckdb", + sql: "SELECT id FROM __dbt_ref(\"orders\", \"getdate\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unquoted ordinary argument conservatively retains full parser", + dialect: "duckdb", + sql: "SELECT id FROM __seed(orders)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unquoted dialect-specific user argument retains full parser", + dialect: "duckdb", + sql: "SELECT id FROM __source(user)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "quoted user is an ordinary Column in the pinned parser", + dialect: "duckdb", + sql: "SELECT id FROM __source(\"user\")", + expected_ast_parses: 0, + }, SimpleTemplateTestCase { description: "dense CTE projections joins and union remove one cold template AST", dialect: "duckdb", @@ -149,10 +245,10 @@ SELECT id, amount FROM final"#, expected_ast_parses: 1, }, SimpleTemplateTestCase { - description: "snowflake quoted names retain its parser", + description: "snowflake ordinary quoted names use recognized facts", dialect: "snowflake", sql: "SELECT id FROM __ref(\"orders\")", - expected_ast_parses: 1, + expected_ast_parses: 0, }, SimpleTemplateTestCase { description: "bigquery backtick names retain its parser", @@ -213,13 +309,22 @@ SELECT id, amount FROM final"#, #[test] fn given_generated_and_mutated_queries_when_recognized_then_every_fact_matches_the_ast() { - let test_cases = [SimpleTemplateCorpusTestCase { - description: "combinations and token deletions preserve parser acceptance and facts", - expected_min_recognized: 200, - expected_mismatches: &[], - }]; + let test_cases = [ + SimpleTemplateCorpusTestCase { + description: "combinations and token deletions preserve parser acceptance and facts", + dialect: "duckdb", + expected_min_recognized: 200, + expected_mismatches: &[], + }, + SimpleTemplateCorpusTestCase { + description: "Snowflake token productions preserve parser acceptance and facts", + dialect: "snowflake", + expected_min_recognized: 200, + expected_mismatches: &[], + }, + ]; for test_case in test_cases { - let outcome = simple_template_corpus_outcome(); + let outcome = simple_template_corpus_outcome(test_case.dialect); assert!( outcome.recognized >= test_case.expected_min_recognized, "{}: {} recognized", diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 1b6964acab..d3884c1bb4 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -195,6 +195,7 @@ pub(crate) struct SimpleTemplateTestCase { pub(crate) struct SimpleTemplateCorpusTestCase { pub description: &'static str, + pub dialect: &'static str, pub expected_min_recognized: usize, pub expected_mismatches: &'static [&'static str], } From 3b695547bff7e92fb87fa8fd322ef6cae8c591b2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:59:28 +0100 Subject: [PATCH 15/28] perf: dispatch SQL non-code scans by opening byte --- .../src/sql_scan/main/non_code_end.rs | 8 +- .../src/sql_scan/tests/helpers.rs | 78 ++++++++++++++- .../src/sql_scan/tests/test_sql_scan.rs | 99 ++++++++++++++++++- .../src/sql_scan/tests/test_types.rs | 14 +++ 4 files changed, 193 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs index 8e8bdaf663..b104206872 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs @@ -6,16 +6,16 @@ use crate::sql_scan::main::quote_end::quote_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; /// Return the end of a comment or quoted text starting at `index`, if one starts there. +#[inline] pub fn non_code_end( sql: &[u8], index: usize, policy: QuotePolicy, ) -> Result, Unclosed> { - if let Some(end) = comment_end(sql, index)? { - return Ok(Some(end)); - } match sql.get(index) { - Some(byte) if policy.is_quote(*byte) => quote_end(sql, index, policy).map(Some), + Some(b'-' | b'/') => comment_end(sql, index), + Some(b'\'' | b'"') => quote_end(sql, index, policy).map(Some), + Some(b'`') if policy.backtick_identifiers => quote_end(sql, index, policy).map(Some), Some(b'$') if policy.dollar_quotes => dollar_quote_end(sql, index), _ => Ok(None), } diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs index 4adf633977..911a581657 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs @@ -1,6 +1,82 @@ //! Shared lexical syntaxes for SQL scanning tests. -use crate::sql_scan::models::LexicalSyntax; +use crate::sql_scan::_helpers::dollar::dollar_quote_end; +use crate::sql_scan::main::comment_end::comment_end; +use crate::sql_scan::main::quote_end::quote_end; +use crate::sql_scan::models::{LexicalSyntax, QuotePolicy, Unclosed}; +use crate::sql_scan::tests::test_types::DispatchTestCase; + +// The pre-optimization comment-first dispatch is the differential oracle. Keep it +// independent of non_code_end so its first-byte dispatch is what the tests exercise. +fn comment_first_end( + sql: &[u8], + index: usize, + policy: QuotePolicy, +) -> Result, Unclosed> { + comment_end(sql, index)?.map_or_else( + || { + sql.get(index) + .filter(|byte| policy.is_quote(**byte)) + .map_or_else( + || { + sql.get(index) + .filter(|byte| **byte == b'$' && policy.dollar_quotes) + .map_or(Ok(None), |_| dollar_quote_end(sql, index)) + }, + |_| quote_end(sql, index, policy).map(Some), + ) + }, + |end| Ok(Some(end)), + ) +} + +pub(crate) fn dispatch_truncation_cases(input: &[u8]) -> Vec { + let mut cases = Vec::new(); + for flags in 0..16 { + let policy = QuotePolicy { + backtick_identifiers: flags & 1 != 0, + single_quote_backslash_escapes: flags & 2 != 0, + double_quote_backslash_escapes: flags & 4 != 0, + dollar_quotes: flags & 8 != 0, + }; + // The byte API must also handle truncations inside a multibyte character. + for length in 0..=input.len() { + let sql = &input[..length]; + for index in 0..=length + 1 { + cases.push(DispatchTestCase { + description: format!( + "input={input:?}, length={length}, index={index}, policy={policy:?}" + ), + sql: sql.to_vec(), + index, + policy, + expected_end: comment_first_end(sql, index, policy), + }); + } + } + } + cases +} + +pub(crate) fn ordinary_byte_cases(suffix: &[u8]) -> Vec { + let mut cases = Vec::new(); + for byte in + (u8::MIN..=u8::MAX).filter(|byte| !matches!(byte, b'-' | b'/' | b'\'' | b'"' | b'`' | b'$')) + { + for policy in [QuotePolicy::COMPILER, QuotePolicy::SQL_LINT] { + let mut sql = vec![byte]; + sql.extend_from_slice(suffix); + cases.push(DispatchTestCase { + description: format!("byte={byte}, suffix={suffix:?}, policy={policy:?}"), + sql, + index: 0, + policy, + expected_end: Ok(None), + }); + } + } + cases +} pub(crate) fn generic() -> LexicalSyntax { LexicalSyntax { diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs index 371063385b..d1b333d3a3 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs @@ -2,7 +2,10 @@ use crate::sql_scan::main::matching_paren::matching_paren; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::QuotePolicy; use crate::sql_scan::models::Unclosed; -use crate::sql_scan::tests::test_types::{MatchingParenPolicyTestCase, NonCodeEndTestCase}; +use crate::sql_scan::tests::helpers::{dispatch_truncation_cases, ordinary_byte_cases}; +use crate::sql_scan::tests::test_types::{ + DispatchInputTestCase, MatchingParenPolicyTestCase, NonCodeEndTestCase, +}; const QUOTES_ONLY: QuotePolicy = QuotePolicy { backtick_identifiers: false, @@ -18,6 +21,100 @@ const BACKSLASH_ONLY: QuotePolicy = QuotePolicy { dollar_quotes: false, }; +#[test] +fn given_every_truncation_and_policy_when_dispatching_then_matches_comment_first_scanning() { + let test_cases = [ + DispatchInputTestCase { + description: "ordinary code and non-nesting comments", + expected_cases: dispatch_truncation_cases, + sql: b"SELECT orders.id - 1 / 2; -- note\r\n/* outer /* inner */ tail */", + }, + DispatchInputTestCase { + description: "multibyte text and doubled quotes", + expected_cases: dispatch_truncation_cases, + sql: "SELECT 'é''ü', \"ñ\"\"名\", `名``é`, $$ ' -- /* ü $$".as_bytes(), + }, + DispatchInputTestCase { + description: "backslash escapes and tagged dollar quotes", + expected_cases: dispatch_truncation_cases, + sql: br#"'a\'b' "c\"d" `e\`f` $tag_2$ /* -- ' " $other$ $tag_2$"#, + }, + DispatchInputTestCase { + description: "invalid dollar tags and unclosed quotes", + expected_cases: dispatch_truncation_cases, + sql: b"$ $1 $1$ $tag $tag$unterminated $$unterminated", + }, + DispatchInputTestCase { + description: "dollar tag word boundaries", + expected_cases: dispatch_truncation_cases, + sql: "word$tag$ é$tag$ _$$ $tag$body$tag$".as_bytes(), + }, + DispatchInputTestCase { + description: "truncated opening delimiters", + expected_cases: dispatch_truncation_cases, + sql: b"- / -- /* ' \" ` \\", + }, + ]; + for test_case in test_cases { + for sample in (test_case.expected_cases)(test_case.sql) { + assert_eq!( + non_code_end(&sample.sql, sample.index, sample.policy), + sample.expected_end, + "{}: {}", + test_case.description, + sample.description, + ); + } + } +} + +#[test] +fn given_non_opening_bytes_when_dispatching_then_returns_no_boundary() { + let test_cases = [ + DispatchInputTestCase { + description: "before a line comment", + expected_cases: ordinary_byte_cases, + sql: b"--", + }, + DispatchInputTestCase { + description: "before a block comment", + expected_cases: ordinary_byte_cases, + sql: b"/*", + }, + DispatchInputTestCase { + description: "before a single quote", + expected_cases: ordinary_byte_cases, + sql: b"'", + }, + DispatchInputTestCase { + description: "before a double quote", + expected_cases: ordinary_byte_cases, + sql: b"\"", + }, + DispatchInputTestCase { + description: "before a backtick", + expected_cases: ordinary_byte_cases, + sql: b"`", + }, + DispatchInputTestCase { + description: "before a dollar quote", + expected_cases: ordinary_byte_cases, + sql: b"$$", + }, + ]; + for test_case in test_cases { + for sample in (test_case.expected_cases)(test_case.sql) { + assert_eq!( + non_code_end(&sample.sql, sample.index, sample.policy), + sample.expected_end, + "{}: {}", + test_case.description, + sample.description, + ); + } + } +} + #[test] fn given_quote_policies_when_matching_parentheses_then_policy_decides_quote_boundaries() { let test_cases = [ diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs index 4507180b90..42ebb866b3 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs @@ -15,6 +15,20 @@ pub(crate) struct NonCodeEndTestCase { pub(crate) expected_end: Result, Unclosed>, } +pub(crate) struct DispatchTestCase { + pub(crate) description: String, + pub(crate) sql: Vec, + pub(crate) index: usize, + pub(crate) policy: QuotePolicy, + pub(crate) expected_end: Result, Unclosed>, +} + +pub(crate) struct DispatchInputTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static [u8], + pub(crate) expected_cases: fn(&[u8]) -> Vec, +} + pub(crate) struct DialectNonCodeEndTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, From 39fbafe84354b427b67c4c9ee085848e61a0d030 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:26:39 +0100 Subject: [PATCH 16/28] perf: key internal rules caches with BLAKE3 Rule-cache, custom-rule-cache, rows-memo and model-finding identities are internal, build-scoped keys. Replace software SHA-256 with BLAKE3 and stream model JSON into the hasher instead of buffering it. Bump the three cache versions so old entries are ignored. --- Cargo.lock | 1 + crates/sqlbuild-rules/Cargo.toml | 1 + crates/sqlbuild-rules/src/constants.rs | 2 +- .../src/custom_rule_cache/_helpers/digest.rs | 10 ++++------ .../src/custom_rule_cache/constants.rs | 2 +- .../src/engine/_helpers/evaluation.rs | 17 ++++++++--------- .../sqlbuild-rules/src/engine/_helpers/memo.rs | 16 ++++++++-------- .../src/rule_cache/_helpers/digest.rs | 10 ++++------ .../sqlbuild-rules/src/rule_cache/constants.rs | 2 +- 9 files changed, 29 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17d744c8d0..6b81e5a5c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1311,6 +1311,7 @@ dependencies = [ name = "sqlbuild-rules" version = "0.157.0" dependencies = [ + "blake3", "fensu-policy", "globset", "polyglot-sql", diff --git a/crates/sqlbuild-rules/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml index 0679b5241b..41b5e84013 100644 --- a/crates/sqlbuild-rules/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +blake3.workspace = true fensu-policy.workspace = true globset.workspace = true polyglot-sql.workspace = true diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index c8e55bb51f..dd728e9a56 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -30,7 +30,7 @@ pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3; pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; -pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v1"; +pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v2"; pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; /// Threshold key setting the fewest test cases a custom rule must ship. pub const MIN_CUSTOM_RULE_TEST_CASES_KEY: &str = "min_custom_rule_test_cases"; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs index 720e15600a..75c4553175 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs @@ -1,21 +1,19 @@ -//! SHA-256 over length-prefixed fields, so no two field lists share an identity. - -use sha2::{Digest, Sha256}; +//! BLAKE3 over length-prefixed fields, so no two field lists share an identity. use crate::constants::NATIVE_BUILD_IDENTITY; use crate::custom_rule_cache::constants::CACHE_VERSION; /// The hex digest of the cache version, the native build and each field in order. pub(crate) fn field_digest(kind: &str, fields: &[&str]) -> String { - let mut digest = Sha256::new(); + let mut digest = blake3::Hasher::new(); for field in [CACHE_VERSION, NATIVE_BUILD_IDENTITY, kind] .iter() .chain(fields.iter()) { - digest.update((field.len() as u64).to_le_bytes()); + digest.update(&(field.len() as u64).to_le_bytes()); digest.update(field.as_bytes()); } - format!("{:x}", digest.finalize()) + digest.finalize().to_hex().to_string() } /// An optional field as a presence flag and its text. diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs b/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs index 18d9b415af..5b0fdb3329 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs @@ -1,7 +1,7 @@ //! The custom-rule result cache format and the inputs its identities separate. /// Bumped whenever identity derivation or the stored entry shape changes. -pub(crate) const CACHE_VERSION: &str = "custom-rules-native-v1"; +pub(crate) const CACHE_VERSION: &str = "custom-rules-native-v2"; pub(crate) const RULE_KIND: &str = "rule"; pub(crate) const PROJECT_SUBJECT_KIND: &str = "project"; pub(crate) const MODEL_SUBJECT_KIND: &str = "model"; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 86c9fa99f2..8878a16fa8 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -20,7 +20,6 @@ use fensu_policy::lifecycle::models::{ #[cfg(test)] use rayon::iter::IndexedParallelIterator; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; -use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use std::time::Instant; @@ -382,13 +381,13 @@ fn model_cache_identities( /// Key one model's findings by payload, model-rules fingerprint, and derived test counts. fn model_cache_identity(model: &crate::models::Model, model_rules: &str) -> Result { - let mut digest = Sha256::new(); + let mut digest = blake3::Hasher::new(); digest.update(MODEL_RULES_IDENTITY_SEED); - digest.update([0]); + digest.update(&[0]); digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); - digest.update([0]); + digest.update(&[0]); digest.update(model_rules.as_bytes()); - digest.update([0]); + digest.update(&[0]); match &model.payload_digest { Some(payload) => { digest.update(b"payload:"); @@ -396,12 +395,12 @@ fn model_cache_identity(model: &crate::models::Model, model_rules: &str) -> Resu } None => { digest.update(b"model:"); - digest.update(serde_json::to_vec(model).map_err(|error| error.to_string())?); + serde_json::to_writer(&mut digest, model).map_err(|error| error.to_string())?; } } - digest.update([0]); - digest.update(model.empty_input_only_test_count.to_le_bytes()); - Ok(format!("{:x}", digest.finalize())) + digest.update(&[0]); + digest.update(&model.empty_input_only_test_count.to_le_bytes()); + Ok(digest.finalize().to_hex().to_string()) } fn validate_suppression_codes( diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index 52219e9666..3452bc99f8 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -3,7 +3,6 @@ use std::path::{Path, PathBuf}; use serde_json::Value; -use sha2::{Digest, Sha256}; use crate::constants::{API_VERSION, NATIVE_ROWS_MEMO_VERSION}; use crate::engine::_helpers::evaluation::evaluate_request; @@ -37,10 +36,11 @@ pub(crate) fn evaluate_rows(rows: RowsRequest) -> Result Result { - let bytes: Vec = serde_json::to_vec(&model).map_err(rules_error)?; - model.payload_digest = Some(format!("{:x}", Sha256::digest(&bytes))); + let mut hasher: blake3::Hasher = blake3::Hasher::new(); + serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; + model.payload_digest = Some(hasher.finalize().to_hex().to_string()); Ok(model) } @@ -50,14 +50,14 @@ fn identity( models: &[Model], code_identity: &str, ) -> Result { - let mut hasher: Sha256 = Sha256::new(); + let mut hasher: blake3::Hasher = blake3::Hasher::new(); for part in [NATIVE_ROWS_MEMO_VERSION, code_identity] { hasher.update(part.as_bytes()); hasher.update(b"\0"); } - hasher.update(serde_json::to_vec(request).map_err(rules_error)?); + serde_json::to_writer(&mut hasher, request).map_err(rules_error)?; hasher.update(b"\0"); - hasher.update((models.len() as u64).to_le_bytes()); + hasher.update(&(models.len() as u64).to_le_bytes()); for model in models { hasher.update( model @@ -68,7 +68,7 @@ fn identity( ); hasher.update(b"\0"); } - Ok(format!("{:x}", hasher.finalize())) + Ok(hasher.finalize().to_hex().to_string()) } /// The stored response for exactly `identity`; an unreadable or foreign memo answers nothing. diff --git a/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs index 50033d3c68..78503eda68 100644 --- a/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs @@ -1,19 +1,17 @@ -//! SHA-256 over length-prefixed fields, so no two field lists share a key. - -use sha2::{Digest, Sha256}; +//! BLAKE3 over length-prefixed fields, so no two field lists share a key. use crate::constants::NATIVE_BUILD_IDENTITY; use crate::rule_cache::constants::CACHE_VERSION; /// The hex digest of the cache version, the native build and each field in order. pub(crate) fn field_digest(kind: &str, fields: &[&str]) -> String { - let mut digest = Sha256::new(); + let mut digest = blake3::Hasher::new(); for field in [CACHE_VERSION, NATIVE_BUILD_IDENTITY, kind] .iter() .chain(fields.iter()) { - digest.update((field.len() as u64).to_le_bytes()); + digest.update(&(field.len() as u64).to_le_bytes()); digest.update(field.as_bytes()); } - format!("{:x}", digest.finalize()) + digest.finalize().to_hex().to_string() } diff --git a/crates/sqlbuild-rules/src/rule_cache/constants.rs b/crates/sqlbuild-rules/src/rule_cache/constants.rs index 4b5361cf03..8e6fff224f 100644 --- a/crates/sqlbuild-rules/src/rule_cache/constants.rs +++ b/crates/sqlbuild-rules/src/rule_cache/constants.rs @@ -1,7 +1,7 @@ //! The SQL rule result cache format and the inputs its keys separate. /// Bumped whenever key derivation or the stored finding shape changes. -pub(crate) const CACHE_VERSION: &str = "sql-rules-v6"; +pub(crate) const CACHE_VERSION: &str = "sql-rules-v7"; pub(crate) const MODEL_KEY_KIND: &str = "model"; pub(crate) const FILE_KEY_KIND: &str = "file"; pub(crate) const LINT_KEY_KIND: &str = "lint"; From 5bce4ecdb112ade68482b7cf778770bf5a03eff9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:52:47 +0100 Subject: [PATCH 17/28] perf: write compiled artifact batches on a small thread pool Large batches of distinct files are written on four threads. The first failure in batch order is still the one reported, unchanged files are still counted, and batches that repeat a path stay sequential so the last queued contents win. --- .../src/artifacts/_helpers/files.rs | 74 ++++++++-- .../sqlbuild-cache/src/artifacts/constants.rs | 4 + crates/sqlbuild-cache/src/artifacts/mod.rs | 3 + crates/sqlbuild-cache/src/artifacts/tests.rs | 6 + .../src/artifacts/tests/helpers.rs | 19 +++ .../src/artifacts/tests/test_types.rs | 26 ++++ .../artifacts/tests/test_write_artifacts.rs | 131 ++++++++++++++++++ 7 files changed, 253 insertions(+), 10 deletions(-) create mode 100644 crates/sqlbuild-cache/src/artifacts/tests.rs create mode 100644 crates/sqlbuild-cache/src/artifacts/tests/helpers.rs create mode 100644 crates/sqlbuild-cache/src/artifacts/tests/test_types.rs create mode 100644 crates/sqlbuild-cache/src/artifacts/tests/test_write_artifacts.rs diff --git a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs index b1b26f57a2..7365790654 100644 --- a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs +++ b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs @@ -5,6 +5,9 @@ use std::fs::{File, OpenOptions}; use std::io::{ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +use crate::artifacts::constants::{PARALLEL_WRITE_MIN_FILES, WRITE_WORKERS}; use crate::artifacts::errors::ArtifactError; use crate::artifacts::models::{Publication, WrittenArtifacts}; @@ -19,23 +22,74 @@ pub(crate) fn write_artifacts( files: &[(PathBuf, Vec)], check_existing: bool, ) -> Result { + if files.len() >= PARALLEL_WRITE_MIN_FILES && distinct_paths(files) { + return write_artifacts_parallel(files, check_existing); + } let mut outcome: WrittenArtifacts = WrittenArtifacts::default(); for (path, contents) in files { - if check_existing && let Some(existing) = read_regular_file(path)? { - if existing == *contents { - outcome.unchanged += 1; - continue; - } - if std::str::from_utf8(&existing).is_err() { - return Err(ArtifactError::ExistingNotUtf8 { path: path.clone() }); - } + if write_artifact(path, contents, check_existing)? { + outcome.written += 1; + } else { + outcome.unchanged += 1; } - overwrite_creating_parent(path, contents)?; - outcome.written += 1; } Ok(outcome) } +/// Write distinct files on a small pool; the first failure in file order is the one reported. +fn write_artifacts_parallel( + files: &[(PathBuf, Vec)], + check_existing: bool, +) -> Result { + let write = || -> Vec> { + files + .par_iter() + .map(|(path, contents)| write_artifact(path, contents, check_existing)) + .collect() + }; + let results: Vec> = match rayon::ThreadPoolBuilder::new() + .num_threads(WRITE_WORKERS) + .build() + { + Ok(pool) => pool.install(write), + Err(_) => write(), + }; + let mut outcome: WrittenArtifacts = WrittenArtifacts::default(); + for result in results { + if result? { + outcome.written += 1; + } else { + outcome.unchanged += 1; + } + } + Ok(outcome) +} + +fn distinct_paths(files: &[(PathBuf, Vec)]) -> bool { + let mut seen: HashSet<&Path> = HashSet::with_capacity(files.len()); + files.iter().all(|(path, _)| seen.insert(path.as_path())) +} + +/// Write one artifact; `false` when a byte-identical existing file was left untouched. +fn write_artifact( + path: &Path, + contents: &[u8], + check_existing: bool, +) -> Result { + if check_existing && let Some(existing) = read_regular_file(path)? { + if existing == contents { + return Ok(false); + } + if std::str::from_utf8(&existing).is_err() { + return Err(ArtifactError::ExistingNotUtf8 { + path: path.to_path_buf(), + }); + } + } + overwrite_creating_parent(path, contents)?; + Ok(true) +} + /// A regular file's bytes, or `None` when nothing comparable is at `path`. fn read_regular_file(path: &Path) -> Result>, ArtifactError> { let mut file: File = match File::open(path) { diff --git a/crates/sqlbuild-cache/src/artifacts/constants.rs b/crates/sqlbuild-cache/src/artifacts/constants.rs index 3feacccd00..f55814e80a 100644 --- a/crates/sqlbuild-cache/src/artifacts/constants.rs +++ b/crates/sqlbuild-cache/src/artifacts/constants.rs @@ -4,3 +4,7 @@ pub const NEW_FILE_MODE: u32 = 0o666; pub(crate) const SQL_EXTENSION: &str = "sql"; #[cfg(unix)] pub(crate) const NANOS_PER_SECOND: i64 = 1_000_000_000; +/// Below this many files a sequential write is cheaper than starting a pool. +pub(crate) const PARALLEL_WRITE_MIN_FILES: usize = 64; +/// Writer threads for one artifact batch, matching SQL test planning's fixed pool. +pub(crate) const WRITE_WORKERS: usize = 4; diff --git a/crates/sqlbuild-cache/src/artifacts/mod.rs b/crates/sqlbuild-cache/src/artifacts/mod.rs index 6b771d183b..dcdc4ec126 100644 --- a/crates/sqlbuild-cache/src/artifacts/mod.rs +++ b/crates/sqlbuild-cache/src/artifacts/mod.rs @@ -5,3 +5,6 @@ pub mod constants; pub mod errors; pub mod main; pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/artifacts/tests.rs b/crates/sqlbuild-cache/src/artifacts/tests.rs new file mode 100644 index 0000000000..991ace2b43 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_write_artifacts.rs"] +mod write_artifacts; diff --git a/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs b/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs new file mode 100644 index 0000000000..febcafc534 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs @@ -0,0 +1,19 @@ +use std::path::{Path, PathBuf}; + +/// `count` distinct SQL artifacts spread over nested directories under `root`. +pub(crate) fn batch(root: &Path, count: usize) -> Vec<(PathBuf, Vec)> { + (0..count) + .map(|index| { + ( + root.join(format!("group_{}/nested/model_{index}.sql", index % 7)), + format!("select {index}\n").into_bytes(), + ) + }) + .collect() +} + +/// Create `path` with `contents`, including its parent directories. +pub(crate) fn place(path: &Path, contents: &[u8]) { + std::fs::create_dir_all(path.parent().expect("artifact parent")).expect("artifact directory"); + std::fs::write(path, contents).expect("existing artifact"); +} diff --git a/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs b/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs new file mode 100644 index 0000000000..0d130d0c77 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs @@ -0,0 +1,26 @@ +/// One artifact batch written over a target where some files already exist. +pub(crate) struct WriteBatchTestCase { + pub(crate) description: &'static str, + pub(crate) file_count: usize, + pub(crate) check_existing: bool, + pub(crate) existing_identical: &'static [usize], + pub(crate) existing_different: &'static [usize], + pub(crate) expected_written: usize, + pub(crate) expected_unchanged: usize, +} + +/// One batch with existing non-UTF-8 files at several positions. +pub(crate) struct FirstFailureTestCase { + pub(crate) description: &'static str, + pub(crate) file_count: usize, + pub(crate) failing: &'static [usize], + pub(crate) expected_failing: usize, +} + +/// One batch that queues the same path more than once. +pub(crate) struct RepeatedPathTestCase { + pub(crate) description: &'static str, + pub(crate) file_count: usize, + pub(crate) repeated: usize, + pub(crate) expected_contents: &'static [u8], +} diff --git a/crates/sqlbuild-cache/src/artifacts/tests/test_write_artifacts.rs b/crates/sqlbuild-cache/src/artifacts/tests/test_write_artifacts.rs new file mode 100644 index 0000000000..0821004a11 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests/test_write_artifacts.rs @@ -0,0 +1,131 @@ +use crate::artifacts::errors::ArtifactError; +use crate::artifacts::main::write_artifacts::write_artifacts; +use crate::artifacts::tests::helpers::{batch, place}; +use crate::artifacts::tests::test_types::{ + FirstFailureTestCase, RepeatedPathTestCase, WriteBatchTestCase, +}; + +#[test] +fn given_artifact_batch_when_written_then_every_file_matches_and_unchanged_files_are_counted() { + let test_cases = [ + WriteBatchTestCase { + description: "small sequential batch", + file_count: 5, + check_existing: true, + existing_identical: &[1], + existing_different: &[3], + expected_written: 4, + expected_unchanged: 1, + }, + WriteBatchTestCase { + description: "large parallel batch", + file_count: 500, + check_existing: true, + existing_identical: &[0, 250, 499], + existing_different: &[10, 400], + expected_written: 497, + expected_unchanged: 3, + }, + WriteBatchTestCase { + description: "parallel batch overwriting without checks", + file_count: 200, + check_existing: false, + existing_identical: &[5], + existing_different: &[6], + expected_written: 200, + expected_unchanged: 0, + }, + WriteBatchTestCase { + description: "parallel batch into an empty target", + file_count: 64, + check_existing: true, + existing_identical: &[], + existing_different: &[], + expected_written: 64, + expected_unchanged: 0, + }, + ]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("target directory"); + let files = batch(root.path(), test_case.file_count); + for &index in test_case.existing_identical { + place(&files[index].0, &files[index].1); + } + for &index in test_case.existing_different { + place(&files[index].0, b"stale\n"); + } + let outcome = write_artifacts(&files, test_case.check_existing).expect("written batch"); + assert_eq!( + outcome.written, test_case.expected_written, + "{}", + test_case.description + ); + assert_eq!( + outcome.unchanged, test_case.expected_unchanged, + "{}", + test_case.description + ); + let matching = files + .iter() + .filter(|(path, contents)| std::fs::read(path).ok().as_ref() == Some(contents)) + .count(); + assert_eq!(matching, test_case.file_count, "{}", test_case.description); + } +} + +#[test] +fn given_several_non_utf8_existing_files_when_written_then_the_first_in_batch_order_is_reported() { + let test_cases = [ + FirstFailureTestCase { + description: "sequential batch", + file_count: 10, + failing: &[3, 9], + expected_failing: 3, + }, + FirstFailureTestCase { + description: "parallel batch", + file_count: 300, + failing: &[100, 299], + expected_failing: 100, + }, + ]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("target directory"); + let files = batch(root.path(), test_case.file_count); + for &index in test_case.failing { + place(&files[index].0, &[0xff, 0xfe]); + } + let error = write_artifacts(&files, true).expect_err("non-UTF-8 existing artifact"); + assert!( + matches!( + &error, + ArtifactError::ExistingNotUtf8 { path } if *path == files[test_case.expected_failing].0 + ), + "{}: {error:?}", + test_case.description + ); + } +} + +#[test] +fn given_repeated_artifact_path_when_written_then_the_last_queued_contents_win() { + let test_cases = [RepeatedPathTestCase { + description: "large batch repeating one path", + file_count: 100, + repeated: 2, + expected_contents: b"select 'last'\n", + }]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("target directory"); + let mut files = batch(root.path(), test_case.file_count); + let repeated = files[test_case.repeated].0.clone(); + files.push((repeated.clone(), test_case.expected_contents.to_vec())); + write_artifacts(&files, true).expect("written batch"); + assert_eq!( + std::fs::read(&repeated).expect("repeated artifact"), + test_case.expected_contents, + "{}", + test_case.description + ); + } +} From 66b65492781d0f96d7c2b2a40976fe1e4af1e465 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:05:12 +0100 Subject: [PATCH 18/28] perf: hash batch-wide analysis settings once per batch Shared-result keys and compact-batch template deduplication re-serialised the batch's function return types for every member. Hash the batch-wide settings once and keep them out of the per-member template identity; the request sent to the analysis pool is unchanged. --- .../_helpers/compact_batch.rs | 51 ++++++++++++------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs index 3092851426..974c90e824 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs @@ -165,6 +165,7 @@ impl SessionCatalog { let mut shared: Vec> = self.shared_queries(batch, &cleaned)?; self.shared_members += shared.iter().flatten().count(); let mut query_digests: HashMap<&str, MemoKey> = HashMap::new(); + let batch_digest: MemoKey = batch_key_digest(batch); let keys: Vec> = batch .members .iter() @@ -174,7 +175,12 @@ impl SessionCatalog { let query: MemoKey = *query_digests .entry(share.key.as_str()) .or_insert_with(|| Sha256::digest(share.key.as_bytes()).into()); - Some(shared_result_key(batch, member, share, &query)) + Some(shared_result_key( + batch, + member, + share, + (&query, &batch_digest), + )) }) .collect(); for (member, sql) in batch.members.iter().zip(&cleaned) { @@ -609,21 +615,22 @@ impl SessionCatalog { } }; query_indexes.push(query_index); - let template: Value = json!({ - "queryIndex": query_index, - "references": lineage_resources(member, share.as_ref()), - "functionReturnTypes": function_return_types, - "declaredColumnOrder": declared_column_order(member, batch.nullability), - "recoverCteFacts": member.recover_cte_facts, - "richTypeInference": batch.rich_type_inference, + let references: Map = lineage_resources(member, share.as_ref()); + let declared: Value = json!(declared_column_order(member, batch.nullability)); + // Batch-wide fields are identical for every member, so they stay out of the dedup key. + let identity: String = + json!([query_index, references, declared, member.recover_cte_facts]).to_string(); + let template_index: usize = *template_keys.entry(identity).or_insert_with(|| { + templates.push(json!({ + "queryIndex": query_index, + "references": references, + "functionReturnTypes": function_return_types, + "declaredColumnOrder": declared, + "recoverCteFacts": member.recover_cte_facts, + "richTypeInference": batch.rich_type_inference, + })); + templates.len() - 1 }); - let template_index: usize = - *template_keys - .entry(template.to_string()) - .or_insert_with(|| { - templates.push(template); - templates.len() - 1 - }); let resource_names: Map = member .lineage_references .iter() @@ -897,12 +904,19 @@ fn named_sources( .collect() } -/// Digest of Python's `shared_result_keys` entry, given its shared query's digest. +/// Digest of the settings every member of `batch` shares, hashed once per batch. +fn batch_key_digest(batch: &Batch<'_>) -> MemoKey { + let shared: String = + json!([batch.rich_type_inference, batch.function_return_types]).to_string(); + Sha256::digest(shared.as_bytes()).into() +} + +/// Digest of Python's `shared_result_keys` entry, given its shared query's and batch's digests. fn shared_result_key( batch: &Batch<'_>, member: &BatchMember<'_>, share: &SharedQuery, - query: &MemoKey, + (query, batch_digest): (&MemoKey, &MemoKey), ) -> MemoKey { let reference_types: Vec<(&str, &str)> = member .lineage_references @@ -911,14 +925,13 @@ fn shared_result_key( .collect(); let rest: String = json!([ member.recover_cte_facts, - batch.rich_type_inference, - batch.function_return_types, declared_column_order(member, batch.nullability), reference_types, ]) .to_string(); let mut hasher: Sha256 = Sha256::new(); hasher.update(query); + hasher.update(batch_digest); hasher.update(rest.as_bytes()); hasher.finalize().into() } From cbbb3633a1060c5786b3142d11c76d48ab827e31 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:10:48 +0100 Subject: [PATCH 19/28] perf: keep discovered model file data native Model files keep their contents, query, header values and path text as Rust data and build the Python dict, strings and pathlib objects only when Python reads them. Model input records share the native query and expansion texts instead of Python strings, and internal readers use the native fields directly. --- .../src/model_files/main/mod.rs | 1 + .../src/model_files/main/model_file_stem.rs | 9 + .../src/model_files/tests.rs | 2 + .../model_files/tests/test_model_file_stem.rs | 15 + .../bindings/_helpers/analysis/model_reads.rs | 6 +- .../compiled_model_analysis.rs | 24 +- .../compiled_project/compiled_model_view.rs | 46 +- .../compiled_project/model_input_view.rs | 31 +- .../_helpers/compiled_project/model_loop.rs | 26 +- .../compiled_project/model_loop_header.rs | 2 +- .../compiled_project/model_loop_host.rs | 78 ++-- .../_helpers/compiled_project/project.rs | 8 +- .../_helpers/compiled_project/project_view.rs | 4 +- .../compiled_project/resource_diagnostics.rs | 26 +- .../_helpers/discovery/declaration_records.rs | 2 +- .../_helpers/discovery/model_file_record.rs | 416 ++++++++++++++---- .../_helpers/discovery/model_records.rs | 75 +++- .../_helpers/discovery/project_files.rs | 76 ++-- .../_helpers/discovery/sql_test_records.rs | 4 +- .../_helpers/records/getter_readers.rs | 12 +- .../bindings/_helpers/rules/fix_session.rs | 11 +- .../_helpers/rules/fixture_schemas.rs | 8 +- .../bindings/_helpers/rules/lint_inputs.rs | 23 +- .../bindings/_helpers/rules/lint_pipeline.rs | 41 +- .../_helpers/sqltext/authored_values.rs | 37 +- .../_helpers/sqltext/model_headers.rs | 2 +- 26 files changed, 643 insertions(+), 342 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs diff --git a/crates/sqlbuild-discovery/src/model_files/main/mod.rs b/crates/sqlbuild-discovery/src/model_files/main/mod.rs index 278eb30e86..93bf47feb0 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/mod.rs @@ -1,2 +1,3 @@ pub mod discover_model_files; +pub mod model_file_stem; pub mod parse_model_text; diff --git a/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs b/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs new file mode 100644 index 0000000000..e537eba920 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs @@ -0,0 +1,9 @@ +//! A model's name: Python's `Path.stem` of its `/`-separated relative path. + +use crate::declarations::_helpers::paths::stem; + +/// Python's `PurePath(relative_path).stem` for a `/`-separated relative path. +#[must_use] +pub fn model_file_stem(relative_path: &str) -> &str { + stem(relative_path.rsplit('/').next().unwrap_or(relative_path)) +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests.rs b/crates/sqlbuild-discovery/src/model_files/tests.rs index ab952d899c..3e8b418280 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests.rs @@ -2,5 +2,7 @@ mod discover_model_files; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_model_file_stem.rs"] +mod model_file_stem; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs new file mode 100644 index 0000000000..a1db4c4032 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs @@ -0,0 +1,15 @@ +use crate::model_files::main::model_file_stem::model_file_stem; + +#[test] +fn given_relative_paths_when_naming_models_then_stems_match_python() { + let test_cases = [ + ("models/orders.sql", "orders"), + ("models/marts/orders.v2.sql", "orders.v2"), + ("models/.hidden", ".hidden"), + ("orders.", "orders."), + ("orders", "orders"), + ]; + for (relative_path, expected) in test_cases { + assert_eq!(model_file_stem(relative_path), expected, "{relative_path}"); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs index f14f73c13e..e1012641f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs @@ -204,11 +204,7 @@ pub(crate) fn model_name(model: &Bound<'_, PyAny>) -> PyResult { /// The model's compiled query SQL. pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { if let Ok(view) = model.downcast::() { - return Ok(view - .get() - .query_sql(model.py())? - .into_bound(model.py()) - .into_any()); + return Ok(view.get().query_sql(model.py())?.into_any()); } native_field(model, "query_sql") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index 516f4988e8..83db2ce746 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -7,7 +7,7 @@ use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; use sqlbuild_analysis::assembly::analysis_session::models::{ ColumnFact, ContractProof, LineageFacts, LineageRow, ModelOutcome, PivotOutcome, @@ -22,10 +22,10 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ CompiledViewClasses, InferredRow, }; use crate::bindings::_helpers::compiled_project::model_input_view::ModelInputRecord; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::record_behaviours::lineage_facts::{ confidence_member, transform_kind_member, }; -use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_PHASE: &str = "compile"; const MODEL_RESOURCE: &str = "model"; @@ -135,7 +135,7 @@ pub(crate) fn analysis_record( if let PivotOutcome::Proof(proof) = contract { record.proof = Some(proof.clone()); } - record.binding = located(py, input, outcome)?; + record.binding = located(input, outcome)?; record.binding_validated = analysed.binding_validated; } if let Some(proof) = record.proof.as_ref().filter(|proof| proof.output_proven) { @@ -146,31 +146,27 @@ pub(crate) fn analysis_record( } /// The model's binding diagnostics at their authored locations. -fn located( - py: Python<'_>, - input: &ModelInputRecord, - outcome: &ModelOutcome, -) -> PyResult> { +fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult> { let rows: &[DiagnosticRow] = &outcome.analysis.binding_diagnostics; if rows.is_empty() { return Ok(Vec::new()); } let cleaned: Cow<'_, str> = resolved_cleaned_sql(outcome).map_err(compiler_error)?; - let model_file: &Bound<'_, PyAny> = input.model_file.bind(py); - let authored: String = native_field(model_file, "contents")?.extract()?; - let query: String = native_field(model_file, "query_sql")?.extract()?; + let model_file: &NativeModelFile = input.model_file.get(); + let authored: &str = model_file.contents_text(); + let query: &str = model_file.query_text(); let expansion = &input.expansion; let passes: Vec> = vec![ expansion.declaration_spans.clone(), expansion.macro_spans.clone(), ]; let source: BindingSource<'_> = BindingSource { - authored: &authored, - query: &query, + authored, + query, cleaned: &cleaned, expansion: expansion .authored - .then_some((expansion.expanded_sql.bind(py).to_str()?, passes.as_slice())), + .then_some((&*expansion.expanded_sql, passes.as_slice())), }; locate_binding_diagnostics(rows, &source).map_err(PyValueError::new_err) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index 8a60cd1169..2cd45e01a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -23,6 +23,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ }; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; @@ -436,10 +437,12 @@ impl NativeCompiledModel { Ok((facts.authored_sql.clone(), facts.query_sql.clone())) } - fn model_file_attribute(&self, py: Python<'_>, name: &str) -> PyResult> { - self.read(py, |record, py| { - Ok(native_field(record.model_file.bind(py), name)?.unbind()) - }) + fn model_file( + &self, + py: Python<'_>, + read: impl FnOnce(&NativeModelFile) -> PyResult, + ) -> PyResult { + self.read(py, |record, _| read(record.model_file.get())) } } @@ -510,12 +513,12 @@ impl NativeCompiledModel { #[getter] pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "relative_path") + self.model_file(py, |file| file.relative_path(py)) } #[getter] - pub(crate) fn query_sql(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| Ok(record.query_sql.clone_ref(py))) + pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| Ok(PyString::new(py, &record.query_sql))) } #[getter] @@ -710,23 +713,29 @@ impl NativeCompiledModel { } #[getter] - pub(crate) fn authored_sql(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "contents") + pub(crate) fn authored_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.model_file(py, |file| Ok(file.contents(py))) } #[getter] - pub(crate) fn authored_query_sql(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "query_sql") + pub(crate) fn authored_query_sql<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + self.model_file(py, |file| Ok(file.query_sql(py))) } #[getter] - pub(crate) fn output_column_locations(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "output_column_locations") + pub(crate) fn output_column_locations<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + self.model_file(py, |file| file.output_column_locations(py)) } #[getter] - pub(crate) fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "extract_implicit_alias_columns") + pub(crate) fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult { + self.model_file(py, |file| Ok(file.extract_implicit_alias_columns())) } #[getter] @@ -759,7 +768,7 @@ impl NativeCompiledModel { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; self.read(py, |record, py| { let name: String = model_name(record); - let path: Bound<'_, PyAny> = native_field(record.model_file.bind(py), "relative_path")?; + let path: Bound<'_, PyAny> = record.model_file.get().path_objects(py)?.1; compiled_of(record)? .analysis .binding_tuple(py, &classes.compiled, (&name, &path)) @@ -848,10 +857,7 @@ pub(crate) fn model_graph_row( tags.push(tag?.str()?.extract()?); } } - let folder: String = native_field(record.model_file.bind(py), "relative_path")? - .getattr("parent")? - .str()? - .extract()?; + let folder: String = record.model_file.get().folder(py)?; Ok(( (MODEL_RESOURCE.to_owned(), model_name(record)), compiled.deps.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index a7ddf57ddf..41b4f0d39f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::SpanRow; @@ -46,9 +47,9 @@ pub(crate) struct SchemaRecord { /// The expansion passes of a model's query, from authored text to expanded SQL. pub(crate) struct ExpansionRecord { /// The authored query below the header. - pub(crate) authored_sql: Py, + pub(crate) authored_sql: Arc, /// The declaration- and macro-expanded SQL before canonicalisation. - pub(crate) expanded_sql: Py, + pub(crate) expanded_sql: Arc, pub(crate) interpolation_spans: Vec, pub(crate) declaration_spans: Vec, pub(crate) macro_spans: Vec, @@ -61,12 +62,12 @@ pub(crate) struct ModelInputRecord { /// The model's name, its file stem. pub(crate) name: String, /// The discovered model file, owned by the discovery inputs. - pub(crate) model_file: Py, + pub(crate) model_file: Py, /// The resolved config, whose values hold user values. pub(crate) config: Py, - /// Immutable text shared with discovery, expansions and every Python-facing projection. - pub(crate) query_sql: Py, - pub(crate) macro_source_sql: Py, + /// Immutable text shared with discovery and expansions; Python reads build strings on demand. + pub(crate) query_sql: Arc, + pub(crate) macro_source_sql: Arc, pub(crate) references: Vec, pub(crate) schema: Option, pub(crate) sql_validation_enabled: bool, @@ -169,7 +170,9 @@ pub(crate) fn view_classes<'py>( impl NativeModelInput { #[getter] pub(crate) fn model_file(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.model_file.clone_ref(py))) + self.read(py, |record, py| { + Ok(record.model_file.clone_ref(py).into_any()) + }) } #[getter] @@ -178,13 +181,15 @@ impl NativeModelInput { } #[getter] - pub(crate) fn query_sql(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.query_sql.clone_ref(py))) + pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| Ok(PyString::new(py, &record.query_sql))) } #[getter] - pub(crate) fn macro_source_sql(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.macro_source_sql.clone_ref(py))) + pub(crate) fn macro_source_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| { + Ok(PyString::new(py, &record.macro_source_sql)) + }) } #[getter] @@ -444,8 +449,8 @@ fn expansion_object( pass_objects.push(PyTuple::new(py, span_objects)?); } let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("authored_sql", expansion.authored_sql.bind(py))?; - fields.set_item("expanded_sql", expansion.expanded_sql.bind(py))?; + fields.set_item("authored_sql", &*expansion.authored_sql)?; + fields.set_item("expanded_sql", &*expansion.expanded_sql)?; fields.set_item("passes", PyTuple::new(py, pass_objects)?)?; Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index 01f51cf0ee..325f0c319b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -267,14 +267,12 @@ fn prepared_rows<'py>( ) -> PyResult> { let hooks = &classes.sql_hooks; let mut sqls: Vec<(String, String)> = Vec::with_capacity(model_files.len()); + let mut paths: Vec = Vec::with_capacity(model_files.len()); for model_file in &model_files { let record: &NativeModelFile = model_file.get(); - let (file_path, _) = record.paths(py); - let (query_sql, _) = record.texts(py); - sqls.push(( - query_sql.to_str()?.to_owned(), - file_path.str()?.to_str()?.to_owned(), - )); + let texts: ModelPaths = record.path_texts(py)?; + sqls.push((record.query_text().to_string(), texts.file_path.clone())); + paths.push(texts); } let interpolated: Vec = interpolation_batch( InterpolationSources( @@ -286,8 +284,11 @@ fn prepared_rows<'py>( &sqls, )?; let mut rows: Vec = Vec::with_capacity(model_files.len()); - for ((model_file, (authored, _)), (rendered, spans, reads, error)) in - model_files.into_iter().zip(sqls).zip(interpolated) + for (((model_file, paths), (authored, _)), (rendered, spans, reads, error)) in model_files + .into_iter() + .zip(paths) + .zip(sqls) + .zip(interpolated) { let sql: String = rendered.unwrap_or(authored); let scan: Option = error.is_none().then(|| declaration_scan(&sql)); @@ -296,7 +297,7 @@ fn prepared_rows<'py>( .map(|(kind, name)| (kind.to_owned(), name)) .collect(); let interpolation: InterpolationRow = (sql, reads, error, spans); - rows.push(ModelLoopRow::new(py, (model_file, scan, interpolation))?); + rows.push(ModelLoopRow::new((model_file, paths), scan, interpolation)); } Ok(rows) } @@ -342,7 +343,7 @@ impl NativeModelLoop { )?; let config: Bound<'py, PyAny> = configs.effective( &context, - record.header_value_dict(py), + &record.header_value_dict(py)?, (name, relative_path), )?; let model: LoopModel<'_, 'py> = LoopModel { @@ -589,7 +590,7 @@ impl NativeModelLoop { relative_path, contents, authored_query, - authored: &interpolated == authored_query, + authored: interpolated.as_str() == &**authored_query, declaration_spans: &declarations.spliced.spans, }; Some(expanded_macros( @@ -634,7 +635,8 @@ impl NativeModelLoop { let scanned = match extract_sql_references(&query_sql, &self.syntax) { ReferenceExtraction::Extracted(scanned) => scanned, ReferenceExtraction::Failed(failure) => { - let authored: bool = &interpolated == authored_query && query_sql == macro_sql; + let authored: bool = + interpolated.as_str() == &**authored_query && query_sql == macro_sql; let origin: ScanFaultOrigin<'_> = ScanFaultOrigin { relative_path: posix_path, contents, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index 95f14eb56b..d58cd4fe8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -105,7 +105,7 @@ pub(crate) fn header_schema<'py>( .columns .map_err(|error| raised_config_error(py, error))?; let locations: Bound<'py, PyDict> = source.model_file.header_locations(py)?; - let model_path: &Bound<'py, PyAny> = source.model_file.paths(py).1; + let model_path: &Bound<'py, PyAny> = &source.model_file.path_objects(py)?.1; let local_columns: DeclaredColumns = DeclaredColumns { specs: parsed_columns .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 1f01666301..437243e1a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -1,8 +1,10 @@ //! The Python host the loop hands finished models and user-code steps to. +use std::sync::Arc; + use pyo3::FromPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiled_project::main::plain_model_facts::plain_model_facts; use sqlbuild_analysis::compiled_project::models::{ CompiledModelFacts, ModelConfigFacts, PlainModel, @@ -37,31 +39,18 @@ const CONSTANTS_ARGUMENT: &str = "constants"; const LOCAL_ENUMS_ATTRIBUTE: &str = "local_enums"; const LOCAL_CONSTANTS_ATTRIBUTE: &str = "local_constants"; const NAME_ATTRIBUTE: &str = "name"; -const STEM_ATTRIBUTE: &str = "stem"; -const AS_POSIX_METHOD: &str = "as_posix"; -/// What the host hands the loop for one model: its file, declaration scan and interpolation. -pub(crate) type ModelLoopInput = ( - Py, - Option, - InterpolationRow, -); /// A model's scanned declaration references and the scan's stop marker. pub(crate) type DeclarationScan = (Vec, Option); /// The names a model's file is known by. -pub(crate) struct ModelPaths { - pub(crate) name: String, - pub(crate) posix_path: String, - pub(crate) relative_path: String, - pub(crate) file_path: String, -} +pub(crate) use crate::bindings::_helpers::discovery::model_file_record::ModelFileTexts as ModelPaths; /// One model the loop builds: its discovered file and what the host prepared for it. pub(crate) struct ModelLoopRow { pub(crate) paths: ModelPaths, - pub(crate) authored_query: String, - pub(crate) contents: String, + pub(crate) authored_query: Arc, + pub(crate) contents: Arc, pub(crate) scan: Option, /// The `@@`-interpolated query, its template reads, its error, and its substitution spans. pub(crate) interpolation: InterpolationRow, @@ -70,24 +59,21 @@ pub(crate) struct ModelLoopRow { } impl ModelLoopRow { - pub(crate) fn new(py: Python<'_>, input: ModelLoopInput) -> PyResult { - let (model_file, scan, interpolation) = input; + pub(crate) fn new( + file: (Py, ModelPaths), + scan: Option, + interpolation: InterpolationRow, + ) -> Self { + let (model_file, paths) = file; let record: &NativeModelFile = model_file.get(); - let (file_path, relative_path) = record.paths(py); - let (query_sql, contents) = record.texts(py); - Ok(Self { - paths: ModelPaths { - name: native_field(file_path, STEM_ATTRIBUTE)?.extract()?, - posix_path: relative_path.call_method0(AS_POSIX_METHOD)?.extract()?, - relative_path: relative_path.str()?.to_str()?.to_owned(), - file_path: file_path.str()?.to_str()?.to_owned(), - }, - authored_query: query_sql.to_str()?.to_owned(), - contents: contents.to_str()?.to_owned(), + Self { + paths, + authored_query: record.query_text().clone(), + contents: record.contents_text().clone(), scan, interpolation, model_file, - }) + } } } /// `(interpolated query, template reads, error, substitution spans)`. @@ -224,19 +210,13 @@ pub(crate) fn finished_record<'py>( type_enforcement: schema.type_enforcement, model_schema: schema.model_schema.clone(), }); - let discovered_query: Py = model_file.get().texts(py).0.clone().unbind(); - let authored: Py = shared_sql(py, &authored_query_sql, &[&discovered_query])?; - let macro_source: Py = - shared_sql(py, &finished.declarations.spliced.sql, &[&authored])?; - let expanded: Py = shared_sql(py, ¯o_sql, &[&authored, ¯o_source])?; - let query: Py = shared_sql( - py, - &finished.query_sql, - &[&authored, ¯o_source, &expanded], - )?; + let authored: Arc = authored_query_sql.clone(); + let macro_source: Arc = shared_sql(&finished.declarations.spliced.sql, &[&authored]); + let expanded: Arc = shared_sql(¯o_sql, &[&authored, ¯o_source]); + let query: Arc = shared_sql(&finished.query_sql, &[&authored, ¯o_source, &expanded]); let record: ModelInputRecord = ModelInputRecord { name: name.clone(), - model_file: model_file.clone_ref(py).into_any(), + model_file, config: config.unbind(), query_sql: query, macro_source_sql: macro_source, @@ -256,7 +236,7 @@ pub(crate) fn finished_record<'py>( macro_usages: PyTuple::new(py, macro_usages)?.unbind(), declaration_usages: PyTuple::new(py, unique(py, declaration_usages)?)?.unbind(), expansion: ExpansionRecord { - authored: interpolated_sql == authored_query_sql, + authored: *interpolated_sql == *authored_query_sql, authored_sql: authored, expanded_sql: expanded, interpolation_spans, @@ -271,8 +251,8 @@ pub(crate) fn finished_record<'py>( name, relative_path, query_sql: finished.query_sql, - authored_query_sql, - authored_sql, + authored_query_sql: authored_query_sql.to_string(), + authored_sql: authored_sql.to_string(), config: ModelConfigFacts { values, encoding_error, @@ -296,13 +276,13 @@ pub(crate) fn finished_record<'py>( } /// Reuse an identical immutable spelling already owned by this model. -fn shared_sql(py: Python<'_>, sql: &str, candidates: &[&Py]) -> PyResult> { +fn shared_sql(sql: &str, candidates: &[&Arc]) -> Arc { for candidate in candidates { - if candidate.bind(py).to_str()? == sql { - return Ok(candidate.clone_ref(py)); + if ***candidate == *sql { + return Arc::clone(candidate); } } - Ok(PyString::new(py, sql).unbind()) + Arc::from(sql) } /// A dict's values in insertion order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index ff757ea087..0f0ebb8f48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -162,9 +162,7 @@ impl NativeCompiledProject { }; owners.push(model_owner( record.name.clone(), - native_field(record.model_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? - .str()? - .to_string(), + record.model_file.get().relative_text(py)?, ( schema.audits.bind(py), (schema.columns.as_ref(), &fact.columns), @@ -443,9 +441,7 @@ impl NativeCompiledProject { .iter() .map(|record| { Ok(( - native_field(record.model_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? - .str()? - .to_string(), + record.model_file.get().relative_text(py)?, record.references.clone(), )) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index 846e81a8c6..a7bf443dcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -25,7 +25,6 @@ use crate::bindings::_helpers::records::fields::field as native_field; const MODELS_FIELD: &str = "models"; const RUN_ID_FIELD: &str = "run_id"; const ENFORCE_EXPLICIT_FIELD: &str = "enforce_explicit_references"; -const FILE_PATH_ATTRIBUTE: &str = "file_path"; const KEY_ATTRIBUTE: &str = "key"; const DEPS_ATTRIBUTE: &str = "deps"; const TAGS_ATTRIBUTE: &str = "tags"; @@ -504,8 +503,7 @@ impl NativeProjectView { for record in owner.inputs() { let expansion: Py = authored_expansion(&classes, record)?; if !expansion.is_none(py) { - let path: Bound<'_, PyAny> = - native_field(record.model_file.bind(py), FILE_PATH_ATTRIBUTE)?; + let path: Bound<'_, PyAny> = record.model_file.get().path_objects(py)?.0; expansions.set_item(path, expansion)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index 8d0aeef213..efd61f55c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -22,6 +22,7 @@ use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::records::fields::field as native_field; const MISSING_DESCRIPTION_CODE: &str = "P010"; @@ -201,6 +202,9 @@ pub(crate) fn resource_diagnostics<'py>( fn orphaned_factories<'py>(diagnostics: &Diagnostics<'_, 'py>) -> PyResult>> { let py: Python<'py> = diagnostics.py; + if diagnostics.settings.audit_factories.is_empty() { + return Ok(Vec::new()); + } let mut referenced: HashSet = HashSet::new(); for model_file in &diagnostics.settings.model_files { let header: Bound<'py, PyDict> = @@ -249,12 +253,10 @@ fn missing_descriptions<'py>( let py: Python<'py> = diagnostics.py; let mut found: Vec>> = Vec::new(); for record in project.inputs() { - let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); + let model_file: &NativeModelFile = record.model_file.get(); let name: String = record.name.clone(); - let path: String = native_field(model_file, "relative_path")? - .str()? - .extract()?; - let contents: String = native_field(model_file, "contents")?.extract()?; + let path: String = model_file.relative_text(py)?; + let contents: &str = model_file.contents_text(); let entry: String = format!("description \"What one row of {name} represents\""); let help: String = header_help(&HeaderHelp { kind: "model", @@ -271,7 +273,7 @@ fn missing_descriptions<'py>( found.push(diagnostics.described( ("model", &name, Some("model")), description, - || diagnostics.header_location(&path, &contents), + || diagnostics.header_location(&path, contents), &help, )?); } @@ -487,7 +489,7 @@ fn cursor_without_inputs<'py>( let materialized: Option = config_text(&values, "materialized")?; let cursor: Option = config_text(&values, CURSOR_KEY)?; let strategy: Option = config_text(&values, "microbatch_strategy")?; - let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); + let model_file: &NativeModelFile = record.model_file.get(); let name: String = record.name.clone(); let Some((message, help)) = cursor_lacks_inputs( &name, @@ -503,13 +505,11 @@ fn cursor_without_inputs<'py>( ) else { continue; }; - let path: String = native_field(model_file, "relative_path")? - .str()? - .extract()?; - let contents: String = native_field(model_file, "contents")?.extract()?; + let path: String = model_file.relative_text(py)?; + let contents: &str = model_file.contents_text(); let span: Option<(usize, usize)> = - model_header_entry_span(&contents, CURSOR_KEY).map_err(PyValueError::new_err)?; - let start: TextPosition = text_position(&contents, span.map_or(0, |span| span.0)); + model_header_entry_span(contents, CURSOR_KEY).map_err(PyValueError::new_err)?; + let start: TextPosition = text_position(contents, span.map_or(0, |span| span.0)); let location: Bound<'py, PyAny> = diagnostics.location( &path, (start.line, start.column), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 1d558d435e..8c3c3aa0d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -110,7 +110,7 @@ fn authored<'py>( py: Python<'py>, values: &[(String, AuthoredValue)], ) -> PyResult> { - Ok(map_to_python(py, values.to_vec())? + Ok(map_to_python(py, values)? .into_bound(py) .downcast_into::()?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index 9bf2858958..5410103050 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -1,15 +1,21 @@ //! A discovered SQL model file as a native record whose Python attributes are built on demand. +use std::ffi::OsString; use std::path::PathBuf; +use std::sync::Arc; use crate::bindings::_helpers::records::construct::new_record; +use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use pyo3::exceptions::PyTypeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_sqltext::compiler::models::AuthoredValue; const FIELD_NAMES: [&str; 10] = [ "file_path", @@ -23,6 +29,8 @@ const FIELD_NAMES: [&str; 10] = [ "constant_declarations", "extract_implicit_alias_columns", ]; +const STEM_ATTRIBUTE: &str = "stem"; +const AS_POSIX_METHOD: &str = "as_posix"; /// Authored locations: native rows until Python reads them, then a cached dict. struct Locations { @@ -60,7 +68,8 @@ impl Locations { } /// Python's `{name: SourceLocation(...)}`: a repeated name keeps its first position and last value. - fn dict<'py>(&self, py: Python<'py>, path: &Py) -> PyResult> { + fn dict<'py>(&self, path: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = path.py(); Ok(self .projected .get_or_try_init(py, || { @@ -69,13 +78,7 @@ impl Locations { let location: Bound<'py, PyAny> = new_record( py, RecordId::SourceLocation, - ( - path.bind(py), - span.line, - span.column, - span.end_line, - span.end_column, - ), + (path, span.line, span.column, span.end_line, span.end_column), None, )?; dict.set_item(name, location)?; @@ -87,36 +90,254 @@ impl Locations { } } +/// The names a model file is known by, as Python's `pathlib` spells them. +#[derive(Clone)] +pub(crate) struct ModelFileTexts { + /// `file_path.stem`, the model's name. + pub(crate) name: String, + /// `relative_path.as_posix()`. + pub(crate) posix_path: String, + /// `str(relative_path)`. + pub(crate) relative_path: String, + /// `str(file_path)`. + pub(crate) file_path: String, +} + +/// A model file's paths: native text, with the Python path objects built on first read. +struct ModelFilePaths { + native_file_path: PathBuf, + /// `None` when a path is not text Python can encode, so readers ask Python as they always did. + texts: Option, + /// `(file_path, relative_path)` as Python path objects. + objects: PyOnceLock<(Py, Py)>, + /// The project directory and relative path the objects are built from, unless given. + origin: Option<(Py, OsString)>, +} + +impl ModelFilePaths { + /// Paths given as Python objects; their text is read once, here. + fn given(py: Python<'_>, file_path: Py, relative_path: Py) -> PyResult { + let native_file_path: PathBuf = file_path.bind(py).extract()?; + let texts: Option = + python_texts(file_path.bind(py), relative_path.bind(py)).ok(); + let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); + let _ = objects.set(py, (file_path, relative_path)); + Ok(Self { + native_file_path, + texts, + objects, + origin: None, + }) + } + + /// Paths native discovery named; Python builds `project_dir / relative_path` only when read. + fn discovered(project_dir: Py, relative_path: &str, file_path: String) -> Self { + let name: String = model_file_stem(relative_path).to_owned(); + Self { + native_file_path: PathBuf::from(&file_path), + texts: Some(ModelFileTexts { + name, + posix_path: relative_path.to_owned(), + relative_path: relative_path.replace('/', std::path::MAIN_SEPARATOR_STR), + file_path, + }), + objects: PyOnceLock::new(), + origin: Some((project_dir, OsString::from(relative_path))), + } + } + + fn shared(&self, py: Python<'_>) -> Self { + let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); + if let Some((file_path, relative_path)) = self.objects.get(py) { + let _ = objects.set(py, (file_path.clone_ref(py), relative_path.clone_ref(py))); + } + Self { + native_file_path: self.native_file_path.clone(), + texts: self.texts.clone(), + objects, + origin: self + .origin + .as_ref() + .map(|(project_dir, relative)| (project_dir.clone_ref(py), relative.clone())), + } + } + + /// `(file_path, relative_path)` as the Python path objects discovery names the file by. + fn objects<'py>(&self, py: Python<'py>) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let (file_path, relative_path) = self.objects.get_or_try_init(py, || { + let Some((project_dir, relative)) = &self.origin else { + return Err(pyo3::exceptions::PyRuntimeError::new_err( + "a model file has no paths", + )); + }; + let relative_path: Bound<'_, PyAny> = + py.import("pathlib")?.getattr("Path")?.call1((relative,))?; + let file_path: Bound<'_, PyAny> = project_dir.bind(py).div(&relative_path)?; + Ok::<_, pyo3::PyErr>((file_path.unbind(), relative_path.unbind())) + })?; + Ok((file_path.bind(py).clone(), relative_path.bind(py).clone())) + } +} + +/// The names of a model file read from its Python path objects. +fn python_texts( + file_path: &Bound<'_, PyAny>, + relative_path: &Bound<'_, PyAny>, +) -> PyResult { + let file_text: String = file_path.str()?.to_str()?.to_owned(); + Ok(ModelFileTexts { + name: native_field(file_path, STEM_ATTRIBUTE)?.extract()?, + posix_path: relative_path.call_method0(AS_POSIX_METHOD)?.extract()?, + relative_path: relative_path.str()?.to_str()?.to_owned(), + file_path: file_text, + }) +} + +/// A model header's authored values: native entries, with the Python dict built on first read. +struct HeaderValues { + native: Option>, + projected: PyOnceLock>, +} + +impl HeaderValues { + fn native(values: Vec<(String, AuthoredValue)>) -> Self { + Self { + native: Some(values.into()), + projected: PyOnceLock::new(), + } + } + + fn given(py: Python<'_>, values: Py) -> Self { + let projected: PyOnceLock> = PyOnceLock::new(); + let _ = projected.set(py, values); + Self { + native: None, + projected, + } + } + + fn shared(&self, py: Python<'_>) -> Self { + let projected: PyOnceLock> = PyOnceLock::new(); + if let Some(dict) = self.projected.get(py) { + let _ = projected.set(py, dict.clone_ref(py)); + } + Self { + native: self.native.clone(), + projected, + } + } + + fn dict<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(self + .projected + .get_or_try_init(py, || { + let values: &[(String, AuthoredValue)] = self.native.as_deref().unwrap_or_default(); + map_to_python(py, values)? + .into_bound(py) + .downcast_into::() + .map(Bound::unbind) + .map_err(pyo3::PyErr::from) + })? + .bind(py) + .clone()) + } +} + /// A discovered SQL model file and its raw contents. #[pyclass(module = "sqlbuild._native", name = "DiscoveredSqlModelFile", frozen)] pub(crate) struct NativeModelFile { - file_path: Py, - /// `file_path` as a native path, read once so hot paths need not call back into `pathlib`. - native_file_path: PathBuf, - relative_path: Py, - contents: Py, - header_values: Py, + paths: ModelFilePaths, + contents: Arc, + header_values: HeaderValues, header_column_locations: Locations, output_column_locations: Locations, - query_sql: Py, + query_sql: Arc, enum_declarations: Py, constant_declarations: Py, extract_implicit_alias_columns: bool, } -/// What native discovery read for one model file, before its locations are projected. +/// What native discovery read for one model file. pub(crate) struct ModelFileParts { - pub(crate) file_path: Py, - pub(crate) relative_path: Py, - pub(crate) contents: Py, - pub(crate) header_values: Py, + pub(crate) contents: String, + pub(crate) header_values: Vec<(String, AuthoredValue)>, pub(crate) header_column_locations: Vec<(String, LineColumnSpan)>, pub(crate) output_column_locations: Vec<(String, LineColumnSpan)>, - pub(crate) query_sql: Py, - pub(crate) declarations: (Py, Py), + pub(crate) query_sql: String, pub(crate) extract_implicit_alias_columns: bool, } +/// How discovery names a model file: natively, or by Python paths for names Python escapes. +pub(crate) enum DiscoveredPaths { + /// The project directory, the `/`-separated relative path and `str(project_dir / relative)`. + Native(Py, String, String), + /// `(file_path, relative_path)` built by Python. + Python(Py, Py), +} + +/// A model file before its header declarations are attached. +pub(crate) struct PendingModelFile { + paths: ModelFilePaths, + parts: ModelFileParts, +} + +impl PendingModelFile { + pub(crate) fn new( + py: Python<'_>, + paths: DiscoveredPaths, + parts: ModelFileParts, + ) -> PyResult { + let paths: ModelFilePaths = match paths { + DiscoveredPaths::Native(project_dir, relative_path, file_path) => { + ModelFilePaths::discovered(project_dir, &relative_path, file_path) + } + DiscoveredPaths::Python(file_path, relative_path) => { + ModelFilePaths::given(py, file_path, relative_path)? + } + }; + Ok(Self { paths, parts }) + } + + /// The model's name, its file stem. + pub(crate) fn name(&self, py: Python<'_>) -> PyResult { + match &self.paths.texts { + Some(texts) => Ok(texts.name.clone()), + None => native_field(&self.paths.objects(py)?.0, STEM_ATTRIBUTE)?.extract(), + } + } + + /// The relative path as the Python path object the record keeps. + pub(crate) fn relative_path<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(self.paths.objects(py)?.1) + } + + /// The authored header value of `key`. + pub(crate) fn header_value(&self, key: &str) -> Option<&AuthoredValue> { + self.parts + .header_values + .iter() + .rev() + .find(|(name, _)| name == key) + .map(|(_, value)| value) + } + + pub(crate) fn finished(self, declarations: (Py, Py)) -> NativeModelFile { + let (enum_declarations, constant_declarations) = declarations; + let parts: ModelFileParts = self.parts; + NativeModelFile { + paths: self.paths, + contents: parts.contents.into(), + header_values: HeaderValues::native(parts.header_values), + header_column_locations: Locations::from_rows(parts.header_column_locations), + output_column_locations: Locations::from_rows(parts.output_column_locations), + query_sql: parts.query_sql.into(), + enum_declarations, + constant_declarations, + extract_implicit_alias_columns: parts.extract_implicit_alias_columns, + } + } +} + /// The keyword fields Python constructs a model file record with. #[derive(FromPyObject)] #[pyo3(from_item_all)] @@ -137,29 +358,19 @@ struct ModelFileFields { } impl NativeModelFile { - pub(crate) fn discovered(py: Python<'_>, parts: ModelFileParts) -> PyResult { - let (enum_declarations, constant_declarations) = parts.declarations; - Ok(Self { - native_file_path: parts.file_path.bind(py).extract()?, - file_path: parts.file_path, - relative_path: parts.relative_path, - contents: parts.contents, - header_values: parts.header_values, - header_column_locations: Locations::from_rows(parts.header_column_locations), - output_column_locations: Locations::from_rows(parts.output_column_locations), - query_sql: parts.query_sql, - enum_declarations, - constant_declarations, - extract_implicit_alias_columns: parts.extract_implicit_alias_columns, - }) + pub(crate) fn header_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_column_locations + .dict(&self.paths.objects(py)?.1) } - pub(crate) fn header_locations<'py>(&self, py: Python<'py>) -> PyResult> { - self.header_column_locations.dict(py, &self.relative_path) + /// The header's authored values as the Python dict config builds read. + pub(crate) fn header_value_dict<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_values.dict(py) } - pub(crate) fn header_value_dict<'py>(&self, py: Python<'py>) -> &Bound<'py, PyDict> { - self.header_values.bind(py) + /// The header's native authored values, unless Python constructed this record. + pub(crate) fn native_header_values(&self) -> Option<&Arc<[(String, AuthoredValue)]>> { + self.header_values.native.as_ref() } pub(crate) fn declaration_tuples<'py>( @@ -174,31 +385,77 @@ impl NativeModelFile { /// The file path as a native path. pub(crate) fn file_path_buf(&self) -> PathBuf { - self.native_file_path.clone() + self.paths.native_file_path.clone() } - pub(crate) fn paths<'py>(&self, py: Python<'py>) -> (&Bound<'py, PyAny>, &Bound<'py, PyAny>) { - (self.file_path.bind(py), self.relative_path.bind(py)) + /// The file's names, read through Python only for paths Python escapes. + pub(crate) fn path_texts(&self, py: Python<'_>) -> PyResult { + match &self.paths.texts { + Some(texts) => Ok(texts.clone()), + None => { + let (file_path, relative_path) = self.paths.objects(py)?; + python_texts(&file_path, &relative_path) + } + } } - pub(crate) fn texts<'py>( + /// `str(relative_path)`. + pub(crate) fn relative_text(&self, py: Python<'_>) -> PyResult { + match &self.paths.texts { + Some(texts) => Ok(texts.relative_path.clone()), + None => Ok(self.paths.objects(py)?.1.str()?.to_string()), + } + } + + /// `str(relative_path.parent)`, the folder project graphs group the model by. + pub(crate) fn folder(&self, py: Python<'_>) -> PyResult { + match &self.paths.texts { + Some(texts) => Ok(texts + .relative_path + .rsplit_once(std::path::MAIN_SEPARATOR) + .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned())), + None => self + .paths + .objects(py)? + .1 + .getattr("parent")? + .str()? + .extract(), + } + } + + /// `(file_path, relative_path)` as Python path objects. + pub(crate) fn path_objects<'py>( &self, py: Python<'py>, - ) -> (&Bound<'py, PyString>, &Bound<'py, PyString>) { - (self.query_sql.bind(py), self.contents.bind(py)) + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + self.paths.objects(py) + } + + /// The authored query below the header. + pub(crate) fn query_text(&self) -> &Arc { + &self.query_sql + } + + /// The whole authored file. + pub(crate) fn contents_text(&self) -> &Arc { + &self.contents } fn fields<'py>(&self, py: Python<'py>) -> PyResult>> { + let (file_path, relative_path) = self.paths.objects(py)?; Ok(vec![ - self.file_path.bind(py).clone(), - self.relative_path.bind(py).clone(), - self.contents.bind(py).clone().into_any(), - self.header_values.bind(py).clone().into_any(), - self.header_locations(py)?.into_any(), + file_path, + relative_path.clone(), + PyString::new(py, &self.contents).into_any(), + self.header_values.dict(py)?.into_any(), + self.header_column_locations + .dict(&relative_path)? + .into_any(), self.output_column_locations - .dict(py, &self.relative_path)? + .dict(&relative_path)? .into_any(), - self.query_sql.bind(py).clone().into_any(), + PyString::new(py, &self.query_sql).into_any(), self.enum_declarations.bind(py).clone().into_any(), self.constant_declarations.bind(py).clone().into_any(), pyo3::types::PyBool::new(py, self.extract_implicit_alias_columns) @@ -228,14 +485,12 @@ impl NativeModelFile { } let fields: ModelFileFields = fields.extract()?; Ok(Self { - native_file_path: fields.file_path.bind(py).extract()?, - file_path: fields.file_path, - relative_path: fields.relative_path, - contents: fields.contents, - header_values: fields.header_values, + paths: ModelFilePaths::given(py, fields.file_path, fields.relative_path)?, + contents: fields.contents.bind(py).to_str()?.into(), + header_values: HeaderValues::given(py, fields.header_values), header_column_locations: Locations::from_dict(py, fields.header_column_locations), output_column_locations: Locations::from_dict(py, fields.output_column_locations), - query_sql: fields.query_sql, + query_sql: fields.query_sql.bind(py).to_str()?.into(), enum_declarations: fields .enum_declarations .unwrap_or_else(|| PyTuple::empty(py).unbind()), @@ -252,40 +507,38 @@ impl NativeModelFile { py: Python<'_>, contents: Py, query_sql: Py, - ) -> Self { - Self { - native_file_path: self.native_file_path.clone(), - file_path: self.file_path.clone_ref(py), - relative_path: self.relative_path.clone_ref(py), - contents, - header_values: self.header_values.clone_ref(py), + ) -> PyResult { + Ok(Self { + paths: self.paths.shared(py), + contents: contents.bind(py).to_str()?.into(), + header_values: self.header_values.shared(py), header_column_locations: self.header_column_locations.shared(py), output_column_locations: self.output_column_locations.shared(py), - query_sql, + query_sql: query_sql.bind(py).to_str()?.into(), enum_declarations: self.enum_declarations.clone_ref(py), constant_declarations: self.constant_declarations.clone_ref(py), extract_implicit_alias_columns: self.extract_implicit_alias_columns, - } + }) } #[getter] - pub(crate) fn file_path(&self, py: Python<'_>) -> Py { - self.file_path.clone_ref(py) + pub(crate) fn file_path(&self, py: Python<'_>) -> PyResult> { + Ok(self.paths.objects(py)?.0.unbind()) } #[getter] - pub(crate) fn relative_path(&self, py: Python<'_>) -> Py { - self.relative_path.clone_ref(py) + pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { + Ok(self.paths.objects(py)?.1.unbind()) } #[getter] - pub(crate) fn contents(&self, py: Python<'_>) -> Py { - self.contents.clone_ref(py) + pub(crate) fn contents<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { + PyString::new(py, &self.contents) } #[getter] - pub(crate) fn header_values(&self, py: Python<'_>) -> Py { - self.header_values.clone_ref(py) + pub(crate) fn header_values(&self, py: Python<'_>) -> PyResult> { + Ok(self.header_values.dict(py)?.unbind()) } #[getter] @@ -301,12 +554,13 @@ impl NativeModelFile { &self, py: Python<'py>, ) -> PyResult> { - self.output_column_locations.dict(py, &self.relative_path) + self.output_column_locations + .dict(&self.paths.objects(py)?.1) } #[getter] - pub(crate) fn query_sql(&self, py: Python<'_>) -> Py { - self.query_sql.clone_ref(py) + pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { + PyString::new(py, &self.query_sql) } #[getter] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index 3b11b2612d..af4529c9e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -1,15 +1,17 @@ //! SQL model files discovered natively and built as Python discovery records. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyTuple}; +use pyo3::types::{PyDict, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; +use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::model_file_record::DiscoveredPaths; use crate::bindings::_helpers::discovery::project_files::{model_record, path_object}; use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; @@ -20,8 +22,27 @@ use crate::bindings::models::NativeProjectTree; const PATHLIB_MODULE: &str = "pathlib"; const STEM_ATTRIBUTE: &str = "stem"; -/// One discovered model file: its record, or the error building it raises. -type ModelItem<'py> = (Bound<'py, PyAny>, PyResult>); +/// One discovered model file: its name and record, or the error building it raises. +struct ModelItem<'py> { + name: Option, + relative_path: RelativePath<'py>, + built: PyResult>, +} + +/// A discovered file's relative path: native text, or the Python path already built for it. +enum RelativePath<'py> { + Native(String), + Python(Bound<'py, PyAny>), +} + +impl<'py> RelativePath<'py> { + fn object(&self, py: Python<'py>) -> PyResult> { + match self { + Self::Native(relative) => imported(py, PATHLIB_MODULE, "Path")?.call1((relative,)), + Self::Python(relative) => Ok(relative.clone()), + } + } +} /// Discover every model file natively, each as its record or the error it raises when built. fn model_items<'py>( @@ -41,6 +62,24 @@ fn model_items<'py>( let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; let mut items: Vec> = Vec::with_capacity(files.len()); for file in files { + let raw: bool = tree.get().inner.raw_relative(&file.relative_path).is_some(); + if !raw && let FileOutcome::Parsed(model) = file.outcome { + let name: String = model_file_stem(&file.relative_path).to_owned(); + let file_path: String = root.display_path(&file.relative_path); + let paths = DiscoveredPaths::Native( + project_dir.clone().unbind(), + file.relative_path.clone(), + file_path, + ); + let built = model_record(py, model, paths, options.extract_implicit_alias_columns)? + .map(|record| record.into_bound(py)); + items.push(ModelItem { + name: Some(name), + relative_path: RelativePath::Native(file.relative_path), + built, + }); + continue; + } let relative_path: Bound<'py, PyAny> = path_class.call1((path_object(py, &tree.get().inner, file.relative_path)?,))?; let file_path: Bound<'py, PyAny> = project_dir.div(&relative_path)?; @@ -48,14 +87,18 @@ fn model_items<'py>( FileOutcome::Parsed(model) => model_record( py, model, - (file_path, relative_path.clone()), + DiscoveredPaths::Python(file_path.unbind(), relative_path.clone().unbind()), options.extract_implicit_alias_columns, )? .map(|record| record.into_bound(py)), FileOutcome::Unreadable(failure) => Err(read_error(failure, &file_path)), FileOutcome::Failed(failure) => Err(failure_error(py, failure)), }; - items.push((relative_path, built)); + items.push(ModelItem { + name: None, + relative_path: RelativePath::Python(relative_path), + built, + }); } Ok(items) }) @@ -90,13 +133,23 @@ pub(crate) fn discovered_model_files<'py>( .selected_model_names .filter(|selected| !selected.is_none()); let mut records: Vec> = Vec::with_capacity(items.len()); - for (relative_path, built) in items { - if let Some(selected) = &selected - && !selected.contains(native_field(&relative_path, STEM_ATTRIBUTE)?)? - { - continue; + for item in items { + if let Some(selected) = &selected { + let name: Bound<'py, PyAny> = match &item.name { + Some(name) => PyString::new(py, name).into_any(), + None => native_field(&item.relative_path.object(py)?, STEM_ATTRIBUTE)?, + }; + if !selected.contains(name)? { + continue; + } + } + match item.built { + Ok(record) => records.push(record), + Err(error) => { + let relative_path: Bound<'py, PyAny> = item.relative_path.object(py)?; + records.extend(faults.kept(&relative_path, Err(error))?); + } } - records.extend(faults.kept(&relative_path, built)?); } record_tuple(py, records) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index ab49541661..c7d5e780bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -4,7 +4,7 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::types::PyTuple; use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; @@ -12,19 +12,19 @@ use sqlbuild_discovery::tree::models::ProjectTree; use std::path::Path; use crate::bindings::_helpers::discovery::declaration_records::source_locations; -use crate::bindings::_helpers::discovery::header_declarations::{ - HeaderDeclarations, model_header_declarations, +use crate::bindings::_helpers::discovery::header_declarations::model_header_declarations; +use crate::bindings::_helpers::discovery::model_file_record::{ + DiscoveredPaths, ModelFileParts, NativeModelFile, PendingModelFile, }; -use crate::bindings::_helpers::discovery::model_file_record::{ModelFileParts, NativeModelFile}; use crate::bindings::_helpers::discovery::raised::failure_error; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::sqltext::authored_values::map_to_python; +use crate::bindings::_helpers::sqltext::authored_values::{ + map_to_python, optional_authored_value_to_python, +}; use crate::bindings::models::{ModelTextRequest, NativeProjectTree}; use crate::bindings::types::CompilerDetach; pub(crate) type PyObject = Py; -const STEM_ATTRIBUTE: &str = "stem"; const ENUMS_KEY: &str = "enums"; const CONSTANTS_KEY: &str = "constants"; @@ -51,46 +51,46 @@ impl NativeProjectTree { } /// One parsed model file's record, or its header declaration error, raised only if it is built. -pub(crate) fn model_record<'py>( - py: Python<'py>, +pub(crate) fn model_record( + py: Python<'_>, model: DiscoveredModelFile, - paths: (Bound<'py, PyAny>, Bound<'py, PyAny>), + paths: DiscoveredPaths, extract_implicit_alias_columns: bool, ) -> PyResult> { - let (file_path, relative_path) = paths; - let header_values: Bound<'py, PyDict> = map_to_python(py, model.header_values)? - .into_bound(py) - .downcast_into::()?; - let model_name: String = native_field(&file_path, STEM_ATTRIBUTE)?.extract()?; - let declarations: HeaderDeclarations<'py> = match model_header_declarations( - py, - ( - header_values.get_item(ENUMS_KEY)?, - header_values.get_item(CONSTANTS_KEY)?, - ), - model_name, - relative_path.clone(), - ) { - Ok(declarations) => declarations, - Err(error) if error.is_instance_of::(py) => return Ok(Err(error)), - Err(error) => return Err(error), - }; - let (enum_declarations, constant_declarations) = declarations; - let record: NativeModelFile = NativeModelFile::discovered( + let pending: PendingModelFile = PendingModelFile::new( py, + paths, ModelFileParts { - file_path: file_path.unbind(), - relative_path: relative_path.unbind(), - contents: PyString::new(py, &model.contents).unbind(), - header_values: header_values.unbind(), + contents: model.contents, + header_values: model.header_values, header_column_locations: model.header_column_locations, output_column_locations: model.output_column_locations, - query_sql: PyString::new(py, &model.query_sql).unbind(), - declarations: (enum_declarations.unbind(), constant_declarations.unbind()), + query_sql: model.query_sql, extract_implicit_alias_columns, }, )?; - Ok(Ok(record.into_py_any(py)?)) + let authored = |key: &str| -> PyResult>> { + Ok( + optional_authored_value_to_python(py, pending.header_value(key))? + .map(|value| value.into_bound(py)), + ) + }; + let (enums, constants) = (authored(ENUMS_KEY)?, authored(CONSTANTS_KEY)?); + let declarations: (Py, Py) = if enums.is_none() && constants.is_none() { + (PyTuple::empty(py).unbind(), PyTuple::empty(py).unbind()) + } else { + match model_header_declarations( + py, + (enums, constants), + pending.name(py)?, + pending.relative_path(py)?, + ) { + Ok((enums, constants)) => (enums.unbind(), constants.unbind()), + Err(error) if error.is_instance_of::(py) => return Ok(Err(error)), + Err(error) => return Err(error), + } + }; + Ok(Ok(pending.finished(declarations).into_py_any(py)?)) } /// Parse model contents as discovery does: `(header_values, query_sql, column locations)`. @@ -115,7 +115,7 @@ fn parse_model_contents( .getattr("Path")? .call1((request.file_path.as_str(),))?; Ok(( - map_to_python(py, model.header_values)?, + map_to_python(py, &model.header_values)?, model.query_sql, source_locations(py, &path, &model.header_column_locations)? .into_any() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index 7fcbca43fa..4ae7d8acca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -98,7 +98,7 @@ fn test_blocks<'py>( let mut native = Vec::with_capacity(blocks.len()); for block in blocks { native.push(( - map_to_python(py, block.header_values)? + map_to_python(py, &block.header_values)? .into_bound(py) .downcast_into::()?, block.sql_body, @@ -154,7 +154,7 @@ fn scenario_file<'py>( ) -> PyResult> { let (file_path, relative_path) = paths; let py: Python<'py> = file_path.py(); - let header_values: Bound<'py, PyDict> = map_to_python(py, file.header_values)? + let header_values: Bound<'py, PyDict> = map_to_python(py, &file.header_values)? .into_bound(py) .downcast_into::()?; let shown: Bound<'py, PyString> = file_path.str()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/getter_readers.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/getter_readers.rs index c2316a7d5e..8b2d3f11e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/getter_readers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/getter_readers.rs @@ -1097,10 +1097,16 @@ pub(crate) fn read_native_model_file<'py>( return None; }; Some(match name { - "file_path" => this.file_path(py).into_bound_py_any(py), - "relative_path" => this.relative_path(py).into_bound_py_any(py), + "file_path" => this + .file_path(py) + .and_then(|value| value.into_bound_py_any(py)), + "relative_path" => this + .relative_path(py) + .and_then(|value| value.into_bound_py_any(py)), "contents" => this.contents(py).into_bound_py_any(py), - "header_values" => this.header_values(py).into_bound_py_any(py), + "header_values" => this + .header_values(py) + .and_then(|value| value.into_bound_py_any(py)), "header_column_locations" => this .header_column_locations(py) .and_then(|value| value.into_bound_py_any(py)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index dc31b364be..03d70f111e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -501,7 +501,7 @@ fn replace<'py>( /// A discovered model file's contents, read in place from a native model file. fn model_contents<'py>(model: &Bound<'py, PyAny>) -> PyResult> { match model.cast::() { - Ok(native) => Ok(native.get().texts(model.py()).1.clone().into_any()), + Ok(native) => Ok(native.get().contents(model.py()).into_any()), Err(_) => native_field(model, "contents"), } } @@ -509,7 +509,7 @@ fn model_contents<'py>(model: &Bound<'py, PyAny>) -> PyResult> /// A discovered model file's path, read in place from a native model file. fn model_file_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { match model.cast::() { - Ok(native) => Ok(native.get().paths(model.py()).0.clone()), + Ok(native) => Ok(native.get().path_objects(model.py())?.0), Err(_) => native_field(model, "file_path"), } } @@ -534,9 +534,8 @@ fn edited_model<'py>( return model.call_method1("with_text", (text, query)); }; let native: &NativeModelFile = native.get(); - let (query_sql, contents) = native.texts(py); - let contents: String = contents.to_str()?.to_owned(); - let Some(offset) = contents.find(query_sql.to_str()?) else { + let (query_sql, contents): (&str, &str) = (native.query_text(), native.contents_text()); + let Some(offset) = contents.find(query_sql) else { return Err(pyo3::exceptions::PyValueError::new_err( "substring not found", )); @@ -549,7 +548,7 @@ fn edited_model<'py>( .nth(start) .map_or("", |(index, _)| &edited[index..]); let query: Bound<'py, PyString> = PyString::new(py, python_strip(tail)); - Ok(Bound::new(py, native.with_text(py, text.unbind(), query.unbind()))?.into_any()) + Ok(Bound::new(py, native.with_text(py, text.unbind(), query.unbind())?)?.into_any()) } fn updated_inputs<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs index 1c60070c08..9d21f0f735 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs @@ -41,11 +41,11 @@ fn model_file_view<'py>(model: &Bound<'py, PyAny>) -> PyResult() { let native: &NativeModelFile = native.get(); - let (file_path, relative_path) = native.paths(py); + let (file_path, relative_path) = native.path_objects(py)?; return Ok(ModelFileView { - file_path: file_path.clone(), - relative_path: relative_path.clone(), - header_values: native.header_value_dict(py).clone(), + file_path, + relative_path, + header_values: native.header_value_dict(py)?, header_column_locations: native.header_locations(py)?, }); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index e66134584b..7e6a2607ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::path::PathBuf; +use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -10,7 +11,6 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ ExpansionRecord, ModelInputRecord, NativeModelInput, model_relation_key_row, }; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, @@ -34,23 +34,18 @@ pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - let model_file: &Bound<'_, PyAny> = record.model_file.bind(py); - match model_file.downcast::() { - Ok(native) => Ok(native.get().file_path_buf()), - Err(_) => model_file.getattr(FILE_PATH_ATTRIBUTE)?.extract(), - } +fn record_path(record: &ModelInputRecord) -> PathBuf { + record.model_file.get().file_path_buf() } /// The record's shared texts with the given passes, read only when their file is linted. fn expansion( expansion: &ExpansionRecord, passes: &[&Vec<(usize, usize, usize, usize)>], - py: Python<'_>, ) -> CompiledSource { CompiledSource::Shared { - authored_sql: expansion.authored_sql.clone_ref(py), - expanded_sql: expansion.expanded_sql.clone_ref(py), + authored_sql: Arc::clone(&expansion.authored_sql), + expanded_sql: Arc::clone(&expansion.expanded_sql), passes: passes.iter().map(|rows| span_pass(rows)).collect(), } } @@ -61,10 +56,10 @@ pub(crate) fn model_lint_expansion( ) -> PyResult> { let py: Python<'_> = model.py(); if let Ok(input) = model.downcast::() { - return input.get().read(py, |record, py| { + return input.get().read(py, |record, _| { let whole: &ExpansionRecord = &record.expansion; Ok(Some(( - record_path(record, py)?, + record_path(record), expansion( whole, &[ @@ -72,7 +67,6 @@ pub(crate) fn model_lint_expansion( &whole.declaration_spans, &whole.macro_spans, ], - py, ), ))) }); @@ -105,11 +99,10 @@ pub(crate) fn project_sql_expansions( let authored: &ExpansionRecord = &record.expansion; if authored.authored { expansions.insert( - record_path(record, py)?, + record_path(record), expansion( authored, &[&authored.declaration_spans, &authored.macro_spans], - py, ), ); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index d9986069f2..7ae7fa68db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -2,11 +2,12 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::text::main::python_isspace::python_isspace; use sqlbuild_core::text::main::python_str_repr::python_str_repr; @@ -176,8 +177,8 @@ impl CompiledExpansion { pub(crate) enum CompiledSource { Owned(CompiledExpansion), Shared { - authored_sql: Py, - expanded_sql: Py, + authored_sql: Arc, + expanded_sql: Arc, passes: Vec>, }, } @@ -192,18 +193,18 @@ impl CompiledSource { } /// The shared texts as an expansion of their own, built only while one file is linted. - fn shared(&self, py: Python<'_>) -> PyResult> { + fn shared(&self) -> Option { match self { - Self::Owned(_) => Ok(None), + Self::Owned(_) => None, Self::Shared { authored_sql, expanded_sql, passes, - } => Ok(Some(CompiledExpansion { - authored_sql: authored_sql.bind(py).to_str()?.to_owned(), - expanded_sql: expanded_sql.bind(py).to_str()?.to_owned(), + } => Some(CompiledExpansion { + authored_sql: authored_sql.to_string(), + expanded_sql: expanded_sql.to_string(), passes: passes.clone(), - })), + }), } } } @@ -592,7 +593,7 @@ pub(crate) fn lint( } }; let file: &LintFile = &files[plan.file]; - let (traits, bodies) = match shared_plan(py, request, file)? { + let (traits, bodies) = match shared_plan(request, file) { Some(Ok(planned)) => planned, Some(Err(failure)) => { scan_failure = Some(failure.into_pyerr(py)); @@ -643,29 +644,25 @@ pub(crate) fn lint( } /// A file's bodies planned again against its shared compile expansion, read only for this file. -fn shared_plan( - py: Python<'_>, - request: &LintRequest<'_>, - file: &LintFile, -) -> PyResult> { +fn shared_plan(request: &LintRequest<'_>, file: &LintFile) -> Option { let Some(source) = request.compiled_expansions.get(&file.path) else { - return Ok(None); + return None; }; - let Some(compiled) = source.shared(py)? else { - return Ok(None); + let Some(compiled) = source.shared() else { + return None; }; let Ok(relative) = file.path.strip_prefix(request.project_dir) else { - return Ok(None); + return None; }; let parts: Vec = path_parts(relative); let text: Vec = file.contents.chars().collect(); let headers: Vec = scan_headers(&text, first_header_only(&parts)); - Ok(Some(plan_bodies( + Some(plan_bodies( request, file, (&text, &headers), (&parts, Some(&compiled)), - ))) + )) } /// Each file's headers, header violations and bodies in path order, up to the first failure. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs index 1be9e9d02e..7d824578c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs @@ -5,10 +5,10 @@ use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{IntoPyObject, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; -use sqlbuild_sqltext::compiler::models::HeaderWord; +use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; -fn word_to_python(py: Python<'_>, word: String) -> PyResult> { - match header_word_value(&word) { +fn word_to_python(py: Python<'_>, word: &str) -> PyResult> { + match header_word_value(word) { HeaderWord::Bool(value) => Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()), HeaderWord::Null => Ok(py.None()), HeaderWord::Integer(digits) => match digits.parse::() { @@ -32,23 +32,17 @@ fn instance( Ok(new_record(py, class, (), Some(arguments))?.unbind()) } -fn values_tuple( - py: Python<'_>, - values: Vec, -) -> PyResult> { +fn values_tuple(py: Python<'_>, values: &[AuthoredValue]) -> PyResult> { let projected = values - .into_iter() + .iter() .map(|item| authored_value_to_python(py, item)) .collect::>>()?; Ok(PyTuple::new(py, projected)?.unbind().into_any()) } -fn constant_call( - py: Python<'_>, - values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, -) -> PyResult> { +fn constant_call(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult> { let arguments = values - .into_iter() + .iter() .map(|(key, value)| { let value = authored_value_to_python(py, value)?; PyTuple::new(py, [key.into_pyobject(py)?.unbind().into_any(), value]) @@ -59,12 +53,11 @@ fn constant_call( instance(py, RecordId::AuthoredSqlValueCall, &keywords) } -fn authored_value_to_python( +/// One authored value as the Python header object discovery hands to config builds. +pub(crate) fn authored_value_to_python( py: Python<'_>, - value: sqlbuild_sqltext::compiler::models::AuthoredValue, + value: &AuthoredValue, ) -> PyResult> { - use sqlbuild_sqltext::compiler::models::AuthoredValue; - match value { AuthoredValue::Null => Ok(py.None()), AuthoredValue::Boolean(value) => { @@ -74,7 +67,7 @@ fn authored_value_to_python( AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), AuthoredValue::List(values) => { let projected = values - .into_iter() + .iter() .map(|item| authored_value_to_python(py, item)) .collect::>>()?; Ok(PyList::new(py, projected)?.unbind().into_any()) @@ -103,7 +96,7 @@ fn authored_value_to_python( pub(crate) fn map_to_python( py: Python<'_>, - values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, + values: &[(String, AuthoredValue)], ) -> PyResult> { let result = PyDict::new(py); for (key, value) in values { @@ -115,8 +108,8 @@ pub(crate) fn map_to_python( fn hook( py: Python<'_>, class: RecordId, - name: String, - kwargs: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, + name: &str, + kwargs: &[(String, AuthoredValue)], ) -> PyResult> { let keywords = PyDict::new(py); keywords.set_item("name", name)?; @@ -126,7 +119,7 @@ fn hook( pub(crate) fn optional_authored_value_to_python( py: Python<'_>, - value: Option, + value: Option<&AuthoredValue>, ) -> PyResult>> { value .map(|item| authored_value_to_python(py, item)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index c66b211658..8d2aab2a9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -27,7 +27,7 @@ pub(crate) fn parse_model_headers( .into_iter() .map(|(value, offsets, error)| { Ok(( - optional_authored_value_to_python(py, value)?, + optional_authored_value_to_python(py, value.as_ref())?, offsets, error, )) From de8c4cec5f298e8f7057c05b5283adae80cf04eb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:22:51 +0100 Subject: [PATCH 20/28] perf: read model headers natively in a parallel pre-pass Before the ordered model loop, a fixed four-worker pool reads every model's native header off the GIL: template and macro presence and config-reuse checks per header entry, and the header's columns. The ordered loop uses a fact only for a config value that is the header's own object, so values that defaults, templates or hooks replaced are still read from Python. Header columns parsed natively build their Python records only when Python reads them. --- Cargo.lock | 1 + .../config_presence/_helpers/macro_walk.rs | 43 +++ .../src/config_presence/_helpers/mod.rs | 1 + .../config_presence/main/entry_macro_path.rs | 10 + .../config_presence/main/first_macro_path.rs | 41 +-- .../src/config_presence/main/mod.rs | 1 + .../tests/test_config_presence.rs | 39 +++ crates/sqlbuild-python/Cargo.toml | 1 + .../bindings/_helpers/compiled_project/mod.rs | 1 + .../_helpers/compiled_project/model_loop.rs | 36 ++- .../compiled_project/model_loop_config.rs | 22 +- .../compiled_project/model_loop_header.rs | 94 +++++- .../compiled_project/model_loop_host.rs | 4 + .../compiled_project/model_loop_prepass.rs | 142 +++++++++ .../_helpers/discovery/model_file_record.rs | 5 + .../_helpers/model_config/header_metadata.rs | 276 ++++++++++++++++-- .../src/bindings/_helpers/model_config/mod.rs | 1 + .../model_config/model_config_builder.rs | 87 ++++-- .../_helpers/model_config/native_nodes.rs | 205 +++++++++++++ 19 files changed, 915 insertions(+), 95 deletions(-) create mode 100644 crates/sqlbuild-model-config/src/config_presence/_helpers/macro_walk.rs create mode 100644 crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/model_config/native_nodes.rs diff --git a/Cargo.lock b/Cargo.lock index 6b81e5a5c1..c795b3af03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1263,6 +1263,7 @@ dependencies = [ "fensu-policy", "mimalloc", "pyo3", + "rayon", "serde", "serde_json", "sqlbuild-analysis", diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_walk.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_walk.rs new file mode 100644 index 0000000000..73e954912d --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_walk.rs @@ -0,0 +1,43 @@ +//! The walk `validate_no_macros_in_config_value` makes over nested config values. + +use crate::config_presence::_helpers::macro_calls::macro_call; +use crate::config_presence::models::MacroPath; +use crate::types::{AuthoredNode, NodeKind}; + +fn walk(node: &N, path: &[String]) -> MacroPath { + match node.kind() { + NodeKind::Str if node.with_text(macro_call).unwrap_or(false) => { + MacroPath::Found(path.to_vec()) + } + NodeKind::Map => walk_entries(&node.entries(), path), + NodeKind::List | NodeKind::Tuple => { + for item in node.items() { + let found = walk(&item, path); + if found != MacroPath::Absent { + return found; + } + } + MacroPath::Absent + } + _ => MacroPath::Absent, + } +} + +/// The first macro path among `entries`, string keys only, each below `path`. +pub(crate) fn walk_entries(entries: &[(N, N)], path: &[String]) -> MacroPath { + for (key, value) in entries { + if key.kind() != NodeKind::Str { + continue; + } + let Some(name) = key.text() else { + continue; + }; + let mut child: Vec = path.to_vec(); + child.push(name); + let found = walk(value, &child); + if found != MacroPath::Absent { + return found; + } + } + MacroPath::Absent +} diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs index 1f894b4988..cb89f7ecbd 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod macro_calls; +pub(crate) mod macro_walk; pub(crate) mod reuse; pub(crate) mod scan; pub(crate) mod template_tokens; diff --git a/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs b/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs new file mode 100644 index 0000000000..0b63052da2 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs @@ -0,0 +1,10 @@ +//! Where Python's config walk finds a macro call within one top-level config entry. + +use crate::config_presence::_helpers::macro_walk::walk_entries; +use crate::config_presence::models::MacroPath; +use crate::types::AuthoredNode; + +/// The first macro path `first_macro_path` reports for the single entry `key: value`. +pub fn entry_macro_path(key: &N, value: &N) -> MacroPath { + walk_entries(&[(key.clone(), value.clone())], &[]) +} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs b/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs index d3ec7c465a..6cde1c9bc3 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs @@ -1,8 +1,8 @@ //! The first config field, in Python's walk order, whose string holds a macro call. -use crate::config_presence::_helpers::macro_calls::macro_call; +use crate::config_presence::_helpers::macro_walk::walk_entries; use crate::config_presence::models::MacroPath; -use crate::types::{AuthoredNode, NodeKind}; +use crate::types::AuthoredNode; /// Walk config values as `validate_no_macros_in_config_value` does, skipping the `skipped` keys. pub fn first_macro_path(values: &N, skipped: &[&str]) -> MacroPath { @@ -10,40 +10,3 @@ pub fn first_macro_path(values: &N, skipped: &[&str]) -> MacroP entries.retain(|(key, _)| !skipped.iter().any(|name| key.is_text(name))); walk_entries(&entries, &[]) } - -fn walk(node: &N, path: &[String]) -> MacroPath { - match node.kind() { - NodeKind::Str if node.with_text(macro_call).unwrap_or(false) => { - MacroPath::Found(path.to_vec()) - } - NodeKind::Map => walk_entries(&node.entries(), path), - NodeKind::List | NodeKind::Tuple => { - for item in node.items() { - let found = walk(&item, path); - if found != MacroPath::Absent { - return found; - } - } - MacroPath::Absent - } - _ => MacroPath::Absent, - } -} - -fn walk_entries(entries: &[(N, N)], path: &[String]) -> MacroPath { - for (key, value) in entries { - if key.kind() != NodeKind::Str { - continue; - } - let Some(name) = key.text() else { - continue; - }; - let mut child: Vec = path.to_vec(); - child.push(name); - let found = walk(value, &child); - if found != MacroPath::Absent { - return found; - } - } - MacroPath::Absent -} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs index e99609df7a..a808b6e464 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs @@ -1,4 +1,5 @@ pub mod blocks_config_reuse; pub mod contains_template; +pub mod entry_macro_path; pub mod first_macro_path; pub mod mutable_config_value; diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs index a6fe61c021..af2c9b6210 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs @@ -147,3 +147,42 @@ fn given_config_values_when_checking_reuse_then_python_answers_are_returned() { ); } } + +#[test] +fn given_config_entries_when_walking_each_entry_then_the_first_found_matches_the_whole_walk() { + use crate::config_presence::main::entry_macro_path::entry_macro_path; + use crate::config_presence::main::first_macro_path::first_macro_path; + use crate::config_presence::models::MacroPath; + + let entries = vec![ + (Value::Str("pre_hooks"), Value::Str("@skipped()")), + (Value::Str("materialized"), Value::Str("table")), + ( + Value::Str("columns"), + Value::Map(vec![( + Value::Str("id"), + Value::Map(vec![(Value::Str("description"), Value::Str("@cents(id)"))]), + )]), + ), + ( + Value::Str("tags"), + Value::List(vec![Value::Str("@later()")]), + ), + ]; + let whole: MacroPath = first_macro_path(&Value::Map(entries.clone()), &["pre_hooks"]); + let per_entry: MacroPath = entries + .iter() + .filter(|(key, _)| !matches!(key, Value::Str("pre_hooks"))) + .map(|(key, value)| entry_macro_path(key, value)) + .find(|found| *found != MacroPath::Absent) + .unwrap_or(MacroPath::Absent); + assert_eq!( + whole, + MacroPath::Found(vec![ + "columns".to_owned(), + "id".to_owned(), + "description".to_owned() + ]) + ); + assert_eq!(per_entry, whole); +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 0ac50aadbd..5645b3946a 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -19,6 +19,7 @@ extension-module = ["pyo3/extension-module"] fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true +rayon.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 63a4cce12f..a56686891e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod model_loop_inputs; pub(crate) mod model_loop_macros; pub(crate) mod model_loop_named; pub(crate) mod model_loop_opt_out; +pub(crate) mod model_loop_prepass; pub(crate) mod model_loop_python_hooks; pub(crate) mod model_loop_schema; pub(crate) mod model_loop_visibility; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index 325f0c319b..7c50ad165e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -34,6 +34,9 @@ use crate::bindings::_helpers::compiled_project::model_loop_named::{ use crate::bindings::_helpers::compiled_project::model_loop_opt_out::{ OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, }; +use crate::bindings::_helpers::compiled_project::model_loop_prepass::{ + HeaderFacts, all_header_facts, +}; use crate::bindings::_helpers::compiled_project::model_loop_python_hooks::{ HookClasses, HookSignatures, }; @@ -90,6 +93,11 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::types::CompilerDetach; +use sqlbuild_sqltext::compiler::models::AuthoredValue; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; @@ -181,6 +189,8 @@ pub(crate) struct LoopContext<'a, 'py> { pub(crate) host: &'a Bound<'py, ConstantRendering>, pub(crate) classes: &'a LoopClasses<'py>, pub(crate) contracts: &'a ContractClasses<'py>, + /// The contract classes as Python passed them, kept by natively parsed columns. + pub(crate) contract_dict: &'a Py, } /// The declarations a model sees and the identity its usages are recorded for. @@ -316,6 +326,7 @@ impl NativeModelLoop { host: &host, classes: &classes, contracts: &contracts, + contract_dict: &self.contract_classes, }; project .borrow_mut() @@ -345,6 +356,7 @@ impl NativeModelLoop { &context, &record.header_value_dict(py)?, (name, relative_path), + row.header.as_ref(), )?; let model: LoopModel<'_, 'py> = LoopModel { row: &row, @@ -382,7 +394,22 @@ impl NativeModelLoop { ) -> PyResult>> { let py: Python<'py> = slf.py(); let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; - let models: Vec = prepared_rows(py, &classes, model_files)?; + let mut models: Vec = prepared_rows(py, &classes, model_files)?; + let headers: Vec, String)>> = models + .iter() + .map(|row| { + row.model_file + .get() + .native_header_values() + .map(|values| (Arc::clone(values), row.paths.relative_path.clone())) + }) + .collect(); + let facts: Vec> = py + .compiler_detach(|| all_header_facts(&headers)) + .map_err(compiler_error)?; + for (row, facts) in models.iter_mut().zip(facts) { + row.header = facts; + } project .borrow_mut() .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); @@ -404,6 +431,7 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, + contract_dict: &self.contract_classes, }; let attached: Vec = attached_sources(self, &context, rows, settings)?; Ok(retain_sources(project, attached)) @@ -424,6 +452,7 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, + contract_dict: &self.contract_classes, }; attached_audits(self, &context, (files, owners), settings) } @@ -443,6 +472,7 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, + contract_dict: &self.contract_classes, }; let validator: Bound<'py, NativeModelValidator> = self.validator.bind(py).downcast()?.clone(); @@ -510,6 +540,7 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, + contract_dict: &self.contract_classes, }; let attached: Vec = attached_functions(self, &context, rows, settings)?; checked_table_function_arities(&context, models, &attached)?; @@ -543,6 +574,7 @@ impl NativeModelLoop { scan, interpolation, model_file, + header: header_facts, } = row; let values: Bound<'py, PyDict> = native_field(&config, VALUES_ATTRIBUTE)?.downcast_into()?; @@ -698,6 +730,8 @@ impl NativeModelLoop { relative_path, visible: (&visible.declarations, &visible.consumer), schema: schema.as_ref(), + facts: header_facts.as_ref(), + model_file_handle: model_file, }; let header: AttachedHeader<'py> = header_schema(py, context, &source)?; let mut named_usages: Vec> = hook_usages; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs index 0a400c0a40..7a2d5a9b44 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs @@ -12,6 +12,7 @@ use sqlbuild_model_config::config_presence::main::mutable_config_value::mutable_ use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, config_error}; +use crate::bindings::_helpers::compiled_project::model_loop_prepass::HeaderFacts; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::model_config_builder::{ BuiltConfig, ConfigBuild, NativeModelConfigBuilder, @@ -69,6 +70,7 @@ impl<'py> ModelConfigs<'py> { context: &LoopContext<'_, 'py>, header: &Bound<'py, PyDict>, model: (&str, &str), + facts: Option<&HeaderFacts>, ) -> PyResult> { let (name, relative_path) = model; let py: Python<'py> = header.py(); @@ -77,7 +79,7 @@ impl<'py> ModelConfigs<'py> { let matched: Option = builder .selected_path_default(py, relative_path)? .map_err(|error| config_error(error.bind(py).as_any(), None))?; - let metadata: Option> = reusable_metadata(header)?; + let metadata: Option> = reusable_metadata(header, facts)?; let reusable: bool = match metadata { Some(_) => self.reusable(context, &matched)?, None => false, @@ -90,7 +92,7 @@ impl<'py> ModelConfigs<'py> { values.update(metadata.as_mapping())?; return view(classes, &values, &cached.fields); } - let built: ConfigBuild<'py> = builder.built(py, header, matched.as_deref(), name)?; + let built: ConfigBuild<'py> = builder.built(py, header, matched.as_deref(), name, facts)?; let built: BuiltConfig<'py> = match built { ConfigBuild::Built(built) => built, ConfigBuild::Rejected(error) => { @@ -168,7 +170,10 @@ impl<'py> ModelConfigs<'py> { } /// `_reusable_metadata`: the header when only plain columns and description vary, else `None`. -fn reusable_metadata<'py>(header: &Bound<'py, PyDict>) -> PyResult>> { +fn reusable_metadata<'py>( + header: &Bound<'py, PyDict>, + facts: Option<&HeaderFacts>, +) -> PyResult>> { let py: Python<'py> = header.py(); if header.is_empty() { return Ok(Some(PyDict::new(py))); @@ -183,9 +188,14 @@ fn reusable_metadata<'py>(header: &Bound<'py, PyDict>) -> PyResult entry.blocks_reuse, + None => blocks_config_reuse(&PyNode(value)), + }; + if blocks { return Ok(None); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index d58cd4fe8e..391837dc0a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -2,11 +2,12 @@ use std::sync::Arc; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods}; use sqlbuild_analysis::compiled_project::models::{ DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, }; +use sqlbuild_discovery::models::LineColumnSpan; use sqlbuild_model_config::header_metadata::main::audit_factory_references::audit_factory_references; use sqlbuild_model_config::header_metadata::main::merged_audit_identities::merged_audit_identities; use sqlbuild_model_config::header_metadata::main::parse_dynamic_columns::parse_dynamic_columns; @@ -21,6 +22,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_contracts::{ EnumColumns, dynamic_family_objects, enum_columns, factory_audits, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; +use crate::bindings::_helpers::compiled_project::model_loop_prepass::HeaderFacts; use crate::bindings::_helpers::compiled_project::model_loop_schema::{ DeclaredColumns, ModelSchema, merged_columns, schema_enums, }; @@ -30,7 +32,8 @@ use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::header_metadata::{ - HeaderColumns, audit_instances, native_schema_column, parsed_audit_identity, + HeaderColumn, HeaderColumns, NativeColumn, NativeColumnSpec, audit_instances, + native_schema_column, parsed_audit_identity, }; use crate::bindings::_helpers::records::fields::field as native_field; @@ -51,6 +54,9 @@ pub(crate) struct HeaderSource<'a, 'py> { /// The model's visible declarations and the identity its usages are recorded for. pub(crate) visible: (&'a Bound<'py, PyAny>, &'a Bound<'py, PyAny>), pub(crate) schema: Option<&'a ModelSchema<'py>>, + /// What the pre-pass read from the model's native header. + pub(crate) facts: Option<&'a HeaderFacts>, + pub(crate) model_file_handle: &'a Py, } /// The header's description and, when it declares any schema metadata, its entry. @@ -100,18 +106,26 @@ pub(crate) fn header_schema<'py>( let none: Bound<'py, PyAny> = py.None().into_bound(py); let node = |value: Option>| PyNode(value.unwrap_or_else(|| none.clone())); let model: (&str, &str) = (source.name, source.relative_path); - let metadata = parse_header_metadata(&node(columns), &node(audits), source.relative_path); + let native: Option<&[NativeColumn]> = native_columns(source, columns.as_ref())?; + let python_columns: Option> = if native.is_some() { None } else { columns }; + let metadata = + parse_header_metadata(&node(python_columns), &node(audits), source.relative_path); let parsed_columns = metadata .columns .map_err(|error| raised_config_error(py, error))?; - let locations: Bound<'py, PyDict> = source.model_file.header_locations(py)?; let model_path: &Bound<'py, PyAny> = &source.model_file.path_objects(py)?.1; - let local_columns: DeclaredColumns = DeclaredColumns { - specs: parsed_columns - .iter() - .map(|column| native_schema_column(py, context.contracts, column, &locations)) - .collect::>()?, - facts: local_column_facts(&parsed_columns)?, + let local_columns: DeclaredColumns = match native { + Some(native) => native_declared_columns(context, source, native)?, + None => { + let locations: Bound<'py, PyDict> = source.model_file.header_locations(py)?; + DeclaredColumns { + specs: parsed_columns + .iter() + .map(|column| native_schema_column(py, context.contracts, column, &locations)) + .collect::>()?, + facts: local_column_facts(&parsed_columns)?, + } + } }; let columns: DeclaredColumns = match source.schema { Some(schema) => merged_columns( @@ -197,6 +211,66 @@ pub(crate) fn header_schema<'py>( }) } +/// The natively parsed header columns, when the config's columns are the header's own value. +fn native_columns<'a>( + source: &HeaderSource<'a, '_>, + columns: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + let (Some(facts), Some(columns)) = (source.facts, columns) else { + return Ok(None); + }; + let Some(parsed) = &facts.columns else { + return Ok(None); + }; + let header: Bound<'_, PyDict> = source.model_file.header_value_dict(columns.py())?; + let own: bool = header + .get_item(COLUMNS_KEY)? + .is_some_and(|value| value.is(columns)); + Ok(own.then_some(parsed.as_slice())) +} + +/// Native header columns as declared columns, their records built only when Python reads them. +fn native_declared_columns( + context: &LoopContext<'_, '_>, + source: &HeaderSource<'_, '_>, + columns: &[NativeColumn], +) -> PyResult { + let py: Python<'_> = context.host.py(); + let rows: &[(String, LineColumnSpan)] = source.model_file.header_location_rows(); + let contracts: Arc> = Arc::new(context.contract_dict.clone_ref(py)); + let mut specs: Vec = Vec::with_capacity(columns.len()); + let mut facts: Vec = Vec::with_capacity(columns.len()); + for column in columns { + let location: Option = rows + .iter() + .rev() + .find(|(name, _)| *name == column.name) + .map(|(_, span)| *span); + facts.push(SchemaColumnFact { + name: column.name.clone(), + data_type: column.column_type.clone(), + nullable: column.nullable, + described: column.description.is_some(), + migrate_from: column.migrate_from.clone(), + audit_definitions: column + .audits + .iter() + .map(|audit| audit.definition_name.clone()) + .collect(), + location_path: None, + }); + specs.push(HeaderColumn::native( + &context.contracts.schema_column, + NativeColumnSpec { + column: column.clone(), + location: location.map(|span| (source.model_file_handle.clone_ref(py), span)), + contracts: Arc::clone(&contracts), + }, + )?); + } + Ok(DeclaredColumns { specs, facts }) +} + /// The header's dynamic column families, each `name_pattern` compiled natively. fn dynamic_families( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 437243e1a7..a45fc59541 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -17,6 +17,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_header::AttachedHead use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ reference_fact, stripped_values, }; +use crate::bindings::_helpers::compiled_project::model_loop_prepass::HeaderFacts; use crate::bindings::_helpers::compiled_project::project::{ CONTRACT_KEY, INCREMENTAL_MODE_KEY, materialization, text_value, }; @@ -56,6 +57,8 @@ pub(crate) struct ModelLoopRow { pub(crate) interpolation: InterpolationRow, /// The discovered model file, owned by the discovery inputs. pub(crate) model_file: Py, + /// What the pre-pass read from the model's native header, when it has one. + pub(crate) header: Option, } impl ModelLoopRow { @@ -73,6 +76,7 @@ impl ModelLoopRow { scan, interpolation, model_file, + header: None, } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs new file mode 100644 index 0000000000..3b44c4b705 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs @@ -0,0 +1,142 @@ +//! What each model's native header values show before the ordered loop, read off the GIL. +//! +//! The ordered loop uses a fact only for a config value that is the header's own object, so a +//! value that path defaults, templates or hooks replaced is still read from Python. + +use std::sync::{Arc, OnceLock}; + +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use rayon::{ThreadPool, ThreadPoolBuilder}; +use sqlbuild_model_config::config_presence::main::blocks_config_reuse::blocks_config_reuse; +use sqlbuild_model_config::config_presence::main::contains_template::contains_template; +use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; +use sqlbuild_model_config::config_presence::models::MacroPath; +use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::bindings::_helpers::model_config::header_metadata::NativeColumn; +use crate::bindings::_helpers::model_config::native_nodes::{NativeNode, dict_entries, exactly}; + +const COLUMNS_KEY: &str = "columns"; +/// The compile pools' fixed worker count. +const PREPASS_WORKERS: usize = 4; +const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; + +static NULL_VALUE: AuthoredValue = AuthoredValue::Null; +static POOL: OnceLock> = OnceLock::new(); + +/// What one top-level header entry holds, as the config checks walk it. +pub(crate) struct EntryFacts { + /// Whether a string in the value holds a `${...}` template. + pub(crate) template: bool, + /// Where `first_macro_path` finds a macro call in this entry. + pub(crate) macro_path: MacroPath, + /// Whether the value keeps the model from reusing another model's config. + pub(crate) blocks_reuse: bool, +} + +/// One model's header facts, read from its native authored values. +pub(crate) struct HeaderFacts { + /// Each header key in the projected dict's order, with its facts. + entries: Vec<(String, EntryFacts)>, + /// The header's columns, when they parse without an error. + pub(crate) columns: Option>, +} + +impl HeaderFacts { + /// The facts of `key`'s value in the header. + pub(crate) fn entry(&self, key: &str) -> Option<&EntryFacts> { + self.entries + .iter() + .find(|(name, _)| name == key) + .map(|(_, facts)| facts) + } + + /// The facts of a config entry, when its value is the header's own value of `key`. + pub(crate) fn own_entry<'py>( + &self, + header: &Bound<'py, PyDict>, + key: &Bound<'py, PyAny>, + value: &Bound<'py, PyAny>, + ) -> PyResult> { + let Ok(name) = key.downcast::() else { + return Ok(None); + }; + let Ok(name) = name.to_str() else { + return Ok(None); + }; + let Some(facts) = self.entry(name) else { + return Ok(None); + }; + Ok(header + .get_item(name)? + .filter(|own| own.is(value)) + .map(|_| facts)) + } +} + +/// The header facts of one model's native authored values; `path` names it in parse messages. +pub(crate) fn header_facts(values: &[(String, AuthoredValue)], path: &str) -> HeaderFacts { + let entries: Vec<(String, EntryFacts)> = dict_entries(values) + .into_iter() + .map(|(key, value)| { + let node: NativeNode<'_> = NativeNode::Value(value); + ( + key.to_owned(), + EntryFacts { + template: contains_template(&node), + macro_path: entry_macro_path(&NativeNode::Key(key), &node), + blocks_reuse: blocks_config_reuse(&node), + }, + ) + }) + .collect(); + let columns: Option> = dict_entries(values) + .into_iter() + .find(|(key, _)| *key == COLUMNS_KEY) + .map(|(_, value)| value) + .filter(|value| **value != AuthoredValue::Null) + .and_then(|value| { + let (metadata, exact) = exactly(|| { + parse_header_metadata( + &NativeNode::Value(value), + &NativeNode::Value(&NULL_VALUE), + path, + ) + }); + let columns = metadata.columns.ok().filter(|_| exact)?; + Some(columns.iter().map(NativeColumn::parsed).collect()) + }); + HeaderFacts { entries, columns } +} + +/// Each model's header facts, computed on the compile pool; `None` where no native header exists. +pub(crate) fn all_header_facts( + headers: &[Option<(Arc<[(String, AuthoredValue)]>, String)>], +) -> Result>, String> { + let read = || -> Vec> { + headers + .par_iter() + .map(|header| { + header + .as_ref() + .map(|(values, path)| header_facts(values, path)) + }) + .collect() + }; + let pool: &ThreadPool = POOL + .get_or_init(|| { + ThreadPoolBuilder::new() + .num_threads(PREPASS_WORKERS) + .stack_size(WORKER_STACK_BYTES) + .thread_name(|index| format!("sqlbuild-model-prepass-{index}")) + .build() + .map_err(|error| error.to_string()) + }) + .as_ref() + .map_err(Clone::clone)?; + Ok(pool.install(read)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index 5410103050..580cbeeedf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -368,6 +368,11 @@ impl NativeModelFile { self.header_values.dict(py) } + /// The header columns' authored locations, unless Python constructed this record. + pub(crate) fn header_location_rows(&self) -> &[(String, LineColumnSpan)] { + &self.header_column_locations.rows + } + /// The header's native authored values, unless Python constructed this record. pub(crate) fn native_header_values(&self) -> Option<&Arc<[(String, AuthoredValue)]>> { self.header_values.native.as_ref() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index 2940dc1a98..29168b65ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -1,29 +1,124 @@ //! Parsed MODEL header columns and audits built into the Python contract objects. -use std::sync::OnceLock; +use std::sync::{Arc, OnceLock}; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject}; +use pyo3::{FromPyObject, IntoPyObject, IntoPyObjectExt}; +use sqlbuild_discovery::models::LineColumnSpan; use sqlbuild_model_config::header_metadata::models::{ ParsedAudit, ParsedColumn, ParsedThresholds, ThresholdBound, }; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::records::construct::record_class; +use crate::bindings::_helpers::model_config::native_nodes::NativeNode; +use crate::bindings::_helpers::records::construct::{new_record, record_class}; use crate::bindings::_helpers::records::fields::all_specs; use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::sqltext::authored_values::authored_value_to_python; /// Native column fields; a Python record is constructed only for public consumers. pub(crate) struct HeaderColumn { - values: Option<[Py; 8]>, + fields: ColumnFields, class: Py, projection: PyOnceLock>, } +/// What a column's Python record is built from. +enum ColumnFields { + /// The record's field values, in declaration order. + Python([Py; 8]), + /// A header column read natively, with what its location and audits are built with. + Native(Box), + /// Only the record itself, built by Python. + Projected, +} + +/// A natively parsed header column and what its Python record is built with. +pub(crate) struct NativeColumnSpec { + pub(crate) column: NativeColumn, + /// The model file whose relative path the column's location names, and the location. + pub(crate) location: Option<(Py, LineColumnSpan)>, + /// The contract classes the record and its audits are built with. + pub(crate) contracts: Arc>, +} + +/// One MODEL column parsed from native authored values. +#[derive(Clone, Debug)] +pub(crate) struct NativeColumn { + pub(crate) name: String, + pub(crate) column_type: Option, + pub(crate) nullable: Option, + pub(crate) description: Option, + pub(crate) audits: Vec, + pub(crate) migrate_from: Option, +} + +/// One audit instance parsed from native authored values. +#[derive(Clone, Debug)] +pub(crate) struct NativeAudit { + pub(crate) definition_name: String, + pub(crate) arguments: Vec<(String, AuthoredValue)>, + pub(crate) name: Option, + pub(crate) description: Option, + pub(crate) severity: Option, + pub(crate) run_scope: Option, + pub(crate) always_run: Option, + pub(crate) thresholds: Option, + pub(crate) minimum_samples: Option, + pub(crate) evidence_limit: Option, +} + +impl NativeColumn { + /// The column a native parse produced; every leaf is the projected text, flag or count. + pub(crate) fn parsed(column: &ParsedColumn>) -> Self { + let text = |node: Option<&NativeNode<'_>>| node.and_then(AuthoredNode::text); + Self { + name: column.name.text().unwrap_or_default(), + column_type: text(column.column_type.as_ref()), + nullable: flag(column.nullable.as_ref()), + description: text(column.description.as_ref()), + audits: column.audits.iter().map(NativeAudit::parsed).collect(), + migrate_from: text(column.migrate_from.as_ref()), + } + } +} + +impl NativeAudit { + fn parsed(audit: &ParsedAudit>) -> Self { + let text = |node: Option<&NativeNode<'_>>| node.and_then(AuthoredNode::text); + Self { + definition_name: audit.definition_name.text().unwrap_or_default(), + arguments: audit + .arguments + .iter() + .map(|(key, value)| (key.text().unwrap_or_default(), value.authored())) + .collect(), + name: text(audit.name.as_ref()), + description: text(audit.description.as_ref()), + severity: text(audit.severity.as_ref()), + run_scope: text(audit.run_scope.as_ref()), + always_run: flag(audit.always_run.as_ref()), + thresholds: audit.thresholds, + minimum_samples: audit.minimum_samples.as_ref().map(NativeNode::authored), + evidence_limit: audit.evidence_limit.as_ref().map(NativeNode::authored), + } + } +} + +fn flag(node: Option<&NativeNode<'_>>) -> Option { + match node.map(AuthoredNode::kind) { + Some(NodeKind::Bool(value)) => Some(value), + _ => None, + } +} + impl HeaderColumn { pub(crate) fn replaced<'py>( class: &Bound<'py, PyAny>, @@ -41,28 +136,35 @@ impl HeaderColumn { } pub(crate) fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { - let Some(values) = &self.values else { - return field(&self.project(py)?, name); + let index = || { + SCHEMA_COLUMN_FIELDS + .iter() + .position(|field| *field == name) + .ok_or_else(|| PyRuntimeError::new_err("unknown column field")) }; - let index = SCHEMA_COLUMN_FIELDS - .iter() - .position(|field| *field == name) - .ok_or_else(|| PyRuntimeError::new_err("unknown column field"))?; - Ok(values[index].bind(py).clone()) + if let Some(projected) = self.projection.get(py) { + return field(projected.bind(py), name); + } + match &self.fields { + ColumnFields::Python(values) => Ok(values[index()?].bind(py).clone()), + ColumnFields::Native(spec) => native_value(py, spec, index()?), + ColumnFields::Projected => field(&self.project(py)?, name), + } } pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { self.projection .get_or_try_init(py, || { - let values = self - .values - .as_ref() - .ok_or_else(|| PyRuntimeError::new_err("missing column projection"))?; - contract_object( - self.class.bind(py), - values.each_ref().map(|value| value.bind(py).clone()), - ) - .map(Bound::unbind) + let values: [Bound<'py, PyAny>; 8] = match &self.fields { + ColumnFields::Python(values) => { + values.each_ref().map(|value| value.bind(py).clone()) + } + ColumnFields::Native(spec) => native_values(py, spec)?, + ColumnFields::Projected => { + return Err(PyRuntimeError::new_err("missing column projection")); + } + }; + contract_object(self.class.bind(py), values).map(Bound::unbind) }) .map(|value| value.bind(py).clone()) } @@ -73,7 +175,7 @@ impl HeaderColumn { let class = value.get_type().into_any().unbind(); let _ = projection.set(py, value.unbind()); Ok(Self { - values: None, + fields: ColumnFields::Projected, class, projection, }) @@ -88,11 +190,141 @@ impl HeaderColumn { } checked_field_orders()?; Ok(Self { - values: Some(values.map(Bound::unbind)), + fields: ColumnFields::Python(values.map(Bound::unbind)), class: class.clone().unbind(), projection: PyOnceLock::new(), }) } + + /// A natively parsed column, whose Python record is built only when Python reads it. + pub(crate) fn native(class: &Bound<'_, PyAny>, spec: NativeColumnSpec) -> PyResult { + let py: Python<'_> = class.py(); + if !class.is(&record_class(py, RecordId::SchemaColumn)?) { + return Self::from_python(contract_object(class, native_values(py, &spec)?)?); + } + checked_field_orders()?; + Ok(Self { + fields: ColumnFields::Native(Box::new(spec)), + class: class.clone().unbind(), + projection: PyOnceLock::new(), + }) + } +} + +/// A native column's record fields, in declaration order. +fn native_values<'py>( + py: Python<'py>, + spec: &NativeColumnSpec, +) -> PyResult<[Bound<'py, PyAny>; 8]> { + let location: Bound<'py, PyAny> = native_location(py, spec)?; + let audits: Bound<'py, PyAny> = native_audits(py, spec, &location)?; + let column: &NativeColumn = &spec.column; + Ok([ + column.name.as_str().into_bound_py_any(py)?, + column.column_type.as_deref().into_bound_py_any(py)?, + column.nullable.into_bound_py_any(py)?, + column.description.as_deref().into_bound_py_any(py)?, + PyDict::new(py).into_any(), + audits, + location, + column.migrate_from.as_deref().into_bound_py_any(py)?, + ]) +} + +/// The native column's record field at `index` of `SCHEMA_COLUMN_FIELDS`. +fn native_value<'py>( + py: Python<'py>, + spec: &NativeColumnSpec, + index: usize, +) -> PyResult> { + let column: &NativeColumn = &spec.column; + match SCHEMA_COLUMN_FIELDS[index] { + "name" => column.name.as_str().into_bound_py_any(py), + "type" => column.column_type.as_deref().into_bound_py_any(py), + "nullable" => column.nullable.into_bound_py_any(py), + "description" => column.description.as_deref().into_bound_py_any(py), + "meta" => Ok(PyDict::new(py).into_any()), + "audits" => native_audits(py, spec, &native_location(py, spec)?), + "location" => native_location(py, spec), + _ => column.migrate_from.as_deref().into_bound_py_any(py), + } +} + +/// The column's `SourceLocation`, or `None`. +fn native_location<'py>(py: Python<'py>, spec: &NativeColumnSpec) -> PyResult> { + match &spec.location { + Some((model_file, span)) => new_record( + py, + RecordId::SourceLocation, + ( + model_file.get().path_objects(py)?.1, + span.line, + span.column, + span.end_line, + span.end_column, + ), + None, + ), + None => Ok(py.None().into_bound(py)), + } +} + +/// The column's audits as a tuple of `SchemaAuditInstance`s at `location`. +fn native_audits<'py>( + py: Python<'py>, + spec: &NativeColumnSpec, + location: &Bound<'py, PyAny>, +) -> PyResult> { + let classes: ContractClasses<'py> = spec.contracts.bind(py).extract()?; + let audits: Vec> = spec + .column + .audits + .iter() + .map(|audit| native_audit_instance(py, &classes, audit, location)) + .collect::>()?; + Ok(PyTuple::new(py, audits)?.into_any()) +} + +/// A native audit as the `SchemaAuditInstance` the Python parse builds. +fn native_audit_instance<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + audit: &NativeAudit, + location: &Bound<'py, PyAny>, +) -> PyResult> { + let arguments = PyDict::new(py); + for (key, value) in &audit.arguments { + arguments.set_item(key, authored_value_to_python(py, value)?)?; + } + let severity = match &audit.severity { + Some(severity) => classes + .severities + .get_item(severity)? + .unwrap_or_else(|| py.None().into_bound(py)), + None => py.None().into_bound(py), + }; + let count = |value: Option<&AuthoredValue>| -> PyResult> { + match value { + Some(value) => Ok(authored_value_to_python(py, value)?.into_bound(py)), + None => Ok(py.None().into_bound(py)), + } + }; + contract_object( + &classes.schema_audit_instance, + [ + audit.definition_name.as_str().into_bound_py_any(py)?, + arguments.into_any(), + audit.name.as_deref().into_bound_py_any(py)?, + audit.description.as_deref().into_bound_py_any(py)?, + severity, + audit.run_scope.as_deref().into_bound_py_any(py)?, + audit.always_run.unwrap_or(false).into_bound_py_any(py)?, + thresholds(py, classes, audit.thresholds.as_ref())?, + count(audit.minimum_samples.as_ref())?, + count(audit.evidence_limit.as_ref())?, + location.clone(), + ], + ) } /// Retained native columns and their shared, lazy public tuple. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index 0d0f9cbb49..a9a70bfe31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -10,6 +10,7 @@ pub(crate) mod header_columns; pub(crate) mod header_metadata; pub(crate) mod model_config_builder; pub(crate) mod model_validator; +pub(crate) mod native_nodes; pub(crate) mod project_defaults; pub(crate) mod registration; pub(crate) mod template_data; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index ab7dc904ee..84c97c4aac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -8,7 +8,7 @@ use pyo3::types::{ }; use pyo3::{PyErr, pyclass, pymethods}; use sqlbuild_model_config::config_presence::main::contains_template::contains_template; -use sqlbuild_model_config::config_presence::main::first_macro_path::first_macro_path; +use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; use sqlbuild_model_config::config_presence::models::MacroPath; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::model_validation::main::retention_override::retention_override; @@ -21,6 +21,7 @@ use sqlbuild_model_config::types::AuthoredNode; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, built_config}; +use crate::bindings::_helpers::compiled_project::model_loop_prepass::{EntryFacts, HeaderFacts}; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; @@ -164,14 +165,14 @@ impl NativeModelConfigBuilder { ) -> PyResult> { let (matched_path_default, model_name) = model; let matched_path_default: Option<&str> = matched_path_default.as_deref(); - compiler_guard( - || match self.built(py, &header, matched_path_default, &model_name)? { + compiler_guard(|| { + match self.built(py, &header, matched_path_default, &model_name, None)? { ConfigBuild::Built(built) => { Ok(built_config(&classes, &built, matched_path_default)?.unbind()) } ConfigBuild::Rejected(error) => Ok(error.into_any()), - }, - ) + } + }) } } @@ -195,8 +196,13 @@ impl NativeModelConfigBuilder { header: &Bound<'py, PyDict>, matched_path_default: Option<&str>, model_name: &str, + facts: Option<&HeaderFacts>, ) -> PyResult> { - match self.build_config(py, header, matched_path_default, model_name) { + let header: HeaderInput<'_, 'py> = HeaderInput { + values: header, + facts, + }; + match self.build_config(py, &header, matched_path_default, model_name) { Ok(built) => Ok(ConfigBuild::Built(built)), Err(Halt::Config(error)) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), Err(Halt::Error(error)) => Err(error), @@ -290,10 +296,11 @@ impl NativeModelConfigBuilder { fn build_config<'py>( &self, py: Python<'py>, - header: &Bound<'py, PyDict>, + input: &HeaderInput<'_, 'py>, matched_path_default: Option<&str>, model_name: &str, ) -> Built> { + let header: &Bound<'py, PyDict> = input.values; check_header_tags(header, model_name)?; let path_values = matched_path_default .map(|key| self.path_default_values(py, key)) @@ -311,7 +318,7 @@ impl NativeModelConfigBuilder { layered.del_item(key)?; } } - let has_templates = contains_template(&PyNode(layered.clone().into_any())); + let has_templates = input.contains_template(&layered)?; let mut expansion = Expansion { builder: self, py, @@ -380,7 +387,7 @@ impl NativeModelConfigBuilder { header.get_item(TABLE_TYPE_KEY)?.map(PyNode).as_ref(), model_name, )?; - check_no_config_macros(&values)?; + input.check_no_config_macros(&values)?; Ok(BuiltConfig { values, header_keys: sorted_keys(header)?, @@ -765,14 +772,60 @@ fn storage_free_values<'py>( Ok(values) } -/// Raise for the first config field outside hooks holding a macro call, as Python's walk does. -fn check_no_config_macros(values: &Bound<'_, PyDict>) -> Built<()> { - match first_macro_path(&PyNode(values.clone().into_any()), &HOOK_KEYS) { - MacroPath::Absent => Ok(()), - MacroPath::Found(path) => Err(Halt::Config(ConfigError::compile(format!( - "model config field '{}' does not allow macros", - path.join(".") - )))), +/// A model's authored header and, when it was read natively, the header's facts. +struct HeaderInput<'a, 'py> { + values: &'a Bound<'py, PyDict>, + facts: Option<&'a HeaderFacts>, +} + +impl<'py> HeaderInput<'_, 'py> { + /// The facts of a config entry whose value is the header's own. + fn own_entry( + &self, + key: &Bound<'py, PyAny>, + value: &Bound<'py, PyAny>, + ) -> PyResult> { + match self.facts { + Some(facts) => facts.own_entry(self.values, key, value), + None => Ok(None), + } + } + + /// `contains_template_data(values)`, reading the header's own entries from their facts. + fn contains_template(&self, values: &Bound<'py, PyDict>) -> Built { + for (key, value) in values.iter() { + let found: bool = match self.own_entry(&key, &value)? { + Some(facts) => facts.template, + None => contains_template(&PyNode(value)), + }; + if found { + return Ok(true); + } + } + Ok(false) + } + + /// Raise for the first config field outside hooks holding a macro call, as Python's walk does. + fn check_no_config_macros(&self, values: &Bound<'py, PyDict>) -> Built<()> { + for (key, value) in values.iter() { + if HOOK_KEYS + .iter() + .any(|name| PyNode(key.clone()).is_text(name)) + { + continue; + } + let found: MacroPath = match self.own_entry(&key, &value)? { + Some(facts) => facts.macro_path.clone(), + None => entry_macro_path(&PyNode(key), &PyNode(value)), + }; + if let MacroPath::Found(path) = found { + return Err(Halt::Config(ConfigError::compile(format!( + "model config field '{}' does not allow macros", + path.join(".") + )))); + } + } + Ok(()) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/native_nodes.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/native_nodes.rs new file mode 100644 index 0000000000..72bc6832f6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/native_nodes.rs @@ -0,0 +1,205 @@ +//! Native authored header values read as the Python objects discovery projects them to. + +use std::cell::Cell; + +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; +use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; +use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; + +thread_local! { + /// Whether a read needed Python's `str` or `repr` of a value only Python can spell. + static INEXACT: Cell = const { Cell::new(false) }; +} + +/// One authored header value, or a mapping key, as its Python projection reads. +#[derive(Clone, Copy, Debug)] +pub(crate) enum NativeNode<'a> { + Value(&'a AuthoredValue), + Key(&'a str), +} + +/// Run `read` and report whether every Python spelling it needed was exact. +pub(crate) fn exactly(read: impl FnOnce() -> T) -> (T, bool) { + INEXACT.with(|flag| flag.set(false)); + let result: T = read(); + (result, !INEXACT.with(Cell::get)) +} + +fn inexact() -> String { + INEXACT.with(|flag| flag.set(true)); + String::new() +} + +/// A bare word's projection: Python's `_parse_word_value`. +fn word(text: &str) -> HeaderWord { + header_word_value(text) +} + +/// Whether integer `digits` (with an optional sign) is below zero, as `int(digits) < 0`. +fn negative(digits: &str) -> bool { + digits.starts_with('-') && digits[1..].bytes().any(|byte| byte != b'0') +} + +/// Python's `str(int(digits))`. +fn integer_text(digits: &str) -> String { + let unsigned: &str = digits.trim_start_matches(['+', '-']); + let trimmed: &str = unsigned.trim_start_matches('0'); + if trimmed.is_empty() { + return "0".to_owned(); + } + if negative(digits) { + format!("-{trimmed}") + } else { + trimmed.to_owned() + } +} + +impl<'a> NativeNode<'a> { + /// The text of a projected string, or `None`. + fn as_text(&self) -> Option<&'a str> { + match self { + Self::Key(key) => Some(key), + Self::Value(AuthoredValue::String(text)) => Some(text), + Self::Value(AuthoredValue::BareWord(text)) => match word(text) { + HeaderWord::Text => Some(text), + _ => None, + }, + Self::Value(_) => None, + } + } + + /// The digits of a projected integer, or `None`. + fn digits(&self) -> Option { + match self { + Self::Value(AuthoredValue::BareWord(text)) => match word(text) { + HeaderWord::Integer(digits) => Some(digits), + _ => None, + }, + _ => None, + } + } + + /// The owned authored value of a value node; a key projects as its string. + pub(crate) fn authored(&self) -> AuthoredValue { + match self { + Self::Key(key) => AuthoredValue::String((*key).to_owned()), + Self::Value(value) => (*value).clone(), + } + } +} + +impl AuthoredNode for NativeNode<'_> { + fn kind(&self) -> NodeKind { + let Self::Value(value) = self else { + return NodeKind::Str; + }; + match value { + AuthoredValue::Null => NodeKind::Null, + AuthoredValue::Boolean(flag) => NodeKind::Bool(*flag), + AuthoredValue::String(_) => NodeKind::Str, + AuthoredValue::List(_) => NodeKind::List, + AuthoredValue::Tuple(_) => NodeKind::Tuple, + AuthoredValue::Map(_) => NodeKind::Map, + AuthoredValue::BareWord(text) => match word(text) { + HeaderWord::Bool(flag) => NodeKind::Bool(flag), + HeaderWord::Null => NodeKind::Null, + HeaderWord::Integer(digits) => NodeKind::Int { + negative: negative(&digits), + }, + HeaderWord::Float(_) => NodeKind::Other, + HeaderWord::Text => NodeKind::Str, + }, + AuthoredValue::Set(_) + | AuthoredValue::TypedConstant(_) + | AuthoredValue::InlineSqlHook(_) + | AuthoredValue::NamedSqlHook(..) + | AuthoredValue::PythonHook(..) => NodeKind::Other, + } + } + + fn integer(&self) -> Option { + self.digits().and_then(|digits| digits.parse::().ok()) + } + + fn number(&self) -> Option { + if let Some(digits) = self.digits() { + return digits.parse::().ok(); + } + match self { + Self::Value(AuthoredValue::BareWord(text)) => match word(text) { + HeaderWord::Float(number) => Some(number), + _ => None, + }, + _ => None, + } + } + + fn text(&self) -> Option { + self.as_text().map(str::to_owned) + } + + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + self.as_text().map(read) + } + + fn is_text(&self, text: &str) -> bool { + self.as_text() == Some(text) + } + + fn python_str(&self) -> String { + if let Some(text) = self.as_text() { + return text.to_owned(); + } + if let Some(digits) = self.digits() { + return integer_text(&digits); + } + match self.kind() { + NodeKind::Null => "None".to_owned(), + NodeKind::Bool(true) => "True".to_owned(), + NodeKind::Bool(false) => "False".to_owned(), + _ => inexact(), + } + } + + fn python_repr(&self) -> String { + match self.kind() { + NodeKind::Null | NodeKind::Bool(_) | NodeKind::Int { .. } => self.python_str(), + _ => inexact(), + } + } + + fn is_date_like(&self) -> bool { + false + } + + fn items(&self) -> Vec { + match self { + Self::Value(AuthoredValue::List(items) | AuthoredValue::Tuple(items)) => { + items.iter().map(Self::Value).collect() + } + _ => Vec::new(), + } + } + + fn entries(&self) -> Vec<(Self, Self)> { + match self { + Self::Value(AuthoredValue::Map(entries)) => dict_entries(entries) + .into_iter() + .map(|(key, value)| (Self::Key(key), Self::Value(value))) + .collect(), + _ => Vec::new(), + } + } +} + +/// Entries as the Python dict built from them holds them: first position, last value. +pub(crate) fn dict_entries(entries: &[(String, AuthoredValue)]) -> Vec<(&str, &AuthoredValue)> { + let mut kept: Vec<(&str, &AuthoredValue)> = Vec::with_capacity(entries.len()); + for (key, value) in entries { + match kept.iter_mut().find(|(name, _)| *name == key.as_str()) { + Some(entry) => entry.1 = value, + None => kept.push((key, value)), + } + } + kept +} From b6476ae7871629f926795c9c6fd19831040ba471 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:38:10 +0100 Subject: [PATCH 21/28] perf: read compiled model text natively in post-analysis consumers Model facts share the model file's query and authored text instead of copying them. Compiled artifacts, metadata checks and resource SQL requests read a native model's name, path and SQL directly, and Python reads of a model's query and file text share one lazily built string. Header facts and their native node reader move to the discovery crate with Rust tests. --- .../src/compiled_project/models.rs | 13 +- .../src/compiled_project/tests/helpers.rs | 2 +- .../src/compiled_project/tests/test_store.rs | 2 +- .../src/model_files/_helpers/mod.rs | 1 + .../src/model_files/_helpers}/native_nodes.rs | 11 +- .../src/model_files/main/header_facts.rs | 99 ++++++++++++ .../src/model_files/main/mod.rs | 1 + .../src/model_files/models.rs | 59 ++++++++ .../src/model_files/tests.rs | 2 + .../src/model_files/tests/helpers.rs | 10 +- .../model_files/tests/test_header_facts.rs | 132 ++++++++++++++++ .../src/model_files/tests/test_types.rs | 19 +++ .../compile_command/artifact_batch.rs | 59 +++++--- .../compile_command/reuse_recording.rs | 39 ++++- .../_helpers/compile_command/target_writer.rs | 47 ++++-- .../compiled_project/compiled_model_view.rs | 53 ++++++- .../compiled_project/model_input_view.rs | 18 ++- .../_helpers/compiled_project/model_loop.rs | 6 +- .../compiled_project/model_loop_config.rs | 2 +- .../compiled_project/model_loop_header.rs | 6 +- .../compiled_project/model_loop_host.rs | 10 +- .../compiled_project/model_loop_prepass.rs | 143 +++++------------- .../_helpers/compiled_project/project.rs | 6 +- .../_helpers/discovery/model_file_record.rs | 38 ++++- .../_helpers/model_config/header_metadata.rs | 73 +-------- .../src/bindings/_helpers/model_config/mod.rs | 1 - .../model_config/model_config_builder.rs | 5 +- .../_helpers/semantic_checks/metadata.rs | 38 +++-- .../_helpers/semantic_checks/resource_sql.rs | 16 +- .../src/engine/_helpers/project_rows.rs | 6 +- 30 files changed, 647 insertions(+), 270 deletions(-) rename crates/{sqlbuild-python/src/bindings/_helpers/model_config => sqlbuild-discovery/src/model_files/_helpers}/native_nodes.rs (95%) create mode 100644 crates/sqlbuild-discovery/src/model_files/main/header_facts.rs create mode 100644 crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs index a3423f5705..3f947b3376 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/models.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -1,6 +1,7 @@ //! Facts a compile's producers retain about each resource, in production order. use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; use serde_json::Value; @@ -17,11 +18,11 @@ pub struct CompiledModelFacts { pub name: String, pub relative_path: String, /// SQL after variable, declaration and macro expansion. - pub query_sql: String, + pub query_sql: Arc, /// The authored query below the `MODEL (...)` header. - pub authored_query_sql: String, + pub authored_query_sql: Arc, /// The whole authored file. - pub authored_sql: String, + pub authored_sql: Arc, pub config: ModelConfigFacts, pub references: Vec, pub schema: Option, @@ -210,9 +211,9 @@ pub struct SplicedSql { pub struct PlainModel { pub name: String, pub relative_path: String, - pub query_sql: String, - pub authored_query_sql: String, - pub authored_sql: String, + pub query_sql: Arc, + pub authored_query_sql: Arc, + pub authored_sql: Arc, pub config: ModelConfigFacts, /// The attached schema entry; present when the header declares description, columns or audits. pub schema: Option, diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs index 1cbd5c6831..6ce0157bf8 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs @@ -3,7 +3,7 @@ use crate::compiled_project::models::{CompiledModelFacts, ModelAnalysisFact}; pub(super) fn model(name: &str, query_sql: &str) -> CompiledModelFacts { CompiledModelFacts { name: name.to_owned(), - query_sql: query_sql.to_owned(), + query_sql: query_sql.into(), ..CompiledModelFacts::default() } } diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs index d3041bb078..e236eaa94e 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs @@ -51,7 +51,7 @@ fn given_recorded_models_when_reading_then_order_and_latest_facts_are_kept() { .map(|facts| facts.name.as_str()) .collect(); let query_sql: Option<&str> = - model_facts(&project, "orders").map(|facts| facts.query_sql.as_str()); + model_facts(&project, "orders").map(|facts| &*facts.query_sql); assert_eq!(order, test_case.expected_order, "{}", test_case.description); assert_eq!( diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs index a39813b4a5..15d2cb619d 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs @@ -1,2 +1,3 @@ +pub(crate) mod native_nodes; pub(crate) mod output_columns; pub(crate) mod parsing; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/native_nodes.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs similarity index 95% rename from crates/sqlbuild-python/src/bindings/_helpers/model_config/native_nodes.rs rename to crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs index 72bc6832f6..e164052c14 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/native_nodes.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs @@ -1,6 +1,7 @@ //! Native authored header values read as the Python objects discovery projects them to. use std::cell::Cell; +use std::collections::HashMap; use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; @@ -195,10 +196,14 @@ impl AuthoredNode for NativeNode<'_> { /// Entries as the Python dict built from them holds them: first position, last value. pub(crate) fn dict_entries(entries: &[(String, AuthoredValue)]) -> Vec<(&str, &AuthoredValue)> { let mut kept: Vec<(&str, &AuthoredValue)> = Vec::with_capacity(entries.len()); + let mut positions: HashMap<&str, usize> = HashMap::with_capacity(entries.len()); for (key, value) in entries { - match kept.iter_mut().find(|(name, _)| *name == key.as_str()) { - Some(entry) => entry.1 = value, - None => kept.push((key, value)), + match positions.get(key.as_str()) { + Some(&index) => kept[index].1 = value, + None => { + positions.insert(key, kept.len()); + kept.push((key, value)); + } } } kept diff --git a/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs b/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs new file mode 100644 index 0000000000..899ea78503 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs @@ -0,0 +1,99 @@ +//! What a model's native header values show, read as their Python projection reads. + +use sqlbuild_model_config::config_presence::main::blocks_config_reuse::blocks_config_reuse; +use sqlbuild_model_config::config_presence::main::contains_template::contains_template; +use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; +use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; +use sqlbuild_model_config::header_metadata::models::{ParsedAudit, ParsedColumn}; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::model_files::_helpers::native_nodes::{NativeNode, dict_entries, exactly}; +use crate::model_files::models::{EntryFacts, HeaderFacts, NativeAudit, NativeColumn}; + +const COLUMNS_KEY: &str = "columns"; + +static NULL_VALUE: AuthoredValue = AuthoredValue::Null; + +/// The header facts of one model's native authored values; `path` names it in parse messages. +#[must_use] +pub fn header_facts(values: &[(String, AuthoredValue)], path: &str) -> HeaderFacts { + let entries: Vec<(&str, &AuthoredValue)> = dict_entries(values); + let facts: Vec<(String, EntryFacts)> = entries + .iter() + .map(|&(key, value)| { + let node: NativeNode<'_> = NativeNode::Value(value); + ( + key.to_owned(), + EntryFacts { + template: contains_template(&node), + macro_path: entry_macro_path(&NativeNode::Key(key), &node), + blocks_reuse: blocks_config_reuse(&node), + }, + ) + }) + .collect(); + let columns: Option> = entries + .iter() + .find(|(key, _)| *key == COLUMNS_KEY) + .map(|&(_, value)| value) + .filter(|value| **value != AuthoredValue::Null) + .and_then(|value| { + let (metadata, exact) = exactly(|| { + parse_header_metadata( + &NativeNode::Value(value), + &NativeNode::Value(&NULL_VALUE), + path, + ) + }); + let columns: Vec>> = + metadata.columns.ok().filter(|_| exact)?; + Some(columns.iter().map(native_column).collect()) + }); + HeaderFacts { + entries: facts, + columns, + } +} + +fn text(node: Option<&NativeNode<'_>>) -> Option { + node.and_then(AuthoredNode::text) +} + +fn flag(node: Option<&NativeNode<'_>>) -> Option { + match node.map(AuthoredNode::kind) { + Some(NodeKind::Bool(value)) => Some(value), + _ => None, + } +} + +/// A parsed column's leaves as the projected text, flags and counts. +fn native_column(column: &ParsedColumn>) -> NativeColumn { + NativeColumn { + name: column.name.text().unwrap_or_default(), + column_type: text(column.column_type.as_ref()), + nullable: flag(column.nullable.as_ref()), + description: text(column.description.as_ref()), + audits: column.audits.iter().map(native_audit).collect(), + migrate_from: text(column.migrate_from.as_ref()), + } +} + +fn native_audit(audit: &ParsedAudit>) -> NativeAudit { + NativeAudit { + definition_name: audit.definition_name.text().unwrap_or_default(), + arguments: audit + .arguments + .iter() + .map(|(key, value)| (key.text().unwrap_or_default(), value.authored())) + .collect(), + name: text(audit.name.as_ref()), + description: text(audit.description.as_ref()), + severity: text(audit.severity.as_ref()), + run_scope: text(audit.run_scope.as_ref()), + always_run: flag(audit.always_run.as_ref()), + thresholds: audit.thresholds, + minimum_samples: audit.minimum_samples.as_ref().map(NativeNode::authored), + evidence_limit: audit.evidence_limit.as_ref().map(NativeNode::authored), + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/main/mod.rs b/crates/sqlbuild-discovery/src/model_files/main/mod.rs index 93bf47feb0..eb6f515e35 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/mod.rs @@ -1,3 +1,4 @@ pub mod discover_model_files; +pub mod header_facts; pub mod model_file_stem; pub mod parse_model_text; diff --git a/crates/sqlbuild-discovery/src/model_files/models.rs b/crates/sqlbuild-discovery/src/model_files/models.rs index f23ad90701..a909c20c7f 100644 --- a/crates/sqlbuild-discovery/src/model_files/models.rs +++ b/crates/sqlbuild-discovery/src/model_files/models.rs @@ -1,6 +1,8 @@ //! Native results for SQL model files. use crate::models::LineColumnSpan; +use sqlbuild_model_config::config_presence::models::MacroPath; +use sqlbuild_model_config::header_metadata::models::ParsedThresholds; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// What the Python facade asked for, with the header key sets Python owns during dual running. @@ -22,3 +24,60 @@ pub struct DiscoveredModelFile { pub output_column_locations: Vec<(String, LineColumnSpan)>, pub query_sql: String, } + +/// What one top-level header entry holds, as the config checks walk it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EntryFacts { + /// Whether a string in the value holds a `${...}` template. + pub template: bool, + /// Where `first_macro_path` finds a macro call in this entry. + pub macro_path: MacroPath, + /// Whether the value keeps the model from reusing another model's config. + pub blocks_reuse: bool, +} + +/// One model's header facts, read from its native authored values. +#[derive(Clone, Debug, PartialEq)] +pub struct HeaderFacts { + /// Each header key in the projected dict's order, with its facts. + pub entries: Vec<(String, EntryFacts)>, + /// The header's columns, when they parse exactly and without an error. + pub columns: Option>, +} + +impl HeaderFacts { + /// The facts of `key`'s value in the header. + #[must_use] + pub fn entry(&self, key: &str) -> Option<&EntryFacts> { + self.entries + .iter() + .find(|(name, _)| name == key) + .map(|(_, facts)| facts) + } +} + +/// One MODEL column parsed from native authored values. +#[derive(Clone, Debug, PartialEq)] +pub struct NativeColumn { + pub name: String, + pub column_type: Option, + pub nullable: Option, + pub description: Option, + pub audits: Vec, + pub migrate_from: Option, +} + +/// One audit instance parsed from native authored values. +#[derive(Clone, Debug, PartialEq)] +pub struct NativeAudit { + pub definition_name: String, + pub arguments: Vec<(String, AuthoredValue)>, + pub name: Option, + pub description: Option, + pub severity: Option, + pub run_scope: Option, + pub always_run: Option, + pub thresholds: Option, + pub minimum_samples: Option, + pub evidence_limit: Option, +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests.rs b/crates/sqlbuild-discovery/src/model_files/tests.rs index 3e8b418280..ae2b46023d 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_discover_model_files.rs"] mod discover_model_files; +#[path = "tests/test_header_facts.rs"] +mod header_facts; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_model_file_stem.rs"] diff --git a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs index fc4979cb4d..2faafbd2e6 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs @@ -1,5 +1,6 @@ +use crate::model_files::main::header_facts::header_facts; use crate::model_files::main::parse_model_text::parse_model_text; -use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; +use crate::model_files::models::{DiscoveredModelFile, HeaderFacts, ModelFileOptions}; use crate::model_files::tests::test_types::{Located, ModelSummary, Span}; use crate::models::LineColumnSpan; @@ -73,3 +74,10 @@ fn owned(locations: &[(&str, Span)]) -> Vec { pub(super) fn failed(message: &str, help: Option<&str>) -> ModelSummary { Err((message.to_owned(), help.map(str::to_owned))) } + +/// The header facts of one model file parsed as `/project/models/orders.sql`. +pub(super) fn parsed_header_facts(contents: &str) -> HeaderFacts { + let model: DiscoveredModelFile = parse_model_text(FILE_PATH, contents.to_owned(), &options()) + .unwrap_or_else(|failure| panic!("{}", failure.message)); + header_facts(&model.header_values, "models/orders.sql") +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs new file mode 100644 index 0000000000..d18d94af98 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs @@ -0,0 +1,132 @@ +use sqlbuild_model_config::config_presence::models::MacroPath; + +use crate::model_files::models::{HeaderFacts, NativeColumn}; +use crate::model_files::tests::helpers::parsed_header_facts; +use crate::model_files::tests::test_types::{ColumnSummary, HeaderFactsTestCase}; + +fn column_summary( + column: &NativeColumn, +) -> ( + String, + Option, + Option, + Option, + Vec, +) { + ( + column.name.clone(), + column.column_type.clone(), + column.nullable, + column.description.clone(), + column + .audits + .iter() + .map(|audit| audit.definition_name.clone()) + .collect(), + ) +} + +fn expected_summary( + column: &ColumnSummary, +) -> ( + String, + Option, + Option, + Option, + Vec, +) { + let (name, column_type, nullable, description, audits) = column; + ( + (*name).to_owned(), + column_type.map(str::to_owned), + *nullable, + description.map(str::to_owned), + audits.iter().map(|audit| (*audit).to_owned()).collect(), + ) +} + +#[test] +fn given_model_headers_when_reading_native_facts_then_they_match_the_python_projection() { + let test_cases = [ + HeaderFactsTestCase { + description: "plain columns parse with their audits and block nothing", + contents: "MODEL (\n description 'Orders',\n materialized table,\n columns (\n id (type INTEGER, nullable false, audits [not_null, accepted_values (values [1, 2])]),\n amount (type DOUBLE, description 'Order amount'),\n ),\n);\nSELECT 1 AS id, 2.0 AS amount", + expected_entries: vec![ + ("description", false, None, false), + ("materialized", false, None, false), + ("columns", false, None, false), + ], + expected_columns: Some(vec![ + ( + "id", + Some("INTEGER"), + Some(false), + None, + vec!["not_null", "accepted_values"], + ), + ("amount", Some("DOUBLE"), None, Some("Order amount"), vec![]), + ]), + }, + HeaderFactsTestCase { + description: "templates and macro calls are found per entry with their path", + contents: "MODEL (\n description '${env.ORDERS}',\n columns (\n id (description '@cents(id)'),\n ),\n);\nSELECT 1 AS id", + expected_entries: vec![ + ("description", true, None, true), + ( + "columns", + false, + Some(vec!["columns", "id", "description"]), + true, + ), + ], + expected_columns: Some(vec![("id", None, None, Some("@cents(id)"), vec![])]), + }, + HeaderFactsTestCase { + description: "columns that fail to parse leave the Python parse to raise the error", + contents: "MODEL (\n columns (\n id (type INTEGER),\n ID (type INTEGER),\n ),\n);\nSELECT 1 AS id", + expected_entries: vec![("columns", false, None, false)], + expected_columns: None, + }, + HeaderFactsTestCase { + description: "a nullable column audited not_null is a parse error", + contents: "MODEL (\n columns (\n id (nullable true, audits [not_null]),\n ),\n);\nSELECT 1 AS id", + expected_entries: vec![("columns", false, None, false)], + expected_columns: None, + }, + ]; + for test_case in test_cases { + let facts: HeaderFacts = parsed_header_facts(test_case.contents); + let entries: Vec<(&str, bool, Option>, bool)> = facts + .entries + .iter() + .map(|(key, entry)| { + ( + key.as_str(), + entry.template, + match &entry.macro_path { + MacroPath::Found(path) => Some(path.iter().map(String::as_str).collect()), + MacroPath::Absent => None, + }, + entry.blocks_reuse, + ) + }) + .collect(); + assert_eq!( + entries, test_case.expected_entries, + "{}", + test_case.description + ); + assert_eq!( + facts + .columns + .as_ref() + .map(|columns| columns.iter().map(column_summary).collect::>()), + test_case + .expected_columns + .as_ref() + .map(|columns| columns.iter().map(expected_summary).collect::>()), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs index 45db919f61..9596a77022 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs @@ -20,3 +20,22 @@ pub(super) struct DeepModelHeaderTestCase { pub(super) depth: usize, pub(super) expected_summary: ModelSummary, } + +/// A model header and what its native header facts must read, as Python reads the header. +pub(super) struct HeaderFactsTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + /// `(key, template, macro path, blocks reuse)` of each header entry, in dict order. + pub(super) expected_entries: Vec<(&'static str, bool, Option>, bool)>, + /// `(name, type, nullable, description, audit definitions)` of each column, or `None`. + pub(super) expected_columns: Option>, +} + +/// `(name, type, nullable, description, audit definitions)` of one parsed column. +pub(super) type ColumnSummary = ( + &'static str, + Option<&'static str>, + Option, + Option<&'static str>, + Vec<&'static str>, +); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs index 9724ba59eb..fae39e9dc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_command::reuse_recording::{ - ArtifactRecorder, artifact_written, + ArtifactRecorder, artifact_written, artifact_written_at, }; use crate::bindings::_helpers::compile_driver::compile_timings::record_compile_timing; use crate::bindings::_helpers::compile_outputs::artifacts::write_compiled_artifacts; @@ -17,6 +17,31 @@ pub(crate) struct ArtifactBatch<'a> { files: Vec<(PathBuf, Vec)>, } +/// `contents` with platform line endings. +fn platform_lines(contents: Vec) -> Vec { + if cfg!(windows) { + contents + .iter() + .flat_map(|byte| { + if *byte == b'\n' { + vec![b'\r', b'\n'] + } else { + vec![*byte] + } + }) + .collect() + } else { + contents + } +} + +/// `sql` with trailing whitespace replaced by one newline. +fn sql_artifact(sql: &str) -> Vec { + let trimmed: &str = + sql.trim_end_matches(sqlbuild_core::text::main::python_isspace::python_isspace); + format!("{trimmed}\n").into_bytes() +} + impl<'a> ArtifactBatch<'a> { pub(crate) fn new(check_existing: bool, recorder: ArtifactRecorder<'a>) -> Self { Self { @@ -28,20 +53,7 @@ impl<'a> ArtifactBatch<'a> { /// Queue one artifact with platform line endings, noting its bytes for compile reuse. pub(crate) fn queue(&mut self, path: &Bound<'_, PyAny>, contents: Vec) -> PyResult<()> { - let contents: Vec = if cfg!(windows) { - contents - .iter() - .flat_map(|byte| { - if *byte == b'\n' { - vec![b'\r', b'\n'] - } else { - vec![*byte] - } - }) - .collect() - } else { - contents - }; + let contents: Vec = platform_lines(contents); artifact_written(self.recorder, path, &contents)?; self.files.push((path.extract()?, contents)); Ok(()) @@ -49,9 +61,20 @@ impl<'a> ArtifactBatch<'a> { /// Queue `sql` with trailing whitespace replaced by one newline. pub(crate) fn queue_sql(&mut self, path: &Bound<'_, PyAny>, sql: &str) -> PyResult<()> { - let trimmed: &str = - sql.trim_end_matches(sqlbuild_core::text::main::python_isspace::python_isspace); - self.queue(path, format!("{trimmed}\n").into_bytes()) + self.queue(path, sql_artifact(sql)) + } + + /// `queue_sql` at a native path. + pub(crate) fn queue_sql_at( + &mut self, + py: Python<'_>, + path: PathBuf, + sql: &str, + ) -> PyResult<()> { + let contents: Vec = platform_lines(sql_artifact(sql)); + artifact_written_at(py, self.recorder, &path, &contents)?; + self.files.push((path, contents)); + Ok(()) } /// Write every queued artifact natively and empty the queue. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 1b80e2cfef..889e29ca16 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -1,7 +1,7 @@ //! What one full compile printed and left in each artifact, and storing it for compile reuse. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::{Mutex, PoisonError}; use pyo3::exceptions::{PyException, PyOSError}; @@ -107,7 +107,8 @@ pub(crate) type ArtifactRecorder<'a> = Option<&'a Py>; /// `os.path.abspath(path)`, computed natively on POSIX for UTF-8 paths and working directories. fn absolute(path: &Bound<'_, PyAny>) -> PyResult { if cfg!(unix) - && let Some(found) = native_absolute(path) + && let Ok(native) = path.extract::() + && let Some(found) = native_absolute(&native) { return Ok(found); } @@ -116,10 +117,19 @@ fn absolute(path: &Bound<'_, PyAny>) -> PyResult { .extract() } -fn native_absolute(path: &Bound<'_, PyAny>) -> Option { - let Ok(path) = path.extract::() else { - return None; - }; +/// `os.path.abspath(path)` of a native path. +fn absolute_path(py: Python<'_>, path: &Path) -> PyResult { + if cfg!(unix) + && let Some(found) = native_absolute(path) + { + return Ok(found); + } + imported(py, "os.path", "abspath")? + .call1((path,))? + .extract() +} + +fn native_absolute(path: &Path) -> Option { let Ok(cwd) = std::env::current_dir() else { return None; }; @@ -152,6 +162,23 @@ pub(crate) fn artifact_written( Ok(()) } +/// `artifact_written` for a native path. +pub(crate) fn artifact_written_at( + py: Python<'_>, + recorder: ArtifactRecorder<'_>, + path: &Path, + contents: &[u8], +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder.get().record( + absolute_path(py, path)?, + RecordedArtifact::Written(artifact_digest(contents)), + ); + Ok(()) +} + /// Note an earlier artifact kept as-is after its size and mtime were validated. pub(crate) fn artifact_kept( recorder: ArtifactRecorder<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 230a79ca3e..c65aeee020 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -1,7 +1,7 @@ //! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{ @@ -35,6 +35,7 @@ use crate::bindings::_helpers::compile_outputs::artifacts::{ use crate::bindings::_helpers::compile_outputs::runtime_target::{ compiled_sql_test_path, planned_sql_test_path, }; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_entries::plan_output::sql_test_model_chain_names_by_key; @@ -85,6 +86,27 @@ fn model_output_path<'py>(relative_path: &Bound<'py, PyAny>) -> PyResult PathBuf { + let mut path: PathBuf = compiled.to_path_buf(); + let mut segments = relative_path + .split('/') + .filter(|segment| !segment.is_empty()) + .peekable(); + if segments.peek() != Some(&MODELS_DIR) { + path.push(MODELS_DIR); + } + for segment in segments { + path.push(segment); + } + path +} + +/// Each path object as a native path. +fn extracted(paths: &[Bound<'_, PyAny>]) -> PyResult> { + paths.iter().map(|path| path.extract()).collect() +} + /// `relative_path` under `functions//`, as a `.sql` file. fn function_output_path<'py>( relative_path: &Bound<'py, PyAny>, @@ -152,13 +174,9 @@ fn function_sql(adapter: &Bound<'_, PyAny>, function: &Bound<'_, PyAny>) -> PyRe } /// Delete compiled files no write group manages. -fn remove_stale(target_dir: &Bound<'_, PyAny>, managed: &[Bound<'_, PyAny>]) -> PyResult<()> { +fn remove_stale(target_dir: &Bound<'_, PyAny>, managed: Vec) -> PyResult<()> { let py: Python<'_> = target_dir.py(); let compiled: PathBuf = joined(target_dir, &[COMPILED_DIR])?.extract()?; - let managed: Vec = managed - .iter() - .map(|path| path.extract()) - .collect::>()?; record_compile_timing("stale_traversal_ms", || { remove_stale_artifacts(py, compiled, managed).map(|_| ()) }) @@ -289,7 +307,7 @@ fn write_planned_target<'py>( } batch.flush(py)?; if remove_stale_files { - remove_stale(target_dir, &managed)?; + remove_stale(target_dir, extracted(&managed)?)?; } write_manifest(target_dir, manifest, None)?; written_target( @@ -573,9 +591,18 @@ pub(crate) fn write_static_target<'py>( let remove_stale_files: bool = prepared_target(target_dir)?; let mut batch: ArtifactBatch = ArtifactBatch::new(remove_stale_files, prepared.recorder); let mut managed: Vec> = Vec::new(); + let mut managed_models: Vec = Vec::new(); let compiled: Bound<'py, PyAny> = joined(target_dir, &[COMPILED_DIR])?; + let compiled_dir: PathBuf = compiled.extract()?; for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; + if let Ok(native) = model.cast::() { + let (relative_path, query_sql) = native.get().output_texts(py)?; + let path: PathBuf = native_model_output_path(&compiled_dir, &relative_path); + batch.queue_sql_at(py, path.clone(), &query_sql)?; + managed_models.push(path); + continue; + } let path: Bound<'py, PyAny> = compiled.div(model_output_path(&native_field(&model, "relative_path")?)?)?; batch.queue_sql( @@ -616,7 +643,9 @@ pub(crate) fn write_static_target<'py>( write_static_tests(target_dir, (adapter, project), prepared, &mut batch)?; managed.extend(test_paths); if remove_stale_files { - remove_stale(target_dir, &managed)?; + let mut paths: Vec = managed_models; + paths.extend(extracted(&managed)?); + remove_stale(target_dir, paths)?; } write_manifest(target_dir, manifest, prepared.recorder)?; written_target( @@ -688,7 +717,7 @@ pub(crate) fn publish_staged_target<'py>( managed.push(path_class(py)?.call1((path,))?); } if existed { - remove_stale(target_dir, &managed)?; + remove_stale(target_dir, extracted(&managed)?)?; } } write_manifest(target_dir, manifest, recorder)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index 2cd45e01a8..7de3aeda56 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -18,8 +18,8 @@ use crate::bindings::_helpers::compiled_project::compiled_model_analysis::{ AnalysisRecord, ProofArgument, SessionArgument, analysis_record, }; use crate::bindings::_helpers::compiled_project::model_input_view::{ - InputViewClasses, ModelInputRecord, model_relation_key_row, read_input, read_input_with_owner, - references, rejected_opt_out, schema_entry, view_classes, + InputViewClasses, ModelInputRecord, model_relation_key_row, query_sql_text, read_input, + read_input_with_owner, references, rejected_opt_out, schema_entry, view_classes, }; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; @@ -409,8 +409,8 @@ impl NativeCompiledModel { let key = keyed .then(|| -> PyResult { Ok(SqlRuleKey { - authored_sql: facts.authored_sql.clone(), - query_sql: facts.query_sql.clone(), + authored_sql: facts.authored_sql.to_string(), + query_sql: facts.query_sql.to_string(), contract: compiled_of(record)? .analysis .proof @@ -427,6 +427,47 @@ impl NativeCompiledModel { }) } + /// The model's name, `str(relative_path)` and authored file text. + pub(crate) fn file_texts(&self, py: Python<'_>) -> PyResult<(String, String, Arc)> { + self.read(py, |record, py| { + let model_file: &NativeModelFile = record.model_file.get(); + Ok(( + model_name(record), + model_file.relative_text(py)?, + Arc::clone(model_file.contents_text()), + )) + }) + } + + /// The name, compiled query and authored file of the model, and whether it calls a function. + pub(crate) fn metadata_texts( + &self, + py: Python<'_>, + function_kinds: &[String], + ) -> PyResult<(String, String, String, bool)> { + self.read(py, |record, _| { + Ok(( + model_name(record), + record.query_sql.to_string(), + record.model_file.get().contents_text().to_string(), + record + .references + .iter() + .any(|(kind, _, _, _)| function_kinds.contains(kind)), + )) + }) + } + + /// `relative_path.as_posix()` and the compiled query, which compiled artifacts are written from. + pub(crate) fn output_texts(&self, py: Python<'_>) -> PyResult<(String, Arc)> { + self.read(py, |record, py| { + Ok(( + record.model_file.get().posix_path(py)?, + Arc::clone(&record.query_sql), + )) + }) + } + /// The model's authored file text and expanded query, from its recorded facts. pub(crate) fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { let name: String = self.name(py)?; @@ -434,7 +475,7 @@ impl NativeCompiledModel { let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") })?; - Ok((facts.authored_sql.clone(), facts.query_sql.clone())) + Ok((facts.authored_sql.to_string(), facts.query_sql.to_string())) } fn model_file( @@ -518,7 +559,7 @@ impl NativeCompiledModel { #[getter] pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| Ok(PyString::new(py, &record.query_sql))) + self.read(py, |record, py| Ok(query_sql_text(py, record))) } #[getter] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index 41b4f0d39f..9b56f17be5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; +use crate::bindings::_helpers::discovery::model_file_record::{NativeModelFile, projected_text}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::SpanRow; @@ -89,6 +89,7 @@ pub(crate) struct ModelInputRecord { /// A model input's Python attribute projections, each built once on first access. pub(crate) struct InputProjections { + query_sql: PyOnceLock>, references: PyOnceLock>, schema_entry: PyOnceLock>, sql_expansion: PyOnceLock>, @@ -98,6 +99,7 @@ pub(crate) struct InputProjections { impl Default for InputProjections { fn default() -> Self { Self { + query_sql: PyOnceLock::new(), references: PyOnceLock::new(), schema_entry: PyOnceLock::new(), sql_expansion: PyOnceLock::new(), @@ -182,7 +184,7 @@ impl NativeModelInput { #[getter] pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { - self.read(py, |record, py| Ok(PyString::new(py, &record.query_sql))) + self.read(py, |record, py| Ok(query_sql_text(py, record))) } #[getter] @@ -319,6 +321,18 @@ impl NativeModelInput { } } +/// The compiled query as the Python string every reader shares; the authored query's when equal. +pub(crate) fn query_sql_text<'py>( + py: Python<'py>, + record: &ModelInputRecord, +) -> Bound<'py, PyString> { + let model_file: &NativeModelFile = record.model_file.get(); + if Arc::ptr_eq(&record.query_sql, model_file.query_text()) { + return model_file.query_sql(py); + } + projected_text(py, &record.projections.query_sql, &record.query_sql) +} + /// The authored-to-expanded passes, when interpolation left the authored query unchanged. pub(crate) fn authored_expansion( classes: &InputViewClasses<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index 7c50ad165e..72023345ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -35,7 +35,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_opt_out::{ OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, }; use crate::bindings::_helpers::compiled_project::model_loop_prepass::{ - HeaderFacts, all_header_facts, + NativeHeader, all_header_facts, }; use crate::bindings::_helpers::compiled_project::model_loop_python_hooks::{ HookClasses, HookSignatures, @@ -89,6 +89,7 @@ use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::v use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_attachments::functions::models::KnownResources; use sqlbuild_attachments::test_targets::models::TargetCatalog; +use sqlbuild_discovery::model_files::models::HeaderFacts; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; @@ -97,7 +98,6 @@ use std::sync::Arc; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; -use sqlbuild_sqltext::compiler::models::AuthoredValue; const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; @@ -395,7 +395,7 @@ impl NativeModelLoop { let py: Python<'py> = slf.py(); let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; let mut models: Vec = prepared_rows(py, &classes, model_files)?; - let headers: Vec, String)>> = models + let headers: Vec> = models .iter() .map(|row| { row.model_file diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs index 7a2d5a9b44..793fdc0948 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs @@ -12,11 +12,11 @@ use sqlbuild_model_config::config_presence::main::mutable_config_value::mutable_ use sqlbuild_model_config::model_validation::models::{RetentionOverride, TableTypeOverride}; use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, config_error}; -use crate::bindings::_helpers::compiled_project::model_loop_prepass::HeaderFacts; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::model_config_builder::{ BuiltConfig, ConfigBuild, NativeModelConfigBuilder, }; +use sqlbuild_discovery::model_files::models::HeaderFacts; const COLUMNS_KEY: &str = "columns"; const DESCRIPTION_KEY: &str = "description"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index 391837dc0a..e8234494cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -22,7 +22,6 @@ use crate::bindings::_helpers::compiled_project::model_loop_contracts::{ EnumColumns, dynamic_family_objects, enum_columns, factory_audits, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; -use crate::bindings::_helpers::compiled_project::model_loop_prepass::HeaderFacts; use crate::bindings::_helpers::compiled_project::model_loop_schema::{ DeclaredColumns, ModelSchema, merged_columns, schema_enums, }; @@ -32,10 +31,11 @@ use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; use crate::bindings::_helpers::model_config::config_errors::native_config_error; use crate::bindings::_helpers::model_config::header_metadata::{ - HeaderColumn, HeaderColumns, NativeColumn, NativeColumnSpec, audit_instances, - native_schema_column, parsed_audit_identity, + HeaderColumn, HeaderColumns, NativeColumnSpec, audit_instances, native_schema_column, + parsed_audit_identity, }; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_discovery::model_files::models::{HeaderFacts, NativeColumn}; const DESCRIPTION_KEY: &str = "description"; const COLUMNS_KEY: &str = "columns"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index a45fc59541..dcb8164d19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -17,7 +17,6 @@ use crate::bindings::_helpers::compiled_project::model_loop_header::AttachedHead use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ reference_fact, stripped_values, }; -use crate::bindings::_helpers::compiled_project::model_loop_prepass::HeaderFacts; use crate::bindings::_helpers::compiled_project::project::{ CONTRACT_KEY, INCREMENTAL_MODE_KEY, materialization, text_value, }; @@ -31,6 +30,7 @@ use crate::bindings::_helpers::render::declaration_expansion::{ }; use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::ExpansionOutput; +use sqlbuild_discovery::model_files::models::HeaderFacts; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; @@ -222,7 +222,7 @@ pub(crate) fn finished_record<'py>( name: name.clone(), model_file, config: config.unbind(), - query_sql: query, + query_sql: Arc::clone(&query), macro_source_sql: macro_source, references: finished.references.clone(), schema, @@ -254,9 +254,9 @@ pub(crate) fn finished_record<'py>( let facts: CompiledModelFacts = plain_model_facts(PlainModel { name, relative_path, - query_sql: finished.query_sql, - authored_query_sql: authored_query_sql.to_string(), - authored_sql: authored_sql.to_string(), + query_sql: query, + authored_query_sql, + authored_sql, config: ModelConfigFacts { values, encoding_error, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs index 3b44c4b705..8df87d7176 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs @@ -1,132 +1,52 @@ -//! What each model's native header values show before the ordered loop, read off the GIL. +//! Each model's native header facts, read before the ordered loop on a fixed pool off the GIL. //! //! The ordered loop uses a fact only for a config value that is the header's own object, so a //! value that path defaults, templates or hooks replaced is still read from Python. use std::sync::{Arc, OnceLock}; -use pyo3::prelude::PyAnyMethods; -use pyo3::prelude::{Bound, PyAny, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; use rayon::{ThreadPool, ThreadPoolBuilder}; -use sqlbuild_model_config::config_presence::main::blocks_config_reuse::blocks_config_reuse; -use sqlbuild_model_config::config_presence::main::contains_template::contains_template; -use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; -use sqlbuild_model_config::config_presence::models::MacroPath; -use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; +use sqlbuild_discovery::model_files::main::header_facts::header_facts; +use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; use sqlbuild_sqltext::compiler::models::AuthoredValue; -use crate::bindings::_helpers::model_config::header_metadata::NativeColumn; -use crate::bindings::_helpers::model_config::native_nodes::{NativeNode, dict_entries, exactly}; - -const COLUMNS_KEY: &str = "columns"; /// The compile pools' fixed worker count. const PREPASS_WORKERS: usize = 4; const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; -static NULL_VALUE: AuthoredValue = AuthoredValue::Null; static POOL: OnceLock> = OnceLock::new(); -/// What one top-level header entry holds, as the config checks walk it. -pub(crate) struct EntryFacts { - /// Whether a string in the value holds a `${...}` template. - pub(crate) template: bool, - /// Where `first_macro_path` finds a macro call in this entry. - pub(crate) macro_path: MacroPath, - /// Whether the value keeps the model from reusing another model's config. - pub(crate) blocks_reuse: bool, -} - -/// One model's header facts, read from its native authored values. -pub(crate) struct HeaderFacts { - /// Each header key in the projected dict's order, with its facts. - entries: Vec<(String, EntryFacts)>, - /// The header's columns, when they parse without an error. - pub(crate) columns: Option>, -} - -impl HeaderFacts { - /// The facts of `key`'s value in the header. - pub(crate) fn entry(&self, key: &str) -> Option<&EntryFacts> { - self.entries - .iter() - .find(|(name, _)| name == key) - .map(|(_, facts)| facts) - } - - /// The facts of a config entry, when its value is the header's own value of `key`. - pub(crate) fn own_entry<'py>( - &self, - header: &Bound<'py, PyDict>, - key: &Bound<'py, PyAny>, - value: &Bound<'py, PyAny>, - ) -> PyResult> { - let Ok(name) = key.downcast::() else { - return Ok(None); - }; - let Ok(name) = name.to_str() else { - return Ok(None); - }; - let Some(facts) = self.entry(name) else { - return Ok(None); - }; - Ok(header - .get_item(name)? - .filter(|own| own.is(value)) - .map(|_| facts)) - } -} +/// A model's native header values and the relative path parse messages name. +pub(crate) type NativeHeader = (Arc<[(String, AuthoredValue)]>, String); -/// The header facts of one model's native authored values; `path` names it in parse messages. -pub(crate) fn header_facts(values: &[(String, AuthoredValue)], path: &str) -> HeaderFacts { - let entries: Vec<(String, EntryFacts)> = dict_entries(values) - .into_iter() - .map(|(key, value)| { - let node: NativeNode<'_> = NativeNode::Value(value); - ( - key.to_owned(), - EntryFacts { - template: contains_template(&node), - macro_path: entry_macro_path(&NativeNode::Key(key), &node), - blocks_reuse: blocks_config_reuse(&node), - }, - ) - }) - .collect(); - let columns: Option> = dict_entries(values) - .into_iter() - .find(|(key, _)| *key == COLUMNS_KEY) - .map(|(_, value)| value) - .filter(|value| **value != AuthoredValue::Null) - .and_then(|value| { - let (metadata, exact) = exactly(|| { - parse_header_metadata( - &NativeNode::Value(value), - &NativeNode::Value(&NULL_VALUE), - path, - ) - }); - let columns = metadata.columns.ok().filter(|_| exact)?; - Some(columns.iter().map(NativeColumn::parsed).collect()) - }); - HeaderFacts { entries, columns } +/// The facts of a config entry, when its value is the header's own value of `key`. +pub(crate) fn own_entry<'a, 'py>( + facts: &'a HeaderFacts, + header: &Bound<'py, PyDict>, + (key, value): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let Ok(name) = key.downcast::() else { + return Ok(None); + }; + let Ok(name) = name.to_str() else { + return Ok(None); + }; + let Some(entry) = facts.entry(name) else { + return Ok(None); + }; + Ok(header + .get_item(name)? + .filter(|own| own.is(value)) + .map(|_| entry)) } /// Each model's header facts, computed on the compile pool; `None` where no native header exists. pub(crate) fn all_header_facts( - headers: &[Option<(Arc<[(String, AuthoredValue)]>, String)>], + headers: &[Option], ) -> Result>, String> { - let read = || -> Vec> { - headers - .par_iter() - .map(|header| { - header - .as_ref() - .map(|(values, path)| header_facts(values, path)) - }) - .collect() - }; let pool: &ThreadPool = POOL .get_or_init(|| { ThreadPoolBuilder::new() @@ -138,5 +58,14 @@ pub(crate) fn all_header_facts( }) .as_ref() .map_err(Clone::clone)?; - Ok(pool.install(read)) + Ok(pool.install(|| { + headers + .par_iter() + .map(|header| { + header + .as_ref() + .map(|(values, path)| header_facts(values, path)) + }) + .collect() + })) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index 0f0ebb8f48..8552f316c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -622,9 +622,9 @@ pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult( + py: Python<'py>, + cache: &PyOnceLock>, + text: &str, +) -> Bound<'py, PyString> { + cache + .get_or_init(py, || PyString::new(py, text).unbind()) + .bind(py) + .clone() +} + /// A model header's authored values: native entries, with the Python dict built on first read. struct HeaderValues { native: Option>, @@ -248,6 +260,8 @@ impl HeaderValues { pub(crate) struct NativeModelFile { paths: ModelFilePaths, contents: Arc, + /// `contents` and `query_sql` as the Python strings readers share, built on first read. + projected_texts: (PyOnceLock>, PyOnceLock>), header_values: HeaderValues, header_column_locations: Locations, output_column_locations: Locations, @@ -327,6 +341,7 @@ impl PendingModelFile { NativeModelFile { paths: self.paths, contents: parts.contents.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), header_values: HeaderValues::native(parts.header_values), header_column_locations: Locations::from_rows(parts.header_column_locations), output_column_locations: Locations::from_rows(parts.output_column_locations), @@ -404,6 +419,19 @@ impl NativeModelFile { } } + /// `relative_path.as_posix()`. + pub(crate) fn posix_path(&self, py: Python<'_>) -> PyResult { + match &self.paths.texts { + Some(texts) => Ok(texts.posix_path.clone()), + None => self + .paths + .objects(py)? + .1 + .call_method0(AS_POSIX_METHOD)? + .extract(), + } + } + /// `str(relative_path)`. pub(crate) fn relative_text(&self, py: Python<'_>) -> PyResult { match &self.paths.texts { @@ -452,7 +480,7 @@ impl NativeModelFile { Ok(vec![ file_path, relative_path.clone(), - PyString::new(py, &self.contents).into_any(), + self.contents(py).into_any(), self.header_values.dict(py)?.into_any(), self.header_column_locations .dict(&relative_path)? @@ -460,7 +488,7 @@ impl NativeModelFile { self.output_column_locations .dict(&relative_path)? .into_any(), - PyString::new(py, &self.query_sql).into_any(), + self.query_sql(py).into_any(), self.enum_declarations.bind(py).clone().into_any(), self.constant_declarations.bind(py).clone().into_any(), pyo3::types::PyBool::new(py, self.extract_implicit_alias_columns) @@ -492,6 +520,7 @@ impl NativeModelFile { Ok(Self { paths: ModelFilePaths::given(py, fields.file_path, fields.relative_path)?, contents: fields.contents.bind(py).to_str()?.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), header_values: HeaderValues::given(py, fields.header_values), header_column_locations: Locations::from_dict(py, fields.header_column_locations), output_column_locations: Locations::from_dict(py, fields.output_column_locations), @@ -516,6 +545,7 @@ impl NativeModelFile { Ok(Self { paths: self.paths.shared(py), contents: contents.bind(py).to_str()?.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), header_values: self.header_values.shared(py), header_column_locations: self.header_column_locations.shared(py), output_column_locations: self.output_column_locations.shared(py), @@ -538,7 +568,7 @@ impl NativeModelFile { #[getter] pub(crate) fn contents<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { - PyString::new(py, &self.contents) + projected_text(py, &self.projected_texts.0, &self.contents) } #[getter] @@ -565,7 +595,7 @@ impl NativeModelFile { #[getter] pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { - PyString::new(py, &self.query_sql) + projected_text(py, &self.projected_texts.1, &self.query_sql) } #[getter] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index 29168b65ec..f1bdbdebbf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -7,16 +7,15 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; use pyo3::{FromPyObject, IntoPyObject, IntoPyObjectExt}; +use sqlbuild_discovery::model_files::models::{NativeAudit, NativeColumn}; use sqlbuild_discovery::models::LineColumnSpan; use sqlbuild_model_config::header_metadata::models::{ ParsedAudit, ParsedColumn, ParsedThresholds, ThresholdBound, }; -use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; use sqlbuild_sqltext::compiler::models::AuthoredValue; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; -use crate::bindings::_helpers::model_config::native_nodes::NativeNode; use crate::bindings::_helpers::records::construct::{new_record, record_class}; use crate::bindings::_helpers::records::fields::all_specs; use crate::bindings::_helpers::records::fields::field; @@ -49,76 +48,6 @@ pub(crate) struct NativeColumnSpec { pub(crate) contracts: Arc>, } -/// One MODEL column parsed from native authored values. -#[derive(Clone, Debug)] -pub(crate) struct NativeColumn { - pub(crate) name: String, - pub(crate) column_type: Option, - pub(crate) nullable: Option, - pub(crate) description: Option, - pub(crate) audits: Vec, - pub(crate) migrate_from: Option, -} - -/// One audit instance parsed from native authored values. -#[derive(Clone, Debug)] -pub(crate) struct NativeAudit { - pub(crate) definition_name: String, - pub(crate) arguments: Vec<(String, AuthoredValue)>, - pub(crate) name: Option, - pub(crate) description: Option, - pub(crate) severity: Option, - pub(crate) run_scope: Option, - pub(crate) always_run: Option, - pub(crate) thresholds: Option, - pub(crate) minimum_samples: Option, - pub(crate) evidence_limit: Option, -} - -impl NativeColumn { - /// The column a native parse produced; every leaf is the projected text, flag or count. - pub(crate) fn parsed(column: &ParsedColumn>) -> Self { - let text = |node: Option<&NativeNode<'_>>| node.and_then(AuthoredNode::text); - Self { - name: column.name.text().unwrap_or_default(), - column_type: text(column.column_type.as_ref()), - nullable: flag(column.nullable.as_ref()), - description: text(column.description.as_ref()), - audits: column.audits.iter().map(NativeAudit::parsed).collect(), - migrate_from: text(column.migrate_from.as_ref()), - } - } -} - -impl NativeAudit { - fn parsed(audit: &ParsedAudit>) -> Self { - let text = |node: Option<&NativeNode<'_>>| node.and_then(AuthoredNode::text); - Self { - definition_name: audit.definition_name.text().unwrap_or_default(), - arguments: audit - .arguments - .iter() - .map(|(key, value)| (key.text().unwrap_or_default(), value.authored())) - .collect(), - name: text(audit.name.as_ref()), - description: text(audit.description.as_ref()), - severity: text(audit.severity.as_ref()), - run_scope: text(audit.run_scope.as_ref()), - always_run: flag(audit.always_run.as_ref()), - thresholds: audit.thresholds, - minimum_samples: audit.minimum_samples.as_ref().map(NativeNode::authored), - evidence_limit: audit.evidence_limit.as_ref().map(NativeNode::authored), - } - } -} - -fn flag(node: Option<&NativeNode<'_>>) -> Option { - match node.map(AuthoredNode::kind) { - Some(NodeKind::Bool(value)) => Some(value), - _ => None, - } -} - impl HeaderColumn { pub(crate) fn replaced<'py>( class: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs index a9a70bfe31..0d0f9cbb49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/mod.rs @@ -10,7 +10,6 @@ pub(crate) mod header_columns; pub(crate) mod header_metadata; pub(crate) mod model_config_builder; pub(crate) mod model_validator; -pub(crate) mod native_nodes; pub(crate) mod project_defaults; pub(crate) mod registration; pub(crate) mod template_data; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 84c97c4aac..708d340421 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -21,7 +21,7 @@ use sqlbuild_model_config::types::AuthoredNode; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, built_config}; -use crate::bindings::_helpers::compiled_project::model_loop_prepass::{EntryFacts, HeaderFacts}; +use crate::bindings::_helpers::compiled_project::model_loop_prepass::own_entry; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; @@ -30,6 +30,7 @@ use crate::bindings::_helpers::model_config::config_templates::{ }; use crate::bindings::_helpers::model_config::errors::NativeConfigError; use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; +use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; /// Hook keys in the sorted order `validate_model_hook_config` checks them. @@ -786,7 +787,7 @@ impl<'py> HeaderInput<'_, 'py> { value: &Bound<'py, PyAny>, ) -> PyResult> { match self.facts { - Some(facts) => facts.own_entry(self.values, key, value), + Some(facts) => own_entry(facts, self.values, (key, value)), None => Ok(None), } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 51db6c40a1..70dc0e9b85 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -444,18 +444,36 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes upstreams.push((upstream.str()?.extract()?, column_names(&columns)?)); } } - let mut calls_functions: bool = false; - for reference in native_field(model, "references")?.try_iter()? { - let kind: Bound<'_, PyAny> = native_field(&reference?, "ref_kind")?; - if kind.eq(&classes.udf_kind)? || kind.eq(&classes.table_function_kind)? { - calls_functions = true; - break; + let (name, query_sql, authored_sql, calls_functions) = match model.cast::() + { + Ok(native) => { + let function_kinds: [String; 2] = [ + classes.udf_kind.getattr("value")?.extract()?, + classes.table_function_kind.getattr("value")?.extract()?, + ]; + native.get().metadata_texts(model.py(), &function_kinds)? } - } + Err(_) => { + let mut calls_functions: bool = false; + for reference in native_field(model, "references")?.try_iter()? { + let kind: Bound<'_, PyAny> = native_field(&reference?, "ref_kind")?; + if kind.eq(&classes.udf_kind)? || kind.eq(&classes.table_function_kind)? { + calls_functions = true; + break; + } + } + ( + native_field(model, "name")?.extract()?, + native_field(model, "query_sql")?.extract()?, + native_field(model, "authored_sql")?.extract()?, + calls_functions, + ) + } + }; Ok(MetadataModel { - name: native_field(model, "name")?.extract()?, - query_sql: native_field(model, "query_sql")?.extract()?, - authored_sql: native_field(model, "authored_sql")?.extract()?, + name, + query_sql, + authored_sql, checked: checked(classes, model)?, calls_functions, references, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 60c4367e72..42644f97bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -15,6 +15,7 @@ use sqlbuild_analysis::semantic_checks::models::{ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::compiled_project::audit_views::audit_reference_rows; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::record_class; @@ -227,10 +228,19 @@ fn model_input( }); } } + let (name, path, authored_sql): (String, String, Arc) = + match model.cast::() { + Ok(native) => native.get().file_texts(model.py())?, + Err(_) => ( + native_field(model, "name")?.extract()?, + native_field(model, "relative_path")?.str()?.extract()?, + Arc::from(native_field(model, "authored_sql")?.extract::()?), + ), + }; Ok(HookModelInput { - name: native_field(model, "name")?.extract()?, - path: native_field(model, "relative_path")?.str()?.extract()?, - authored_sql: Arc::from(native_field(model, "authored_sql")?.extract::()?), + name, + path, + authored_sql, analysis_disabled: opts_out(&values)?, hooks, }) diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index ece8718b45..9fedfc925e 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -18,11 +18,11 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result Ok(ModelRow { name: facts.name.clone(), relative_path: facts.relative_path.clone(), - query_sql: facts.query_sql.clone(), + query_sql: facts.query_sql.to_string(), authored_sql: if facts.authored_query_sql.is_empty() { - facts.authored_sql.clone() + facts.authored_sql.to_string() } else { - facts.authored_query_sql.clone() + facts.authored_query_sql.to_string() }, config: facts.config.values.clone(), authored_config_keys: facts.config.header_keys.clone(), From 046c10ebde82acb1e14a546bde71ce66d0193cb4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 20:40:08 +0100 Subject: [PATCH 22/28] test: cover native header columns through the real CLI Compiles with several failing models report the first failure in model order, native and templated header columns produce the same contract diagnostics, and native column audits run against DuckDB. --- .../sqlbuild/cli/commands/main/_test_types.py | 19 ++ .../main/test_compile_model_headers.py | 187 ++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 22a1024976..6e894a8e10 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1148,3 +1148,22 @@ class ProviderAnnotationToleranceTestCase: annotation: str expected_exit_code: int expected_annotation_class_name: str | None + + +@dataclass(frozen=True) +class FirstModelErrorTestCase: + """Model files in path order and the one error compile reports for them.""" + + description: str + models: dict[str, str] + expected_message: str + + +@dataclass(frozen=True) +class HeaderColumnAuditBuildTestCase: + """Accepted values of a second model and whether its audits pass in DuckDB.""" + + description: str + accepted_values: str + expected_exit_code: int + expected_failed_models: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py new file mode 100644 index 0000000000..09dcd0a5ed --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py @@ -0,0 +1,187 @@ +"""Real CLI and DuckDB coverage for header columns read natively before the ordered model loop.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import cast + +import duckdb +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + FirstModelErrorTestCase, + HeaderColumnAuditBuildTestCase, +) + +PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nlabel = "Order amount"\n\n' + '[connection]\ndatabase = "orders.duckdb"\n' +) +HOOKED_MODEL: str = ( + "MODEL (description 'Hooked orders.', pre_hooks [inline_sql('SELECT 1')], " + "columns (id (type INTEGER)));\nSELECT CAST(1 AS INTEGER) AS id\n" +) +MACRO_COLUMN_MODEL: str = ( + "MODEL (description 'Orders.', columns (id (type INTEGER, description '@cents(id)')));\n" + "SELECT CAST(1 AS INTEGER) AS id\n" +) +DUPLICATE_COLUMN_MODEL: str = ( + "MODEL (description 'Orders.', columns (id (type INTEGER), ID (type INTEGER)));\n" + "SELECT CAST(1 AS INTEGER) AS id\n" +) +NULLABLE_NOT_NULL_MODEL: str = ( + "MODEL (description 'Orders.', columns (id (nullable true, audits [not_null])));\n" + "SELECT CAST(1 AS INTEGER) AS id\n" +) +UNKNOWN_TEMPLATE_MODEL: str = ( + "MODEL (description 'Orders.', columns (id (type INTEGER, description '${missing}')));\n" + "SELECT CAST(1 AS INTEGER) AS id\n" +) +CONTRACT_MODEL: str = ( + "MODEL (description 'Orders.', contract enforced, columns (\n" + " id (type INTEGER, nullable false, audits [not_null, accepted_values (values [1, 2])]),\n" + " amount (type {amount_type}, description {description}),\n" + "));\n" + "SELECT CAST(1 AS INTEGER) AS id, CAST(2.0 AS DOUBLE) AS amount\n" +) + + +def write_project(project_dir: Path, models: dict[str, str]) -> None: + (project_dir / "sqlbuild_project.toml").write_text(PROJECT_TOML) + model_dir: Path = project_dir / "models" + model_dir.mkdir() + for name, contents in models.items(): + (model_dir / f"{name}.sql").write_text(contents) + + +def compile_diagnostics( + project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[int, list[dict[str, object]]]: + exit_code: int = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + payload: dict[str, object] = json.loads(capsys.readouterr().out) + return exit_code, cast(list[dict[str, object]], payload["diagnostics"]) + + +@pytest.mark.parametrize( + "test_case", + ( + FirstModelErrorTestCase( + description="a macro in a column stops before a later duplicate column", + models={ + "a_orders": HOOKED_MODEL, + "b_orders": MACRO_COLUMN_MODEL, + "c_orders": DUPLICATE_COLUMN_MODEL, + }, + expected_message="model config field 'columns.id.description' does not allow macros", + ), + FirstModelErrorTestCase( + description="a duplicate column stops before a later macro", + models={ + "a_orders": HOOKED_MODEL, + "b_orders": DUPLICATE_COLUMN_MODEL, + "c_orders": MACRO_COLUMN_MODEL, + }, + expected_message=( + "models/b_orders.sql model has duplicate column 'ID' " + "(column names are case-insensitive)" + ), + ), + FirstModelErrorTestCase( + description="a nullable not_null column stops before a later duplicate column", + models={"b_orders": NULLABLE_NOT_NULL_MODEL, "c_orders": DUPLICATE_COLUMN_MODEL}, + expected_message=( + "models/b_orders.sql column 'id' cannot set nullable = true and audit not_null" + ), + ), + FirstModelErrorTestCase( + description="a templated column read by Python stops before a later duplicate column", + models={"b_orders": UNKNOWN_TEMPLATE_MODEL, "c_orders": DUPLICATE_COLUMN_MODEL}, + expected_message="model config references unknown variable 'missing'", + ), + ), + ids=lambda case: case.description, +) +def test_given_several_failing_models_when_compiling_then_the_first_in_model_order_is_reported( + test_case: FirstModelErrorTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project(tmp_path, test_case.models) + + exit_code, diagnostics = compile_diagnostics(tmp_path, capsys) + + assert exit_code == 1 + assert [item["message"] for item in diagnostics] == [test_case.expected_message] + + +def test_given_native_and_templated_header_columns_when_compiling_then_diagnostics_agree( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_project( + tmp_path, + { + "a_orders": CONTRACT_MODEL.format( + amount_type="VARCHAR", description="'Order amount'" + ), + "b_orders": CONTRACT_MODEL.format(amount_type="VARCHAR", description="'${label}'"), + }, + ) + + exit_code, diagnostics = compile_diagnostics(tmp_path, capsys) + + by_model: dict[str, str] = {} + for item in diagnostics: + name: str = cast(str, item["resource_name"]) + by_model[name] = json.dumps(item, sort_keys=True).replace(name, "") + assert exit_code == 1 + assert sorted(by_model) == ["a_orders", "b_orders"] + assert by_model["a_orders"] == by_model["b_orders"] + assert diagnostics[0]["location"] == { + "path": "models/a_orders.sql", + "line": 3, + "column": 3, + "end_line": 3, + "end_column": 9, + } + + +@pytest.mark.parametrize( + "test_case", + ( + HeaderColumnAuditBuildTestCase( + description="native column audits pass", + accepted_values="[1, 2]", + expected_exit_code=0, + expected_failed_models=(), + ), + HeaderColumnAuditBuildTestCase( + description="a native accepted_values audit fails the build", + accepted_values="[2, 3]", + expected_exit_code=1, + expected_failed_models=("b_orders",), + ), + ), + ids=lambda case: case.description, +) +def test_given_native_header_column_audits_when_building_then_duckdb_runs_them( + test_case: HeaderColumnAuditBuildTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + contract: str = CONTRACT_MODEL.format(amount_type="DOUBLE", description="'Order amount'") + write_project( + tmp_path, + { + "a_orders": contract, + "b_orders": contract.replace("[1, 2]", test_case.accepted_values), + }, + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", "build"]) + output: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, output + for name in test_case.expected_failed_models: + assert f"final audit for '{name}' failed" in output + with duckdb.connect(str(tmp_path / "orders.duckdb"), read_only=True) as connection: + assert connection.execute("SELECT id, amount FROM a_orders").fetchall() == [(1, 2.0)] From 2877bfd64bec1af5701858f01e07252e45aeea05 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 21:00:39 +0100 Subject: [PATCH 23/28] refactor: satisfy native boundary policy for native model data Model file paths build their pathlib objects natively on first read, header column parsing helpers move under the discovery helpers, the unused whole-config macro walk is removed in favour of the per-entry walk, natively parsed columns use the default contract classes, and pending Python-crossing budgets drop to the new counts. --- .../model_files/_helpers/header_columns.rs | 54 +++++++ .../src/model_files/_helpers/mod.rs | 1 + .../src/model_files/_helpers/native_nodes.rs | 11 +- .../src/model_files/main/header_facts.rs | 58 +------- .../src/model_files/main/model_file_stem.rs | 10 +- .../src/model_files/models.rs | 2 +- .../src/model_files/tests/helpers.rs | 53 ++++++- .../model_files/tests/test_header_facts.rs | 77 +--------- .../model_files/tests/test_model_file_stem.rs | 40 +++++- .../src/model_files/tests/test_types.rs | 25 ++-- .../config_presence/main/entry_macro_path.rs | 2 +- .../config_presence/main/first_macro_path.rs | 12 -- .../src/config_presence/main/mod.rs | 1 - .../tests/test_config_presence.rs | 81 ++++++----- .../src/config_presence/tests/test_types.rs | 9 ++ .../compile_command/reuse_recording.rs | 10 +- .../compiled_project/compiled_model_view.rs | 4 +- .../_helpers/compiled_project/model_loop.rs | 34 ++--- .../compiled_project/model_loop_config.rs | 3 +- .../compiled_project/model_loop_header.rs | 8 +- .../compiled_project/model_loop_host.rs | 11 +- .../compiled_project/model_loop_prepass.rs | 29 ++-- .../_helpers/discovery/model_file_record.rs | 135 ++++++++---------- .../_helpers/discovery/model_records.rs | 59 +++----- .../_helpers/model_config/header_metadata.rs | 102 +++++++------ .../model_config/model_config_builder.rs | 108 +++++++------- .../bindings/_helpers/rules/lint_pipeline.rs | 7 +- .../_helpers/semantic_checks/metadata.rs | 12 +- fensu.toml | 12 +- .../sqlbuild/cli/commands/main/_test_types.py | 10 ++ .../src/sqlbuild/cli/commands/main/helpers.py | 30 ++++ .../main/test_compile_model_headers.py | 87 ++++++----- 32 files changed, 575 insertions(+), 522 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/model_files/_helpers/header_columns.rs delete mode 100644 crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/header_columns.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/header_columns.rs new file mode 100644 index 0000000000..8991ccd664 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/header_columns.rs @@ -0,0 +1,54 @@ +//! Natively parsed header columns as their projected text, flags and counts. + +use sqlbuild_model_config::header_metadata::models::{ParsedAudit, ParsedColumn}; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::model_files::_helpers::native_nodes::NativeNode; +use crate::model_files::models::{NativeAudit, NativeColumn}; + +pub(crate) const COLUMNS_KEY: &str = "columns"; + +pub(crate) static NULL_VALUE: AuthoredValue = AuthoredValue::Null; + +fn text(node: Option<&NativeNode<'_>>) -> Option { + node.and_then(AuthoredNode::text) +} + +fn flag(node: Option<&NativeNode<'_>>) -> Option { + match node.map(AuthoredNode::kind) { + Some(NodeKind::Bool(value)) => Some(value), + _ => None, + } +} + +/// A parsed column's leaves as the projected text, flags and counts. +pub(crate) fn native_column(column: &ParsedColumn>) -> NativeColumn { + NativeColumn { + name: column.name.text().unwrap_or_default(), + column_type: text(column.column_type.as_ref()), + nullable: flag(column.nullable.as_ref()), + description: text(column.description.as_ref()), + audits: column.audits.iter().map(native_audit).collect(), + migrate_from: text(column.migrate_from.as_ref()), + } +} + +fn native_audit(audit: &ParsedAudit>) -> NativeAudit { + NativeAudit { + definition_name: audit.definition_name.text().unwrap_or_default(), + arguments: audit + .arguments + .iter() + .map(|(key, value)| (key.text().unwrap_or_default(), value.authored())) + .collect(), + name: text(audit.name.as_ref()), + description: text(audit.description.as_ref()), + severity: text(audit.severity.as_ref()), + run_scope: text(audit.run_scope.as_ref()), + always_run: flag(audit.always_run.as_ref()), + thresholds: audit.thresholds, + minimum_samples: audit.minimum_samples.as_ref().map(NativeNode::authored), + evidence_limit: audit.evidence_limit.as_ref().map(NativeNode::authored), + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs index 15d2cb619d..a6f8930a19 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod header_columns; pub(crate) mod native_nodes; pub(crate) mod output_columns; pub(crate) mod parsing; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs index e164052c14..7807df5a27 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs @@ -119,12 +119,19 @@ impl AuthoredNode for NativeNode<'_> { } fn integer(&self) -> Option { - self.digits().and_then(|digits| digits.parse::().ok()) + let digits: String = self.digits()?; + let Ok(number) = digits.parse::() else { + return None; + }; + Some(number) } fn number(&self) -> Option { if let Some(digits) = self.digits() { - return digits.parse::().ok(); + let Ok(number) = digits.parse::() else { + return None; + }; + return Some(number); } match self { Self::Value(AuthoredValue::BareWord(text)) => match word(text) { diff --git a/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs b/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs index 899ea78503..2512a6e1d7 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs @@ -4,16 +4,12 @@ use sqlbuild_model_config::config_presence::main::blocks_config_reuse::blocks_co use sqlbuild_model_config::config_presence::main::contains_template::contains_template; use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; -use sqlbuild_model_config::header_metadata::models::{ParsedAudit, ParsedColumn}; -use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; +use sqlbuild_model_config::header_metadata::models::ParsedColumn; use sqlbuild_sqltext::compiler::models::AuthoredValue; +use crate::model_files::_helpers::header_columns::{COLUMNS_KEY, NULL_VALUE, native_column}; use crate::model_files::_helpers::native_nodes::{NativeNode, dict_entries, exactly}; -use crate::model_files::models::{EntryFacts, HeaderFacts, NativeAudit, NativeColumn}; - -const COLUMNS_KEY: &str = "columns"; - -static NULL_VALUE: AuthoredValue = AuthoredValue::Null; +use crate::model_files::models::{EntryFacts, HeaderFacts, NativeColumn}; /// The header facts of one model's native authored values; `path` names it in parse messages. #[must_use] @@ -46,8 +42,10 @@ pub fn header_facts(values: &[(String, AuthoredValue)], path: &str) -> HeaderFac path, ) }); - let columns: Vec>> = - metadata.columns.ok().filter(|_| exact)?; + let Ok(columns) = metadata.columns else { + return None; + }; + let columns: Vec>> = exact.then_some(columns)?; Some(columns.iter().map(native_column).collect()) }); HeaderFacts { @@ -55,45 +53,3 @@ pub fn header_facts(values: &[(String, AuthoredValue)], path: &str) -> HeaderFac columns, } } - -fn text(node: Option<&NativeNode<'_>>) -> Option { - node.and_then(AuthoredNode::text) -} - -fn flag(node: Option<&NativeNode<'_>>) -> Option { - match node.map(AuthoredNode::kind) { - Some(NodeKind::Bool(value)) => Some(value), - _ => None, - } -} - -/// A parsed column's leaves as the projected text, flags and counts. -fn native_column(column: &ParsedColumn>) -> NativeColumn { - NativeColumn { - name: column.name.text().unwrap_or_default(), - column_type: text(column.column_type.as_ref()), - nullable: flag(column.nullable.as_ref()), - description: text(column.description.as_ref()), - audits: column.audits.iter().map(native_audit).collect(), - migrate_from: text(column.migrate_from.as_ref()), - } -} - -fn native_audit(audit: &ParsedAudit>) -> NativeAudit { - NativeAudit { - definition_name: audit.definition_name.text().unwrap_or_default(), - arguments: audit - .arguments - .iter() - .map(|(key, value)| (key.text().unwrap_or_default(), value.authored())) - .collect(), - name: text(audit.name.as_ref()), - description: text(audit.description.as_ref()), - severity: text(audit.severity.as_ref()), - run_scope: text(audit.run_scope.as_ref()), - always_run: flag(audit.always_run.as_ref()), - thresholds: audit.thresholds, - minimum_samples: audit.minimum_samples.as_ref().map(NativeNode::authored), - evidence_limit: audit.evidence_limit.as_ref().map(NativeNode::authored), - } -} diff --git a/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs b/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs index e537eba920..21db2a96a6 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs @@ -1,9 +1,13 @@ //! A model's name: Python's `Path.stem` of its `/`-separated relative path. -use crate::declarations::_helpers::paths::stem; - /// Python's `PurePath(relative_path).stem` for a `/`-separated relative path. #[must_use] pub fn model_file_stem(relative_path: &str) -> &str { - stem(relative_path.rsplit('/').next().unwrap_or(relative_path)) + let name: &str = relative_path + .rsplit_once('/') + .map_or(relative_path, |(_, name)| name); + match name.rfind('.') { + Some(index) if index > 0 && index + 1 < name.len() => &name[..index], + _ => name, + } } diff --git a/crates/sqlbuild-discovery/src/model_files/models.rs b/crates/sqlbuild-discovery/src/model_files/models.rs index a909c20c7f..a42b7646d4 100644 --- a/crates/sqlbuild-discovery/src/model_files/models.rs +++ b/crates/sqlbuild-discovery/src/model_files/models.rs @@ -30,7 +30,7 @@ pub struct DiscoveredModelFile { pub struct EntryFacts { /// Whether a string in the value holds a `${...}` template. pub template: bool, - /// Where `first_macro_path` finds a macro call in this entry. + /// Where the config macro walk finds a macro call in this entry. pub macro_path: MacroPath, /// Whether the value keeps the model from reusing another model's config. pub blocks_reuse: bool, diff --git a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs index 2faafbd2e6..6b8beef554 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs @@ -1,7 +1,13 @@ +use sqlbuild_model_config::config_presence::models::MacroPath; + use crate::model_files::main::header_facts::header_facts; use crate::model_files::main::parse_model_text::parse_model_text; -use crate::model_files::models::{DiscoveredModelFile, HeaderFacts, ModelFileOptions}; -use crate::model_files::tests::test_types::{Located, ModelSummary, Span}; +use crate::model_files::models::{ + DiscoveredModelFile, HeaderFacts, ModelFileOptions, NativeColumn, +}; +use crate::model_files::tests::test_types::{ + ColumnSummary, EntrySummary, Located, ModelSummary, Span, +}; use crate::models::LineColumnSpan; const FILE_PATH: &str = "/project/models/orders.sql"; @@ -81,3 +87,46 @@ pub(super) fn parsed_header_facts(contents: &str) -> HeaderFacts { .unwrap_or_else(|failure| panic!("{}", failure.message)); header_facts(&model.header_values, "models/orders.sql") } + +fn macro_path(path: &MacroPath) -> Option> { + let MacroPath::Found(path) = path else { + return None; + }; + Some(path.iter().map(String::as_str).collect()) +} + +/// `(key, template, macro path, blocks reuse)` of each header entry, in dict order. +pub(super) fn entry_summaries(facts: &HeaderFacts) -> Vec> { + facts + .entries + .iter() + .map(|(key, entry)| { + ( + key.as_str(), + entry.template, + macro_path(&entry.macro_path), + entry.blocks_reuse, + ) + }) + .collect() +} + +/// `(name, type, nullable, description, audit definitions)` of each parsed column. +pub(super) fn column_summaries(facts: &HeaderFacts) -> Option>> { + let columns: &Vec = facts.columns.as_ref()?; + Some(columns.iter().map(column_summary).collect()) +} + +fn column_summary(column: &NativeColumn) -> ColumnSummary<'_> { + ( + column.name.as_str(), + column.column_type.as_deref(), + column.nullable, + column.description.as_deref(), + column + .audits + .iter() + .map(|audit| audit.definition_name.as_str()) + .collect(), + ) +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs index d18d94af98..7fe9587166 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs @@ -1,49 +1,6 @@ -use sqlbuild_model_config::config_presence::models::MacroPath; - -use crate::model_files::models::{HeaderFacts, NativeColumn}; -use crate::model_files::tests::helpers::parsed_header_facts; -use crate::model_files::tests::test_types::{ColumnSummary, HeaderFactsTestCase}; - -fn column_summary( - column: &NativeColumn, -) -> ( - String, - Option, - Option, - Option, - Vec, -) { - ( - column.name.clone(), - column.column_type.clone(), - column.nullable, - column.description.clone(), - column - .audits - .iter() - .map(|audit| audit.definition_name.clone()) - .collect(), - ) -} - -fn expected_summary( - column: &ColumnSummary, -) -> ( - String, - Option, - Option, - Option, - Vec, -) { - let (name, column_type, nullable, description, audits) = column; - ( - (*name).to_owned(), - column_type.map(str::to_owned), - *nullable, - description.map(str::to_owned), - audits.iter().map(|audit| (*audit).to_owned()).collect(), - ) -} +use crate::model_files::models::HeaderFacts; +use crate::model_files::tests::helpers::{column_summaries, entry_summaries, parsed_header_facts}; +use crate::model_files::tests::test_types::HeaderFactsTestCase; #[test] fn given_model_headers_when_reading_native_facts_then_they_match_the_python_projection() { @@ -96,35 +53,15 @@ fn given_model_headers_when_reading_native_facts_then_they_match_the_python_proj ]; for test_case in test_cases { let facts: HeaderFacts = parsed_header_facts(test_case.contents); - let entries: Vec<(&str, bool, Option>, bool)> = facts - .entries - .iter() - .map(|(key, entry)| { - ( - key.as_str(), - entry.template, - match &entry.macro_path { - MacroPath::Found(path) => Some(path.iter().map(String::as_str).collect()), - MacroPath::Absent => None, - }, - entry.blocks_reuse, - ) - }) - .collect(); assert_eq!( - entries, test_case.expected_entries, + entry_summaries(&facts), + test_case.expected_entries, "{}", test_case.description ); assert_eq!( - facts - .columns - .as_ref() - .map(|columns| columns.iter().map(column_summary).collect::>()), - test_case - .expected_columns - .as_ref() - .map(|columns| columns.iter().map(expected_summary).collect::>()), + column_summaries(&facts), + test_case.expected_columns, "{}", test_case.description ); diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs index a1db4c4032..aae2d1b2c2 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs @@ -1,15 +1,41 @@ use crate::model_files::main::model_file_stem::model_file_stem; +use crate::model_files::tests::test_types::ModelFileStemTestCase; #[test] fn given_relative_paths_when_naming_models_then_stems_match_python() { let test_cases = [ - ("models/orders.sql", "orders"), - ("models/marts/orders.v2.sql", "orders.v2"), - ("models/.hidden", ".hidden"), - ("orders.", "orders."), - ("orders", "orders"), + ModelFileStemTestCase { + description: "a final suffix is dropped", + relative_path: "models/orders.sql", + expected_stem: "orders", + }, + ModelFileStemTestCase { + description: "only the last suffix is dropped", + relative_path: "models/marts/orders.v2.sql", + expected_stem: "orders.v2", + }, + ModelFileStemTestCase { + description: "a leading dot is not a suffix", + relative_path: "models/.hidden", + expected_stem: ".hidden", + }, + ModelFileStemTestCase { + description: "a trailing dot is kept", + relative_path: "orders.", + expected_stem: "orders.", + }, + ModelFileStemTestCase { + description: "a name without a suffix is kept", + relative_path: "orders", + expected_stem: "orders", + }, ]; - for (relative_path, expected) in test_cases { - assert_eq!(model_file_stem(relative_path), expected, "{relative_path}"); + for test_case in test_cases { + assert_eq!( + model_file_stem(test_case.relative_path), + test_case.expected_stem, + "{}", + test_case.description + ); } } diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs index 9596a77022..db361aab2d 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs @@ -25,17 +25,24 @@ pub(super) struct DeepModelHeaderTestCase { pub(super) struct HeaderFactsTestCase { pub(super) description: &'static str, pub(super) contents: &'static str, - /// `(key, template, macro path, blocks reuse)` of each header entry, in dict order. - pub(super) expected_entries: Vec<(&'static str, bool, Option>, bool)>, - /// `(name, type, nullable, description, audit definitions)` of each column, or `None`. - pub(super) expected_columns: Option>, + pub(super) expected_entries: Vec>, + pub(super) expected_columns: Option>>, } +/// `(key, template, macro path, blocks reuse)` of one header entry. +pub(super) type EntrySummary<'a> = (&'a str, bool, Option>, bool); + /// `(name, type, nullable, description, audit definitions)` of one parsed column. -pub(super) type ColumnSummary = ( - &'static str, - Option<&'static str>, +pub(super) type ColumnSummary<'a> = ( + &'a str, + Option<&'a str>, Option, - Option<&'static str>, - Vec<&'static str>, + Option<&'a str>, + Vec<&'a str>, ); + +pub(super) struct ModelFileStemTestCase { + pub(super) description: &'static str, + pub(super) relative_path: &'static str, + pub(super) expected_stem: &'static str, +} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs b/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs index 0b63052da2..1704dd4c74 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs @@ -4,7 +4,7 @@ use crate::config_presence::_helpers::macro_walk::walk_entries; use crate::config_presence::models::MacroPath; use crate::types::AuthoredNode; -/// The first macro path `first_macro_path` reports for the single entry `key: value`. +/// The first macro path `validate_no_macros_in_config_value` reports for the entry `key: value`. pub fn entry_macro_path(key: &N, value: &N) -> MacroPath { walk_entries(&[(key.clone(), value.clone())], &[]) } diff --git a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs b/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs deleted file mode 100644 index 6cde1c9bc3..0000000000 --- a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs +++ /dev/null @@ -1,12 +0,0 @@ -//! The first config field, in Python's walk order, whose string holds a macro call. - -use crate::config_presence::_helpers::macro_walk::walk_entries; -use crate::config_presence::models::MacroPath; -use crate::types::AuthoredNode; - -/// Walk config values as `validate_no_macros_in_config_value` does, skipping the `skipped` keys. -pub fn first_macro_path(values: &N, skipped: &[&str]) -> MacroPath { - let mut entries = values.entries(); - entries.retain(|(key, _)| !skipped.iter().any(|name| key.is_text(name))); - walk_entries(&entries, &[]) -} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs index a808b6e464..4a702d7a1d 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs @@ -1,5 +1,4 @@ pub mod blocks_config_reuse; pub mod contains_template; pub mod entry_macro_path; -pub mod first_macro_path; pub mod mutable_config_value; diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs index af2c9b6210..d13d360240 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs @@ -2,8 +2,12 @@ use crate::config_presence::_helpers::macro_calls::macro_call; use crate::config_presence::_helpers::scan::scan; use crate::config_presence::main::blocks_config_reuse::blocks_config_reuse; use crate::config_presence::main::contains_template::contains_template; +use crate::config_presence::main::entry_macro_path::entry_macro_path; use crate::config_presence::main::mutable_config_value::mutable_config_value; -use crate::config_presence::tests::test_types::{PresenceTestCase, ReuseTestCase}; +use crate::config_presence::models::MacroPath; +use crate::config_presence::tests::test_types::{ + EntryMacroPathTestCase, PresenceTestCase, ReuseTestCase, +}; use crate::tests::test_types::Value; #[test] @@ -149,40 +153,49 @@ fn given_config_values_when_checking_reuse_then_python_answers_are_returned() { } #[test] -fn given_config_entries_when_walking_each_entry_then_the_first_found_matches_the_whole_walk() { - use crate::config_presence::main::entry_macro_path::entry_macro_path; - use crate::config_presence::main::first_macro_path::first_macro_path; - use crate::config_presence::models::MacroPath; - - let entries = vec![ - (Value::Str("pre_hooks"), Value::Str("@skipped()")), - (Value::Str("materialized"), Value::Str("table")), - ( - Value::Str("columns"), - Value::Map(vec![( +fn given_config_entries_when_walking_for_macros_then_the_first_path_matches_python() { + let test_cases = [ + EntryMacroPathTestCase { + description: "a plain scalar holds no macro", + key: Value::Str("materialized"), + value: Value::Str("table"), + expected_path: MacroPath::Absent, + }, + EntryMacroPathTestCase { + description: "a nested mapping names every key down to the string", + key: Value::Str("columns"), + value: Value::Map(vec![( Value::Str("id"), - Value::Map(vec![(Value::Str("description"), Value::Str("@cents(id)"))]), + Value::Map(vec![ + (Value::Str("type"), Value::Str("INTEGER")), + (Value::Str("description"), Value::Str("@cents(id)")), + ]), )]), - ), - ( - Value::Str("tags"), - Value::List(vec![Value::Str("@later()")]), - ), + expected_path: MacroPath::Found(vec![ + "columns".to_owned(), + "id".to_owned(), + "description".to_owned(), + ]), + }, + EntryMacroPathTestCase { + description: "list items report the path of the list", + key: Value::Str("tags"), + value: Value::List(vec![Value::Str("daily"), Value::Str("@later()")]), + expected_path: MacroPath::Found(vec!["tags".to_owned()]), + }, + EntryMacroPathTestCase { + description: "a mapping key that is not a string is skipped", + key: Value::Str("config"), + value: Value::Map(vec![(Value::Int(1), Value::Str("@skipped()"))]), + expected_path: MacroPath::Absent, + }, ]; - let whole: MacroPath = first_macro_path(&Value::Map(entries.clone()), &["pre_hooks"]); - let per_entry: MacroPath = entries - .iter() - .filter(|(key, _)| !matches!(key, Value::Str("pre_hooks"))) - .map(|(key, value)| entry_macro_path(key, value)) - .find(|found| *found != MacroPath::Absent) - .unwrap_or(MacroPath::Absent); - assert_eq!( - whole, - MacroPath::Found(vec![ - "columns".to_owned(), - "id".to_owned(), - "description".to_owned() - ]) - ); - assert_eq!(per_entry, whole); + for test_case in test_cases { + assert_eq!( + entry_macro_path(&test_case.key, &test_case.value), + test_case.expected_path, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs index 9b46ae4c75..e371ab2cef 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs @@ -1,3 +1,4 @@ +use crate::config_presence::models::MacroPath; use crate::tests::test_types::Value; /// One authored config value and what each recursive scan finds in it. @@ -15,3 +16,11 @@ pub(super) struct ReuseTestCase { pub(super) expected_blocks_reuse: bool, pub(super) expected_mutable: bool, } + +/// One top-level config entry and the macro path the walk reports for it. +pub(super) struct EntryMacroPathTestCase { + pub(super) description: &'static str, + pub(super) key: Value, + pub(super) value: Value, + pub(super) expected_path: MacroPath, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 889e29ca16..01eab6546c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -106,15 +106,7 @@ pub(crate) type ArtifactRecorder<'a> = Option<&'a Py>; /// `os.path.abspath(path)`, computed natively on POSIX for UTF-8 paths and working directories. fn absolute(path: &Bound<'_, PyAny>) -> PyResult { - if cfg!(unix) - && let Ok(native) = path.extract::() - && let Some(found) = native_absolute(&native) - { - return Ok(found); - } - imported(path.py(), "os.path", "abspath")? - .call1((path,))? - .extract() + absolute_path(path.py(), &path.extract::()?) } /// `os.path.abspath(path)` of a native path. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index 7de3aeda56..a9f7b3142f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -443,7 +443,7 @@ impl NativeCompiledModel { pub(crate) fn metadata_texts( &self, py: Python<'_>, - function_kinds: &[String], + function_kinds: &[&str], ) -> PyResult<(String, String, String, bool)> { self.read(py, |record, _| { Ok(( @@ -453,7 +453,7 @@ impl NativeCompiledModel { record .references .iter() - .any(|(kind, _, _, _)| function_kinds.contains(kind)), + .any(|(kind, _, _, _)| function_kinds.contains(&kind.as_str())), )) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index 72023345ae..4b87d7c85e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -19,8 +19,8 @@ use crate::bindings::_helpers::compiled_project::model_loop_hooks::{ ExpandedHooks, HookModel, SqlHookClasses, expanded_hooks, }; use crate::bindings::_helpers::compiled_project::model_loop_host::{ - DeclarationScan, FinishedModel, InterpolationRow, ModelLoopRow, ModelPaths, RecordClasses, - RecordInputs, ReferenceRow, finished_record, + DeclarationScan, FinishedModel, InterpolationRow, ModelLoopRow, RecordClasses, RecordInputs, + ReferenceRow, finished_record, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ analysis_override, hook_presence, reference_row, @@ -35,7 +35,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_opt_out::{ OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, }; use crate::bindings::_helpers::compiled_project::model_loop_prepass::{ - NativeHeader, all_header_facts, + NativeHeader, all_header_facts, native_header, }; use crate::bindings::_helpers::compiled_project::model_loop_python_hooks::{ HookClasses, HookSignatures, @@ -57,7 +57,7 @@ use crate::bindings::_helpers::compiled_project::source_views::{ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ AttachedSqlTests, SqlTestSettings, TestFileRow, TestScope, }; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::model_config::config_errors::raised_config_exception; use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; @@ -94,7 +94,6 @@ use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_ use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::collections::{HashMap, HashSet}; -use std::sync::Arc; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; @@ -189,8 +188,6 @@ pub(crate) struct LoopContext<'a, 'py> { pub(crate) host: &'a Bound<'py, ConstantRendering>, pub(crate) classes: &'a LoopClasses<'py>, pub(crate) contracts: &'a ContractClasses<'py>, - /// The contract classes as Python passed them, kept by natively parsed columns. - pub(crate) contract_dict: &'a Py, } /// The declarations a model sees and the identity its usages are recorded for. @@ -277,10 +274,10 @@ fn prepared_rows<'py>( ) -> PyResult> { let hooks = &classes.sql_hooks; let mut sqls: Vec<(String, String)> = Vec::with_capacity(model_files.len()); - let mut paths: Vec = Vec::with_capacity(model_files.len()); + let mut paths: Vec = Vec::with_capacity(model_files.len()); for model_file in &model_files { let record: &NativeModelFile = model_file.get(); - let texts: ModelPaths = record.path_texts(py)?; + let texts: ModelFileTexts = record.path_texts(py)?; sqls.push((record.query_text().to_string(), texts.file_path.clone())); paths.push(texts); } @@ -326,7 +323,6 @@ impl NativeModelLoop { host: &host, classes: &classes, contracts: &contracts, - contract_dict: &self.contract_classes, }; project .borrow_mut() @@ -336,7 +332,7 @@ impl NativeModelLoop { let visibility: ModelVisibility<'py> = ModelVisibility::default(); let signatures: HookSignatures = HookSignatures::default(); for row in models { - let ModelPaths { + let ModelFileTexts { name, relative_path, file_path, @@ -395,15 +391,7 @@ impl NativeModelLoop { let py: Python<'py> = slf.py(); let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; let mut models: Vec = prepared_rows(py, &classes, model_files)?; - let headers: Vec> = models - .iter() - .map(|row| { - row.model_file - .get() - .native_header_values() - .map(|values| (Arc::clone(values), row.paths.relative_path.clone())) - }) - .collect(); + let headers: Vec> = models.iter().map(native_header).collect(); let facts: Vec> = py .compiler_detach(|| all_header_facts(&headers)) .map_err(compiler_error)?; @@ -431,7 +419,6 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, - contract_dict: &self.contract_classes, }; let attached: Vec = attached_sources(self, &context, rows, settings)?; Ok(retain_sources(project, attached)) @@ -452,7 +439,6 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, - contract_dict: &self.contract_classes, }; attached_audits(self, &context, (files, owners), settings) } @@ -472,7 +458,6 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, - contract_dict: &self.contract_classes, }; let validator: Bound<'py, NativeModelValidator> = self.validator.bind(py).downcast()?.clone(); @@ -540,7 +525,6 @@ impl NativeModelLoop { host, classes: &classes, contracts: &contracts, - contract_dict: &self.contract_classes, }; let attached: Vec = attached_functions(self, &context, rows, settings)?; checked_table_function_arities(&context, models, &attached)?; @@ -563,7 +547,7 @@ impl NativeModelLoop { } = model; let ModelLoopRow { paths: - ModelPaths { + ModelFileTexts { name, posix_path, relative_path, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs index 793fdc0948..a5d1223038 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs @@ -92,7 +92,8 @@ impl<'py> ModelConfigs<'py> { values.update(metadata.as_mapping())?; return view(classes, &values, &cached.fields); } - let built: ConfigBuild<'py> = builder.built(py, header, matched.as_deref(), name, facts)?; + let built: ConfigBuild<'py> = + builder.built(py, (header, facts), matched.as_deref(), name)?; let built: BuiltConfig<'py> = match built { ConfigBuild::Built(built) => built, ConfigBuild::Rejected(error) => { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index e8234494cf..bc6456e4a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -106,7 +106,7 @@ pub(crate) fn header_schema<'py>( let none: Bound<'py, PyAny> = py.None().into_bound(py); let node = |value: Option>| PyNode(value.unwrap_or_else(|| none.clone())); let model: (&str, &str) = (source.name, source.relative_path); - let native: Option<&[NativeColumn]> = native_columns(source, columns.as_ref())?; + let native: Option<&[NativeColumn]> = native_columns(context, source, columns.as_ref())?; let python_columns: Option> = if native.is_some() { None } else { columns }; let metadata = parse_header_metadata(&node(python_columns), &node(audits), source.relative_path); @@ -213,12 +213,16 @@ pub(crate) fn header_schema<'py>( /// The natively parsed header columns, when the config's columns are the header's own value. fn native_columns<'a>( + context: &LoopContext<'_, '_>, source: &HeaderSource<'a, '_>, columns: Option<&Bound<'_, PyAny>>, ) -> PyResult> { let (Some(facts), Some(columns)) = (source.facts, columns) else { return Ok(None); }; + if !context.contracts.uses_default_classes()? { + return Ok(None); + } let Some(parsed) = &facts.columns else { return Ok(None); }; @@ -237,7 +241,6 @@ fn native_declared_columns( ) -> PyResult { let py: Python<'_> = context.host.py(); let rows: &[(String, LineColumnSpan)] = source.model_file.header_location_rows(); - let contracts: Arc> = Arc::new(context.contract_dict.clone_ref(py)); let mut specs: Vec = Vec::with_capacity(columns.len()); let mut facts: Vec = Vec::with_capacity(columns.len()); for column in columns { @@ -264,7 +267,6 @@ fn native_declared_columns( NativeColumnSpec { column: column.clone(), location: location.map(|span| (source.model_file_handle.clone_ref(py), span)), - contracts: Arc::clone(&contracts), }, )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index dcb8164d19..79036fdaeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -23,7 +23,7 @@ use crate::bindings::_helpers::compiled_project::project::{ use crate::bindings::_helpers::compiled_project::project::{ DeclarationPy, config_values, declarations, }; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::{ ExpandedDeclarations, ScannedReference, span_rows, @@ -44,12 +44,9 @@ const NAME_ATTRIBUTE: &str = "name"; /// A model's scanned declaration references and the scan's stop marker. pub(crate) type DeclarationScan = (Vec, Option); -/// The names a model's file is known by. -pub(crate) use crate::bindings::_helpers::discovery::model_file_record::ModelFileTexts as ModelPaths; - /// One model the loop builds: its discovered file and what the host prepared for it. pub(crate) struct ModelLoopRow { - pub(crate) paths: ModelPaths, + pub(crate) paths: ModelFileTexts, pub(crate) authored_query: Arc, pub(crate) contents: Arc, pub(crate) scan: Option, @@ -63,7 +60,7 @@ pub(crate) struct ModelLoopRow { impl ModelLoopRow { pub(crate) fn new( - file: (Py, ModelPaths), + file: (Py, ModelFileTexts), scan: Option, interpolation: InterpolationRow, ) -> Self { @@ -128,7 +125,7 @@ pub(crate) fn finished_record<'py>( let py: Python<'py> = inputs.visible.py(); let ModelLoopRow { paths: - ModelPaths { + ModelFileTexts { name, posix_path: relative_path, .. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs index 8df87d7176..a0ff116bfd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs @@ -1,7 +1,4 @@ //! Each model's native header facts, read before the ordered loop on a fixed pool off the GIL. -//! -//! The ordered loop uses a fact only for a config value that is the header's own object, so a -//! value that path defaults, templates or hooks replaced is still read from Python. use std::sync::{Arc, OnceLock}; @@ -13,6 +10,8 @@ use sqlbuild_discovery::model_files::main::header_facts::header_facts; use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; use sqlbuild_sqltext::compiler::models::AuthoredValue; +use crate::bindings::_helpers::compiled_project::model_loop_host::ModelLoopRow; + /// The compile pools' fixed worker count. const PREPASS_WORKERS: usize = 4; const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; @@ -22,7 +21,7 @@ static POOL: OnceLock> = OnceLock::new(); /// A model's native header values and the relative path parse messages name. pub(crate) type NativeHeader = (Arc<[(String, AuthoredValue)]>, String); -/// The facts of a config entry, when its value is the header's own value of `key`. +/// An entry's facts only when its value is the header's own, so replaced values read from Python. pub(crate) fn own_entry<'a, 'py>( facts: &'a HeaderFacts, header: &Bound<'py, PyDict>, @@ -58,14 +57,16 @@ pub(crate) fn all_header_facts( }) .as_ref() .map_err(Clone::clone)?; - Ok(pool.install(|| { - headers - .par_iter() - .map(|header| { - header - .as_ref() - .map(|(values, path)| header_facts(values, path)) - }) - .collect() - })) + Ok(pool.install(|| headers.par_iter().map(facts_of).collect())) +} + +fn facts_of(header: &Option) -> Option { + let (values, path) = header.as_ref()?; + Some(header_facts(values, path)) +} + +/// The row's native header values and relative path, unless Python constructed its file. +pub(crate) fn native_header(row: &ModelLoopRow) -> Option { + let values: &Arc<[(String, AuthoredValue)]> = row.model_file.get().native_header_values()?; + Some((Arc::clone(values), row.paths.relative_path.clone())) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index 7c650d1199..deaf9dbb15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -1,18 +1,17 @@ //! A discovered SQL model file as a native record whose Python attributes are built on demand. -use std::ffi::OsString; use std::path::PathBuf; use std::sync::Arc; use crate::bindings::_helpers::records::construct::new_record; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; -use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::{PyRuntimeError, PyTypeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use pyo3::{FromPyObject, IntoPyObject, IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_discovery::models::LineColumnSpan; use sqlbuild_sqltext::compiler::models::AuthoredValue; @@ -30,7 +29,6 @@ const FIELD_NAMES: [&str; 10] = [ "extract_implicit_alias_columns", ]; const STEM_ATTRIBUTE: &str = "stem"; -const AS_POSIX_METHOD: &str = "as_posix"; /// Authored locations: native rows until Python reads them, then a cached dict. struct Locations { @@ -106,32 +104,41 @@ pub(crate) struct ModelFileTexts { /// A model file's paths: native text, with the Python path objects built on first read. struct ModelFilePaths { native_file_path: PathBuf, - /// `None` when a path is not text Python can encode, so readers ask Python as they always did. + /// `None` when a path is not text Python can encode, so readers raise as Python does. texts: Option, /// `(file_path, relative_path)` as Python path objects. objects: PyOnceLock<(Py, Py)>, - /// The project directory and relative path the objects are built from, unless given. - origin: Option<(Py, OsString)>, + /// The relative path the objects are built from, unless they were given. + relative: Option, } impl ModelFilePaths { /// Paths given as Python objects; their text is read once, here. fn given(py: Python<'_>, file_path: Py, relative_path: Py) -> PyResult { let native_file_path: PathBuf = file_path.bind(py).extract()?; + let relative: PathBuf = relative_path.bind(py).extract()?; let texts: Option = - python_texts(file_path.bind(py), relative_path.bind(py)).ok(); + native_file_path + .to_str() + .zip(relative.to_str()) + .map(|(file_text, relative_text)| ModelFileTexts { + name: model_file_stem(&posix(file_text)).to_owned(), + posix_path: posix(relative_text), + relative_path: relative_text.to_owned(), + file_path: file_text.to_owned(), + }); let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); let _ = objects.set(py, (file_path, relative_path)); Ok(Self { native_file_path, texts, objects, - origin: None, + relative: None, }) } - /// Paths native discovery named; Python builds `project_dir / relative_path` only when read. - fn discovered(project_dir: Py, relative_path: &str, file_path: String) -> Self { + /// Paths native discovery named; their Python path objects are built only when read. + fn discovered(relative_path: &str, file_path: String) -> Self { let name: String = model_file_stem(relative_path).to_owned(); Self { native_file_path: PathBuf::from(&file_path), @@ -142,7 +149,7 @@ impl ModelFilePaths { file_path, }), objects: PyOnceLock::new(), - origin: Some((project_dir, OsString::from(relative_path))), + relative: Some(PathBuf::from(relative_path)), } } @@ -155,42 +162,40 @@ impl ModelFilePaths { native_file_path: self.native_file_path.clone(), texts: self.texts.clone(), objects, - origin: self - .origin - .as_ref() - .map(|(project_dir, relative)| (project_dir.clone_ref(py), relative.clone())), + relative: self.relative.clone(), } } - /// `(file_path, relative_path)` as the Python path objects discovery names the file by. + /// `(file_path, relative_path)` as the `pathlib` objects discovery names the file by. fn objects<'py>(&self, py: Python<'py>) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let (file_path, relative_path) = self.objects.get_or_try_init(py, || { - let Some((project_dir, relative)) = &self.origin else { - return Err(pyo3::exceptions::PyRuntimeError::new_err( - "a model file has no paths", - )); + let Some(relative) = &self.relative else { + return Err(PyRuntimeError::new_err("a model file has no paths")); }; - let relative_path: Bound<'_, PyAny> = - py.import("pathlib")?.getattr("Path")?.call1((relative,))?; - let file_path: Bound<'_, PyAny> = project_dir.bind(py).div(&relative_path)?; - Ok::<_, pyo3::PyErr>((file_path.unbind(), relative_path.unbind())) + Ok::<_, pyo3::PyErr>(( + self.native_file_path.as_path().into_pyobject(py)?.unbind(), + relative.as_path().into_pyobject(py)?.unbind(), + )) })?; Ok((file_path.bind(py).clone(), relative_path.bind(py).clone())) } + + /// The file's names, or the error Python raises reading a name that is not UTF-8 text. + fn texts(&self, py: Python<'_>) -> PyResult<&ModelFileTexts> { + if let Some(texts) = &self.texts { + return Ok(texts); + } + let (file_path, relative_path) = self.objects(py)?; + for path in [file_path, relative_path] { + path.str()?.to_str()?; + } + Err(PyRuntimeError::new_err("a model file path is not text")) + } } -/// The names of a model file read from its Python path objects. -fn python_texts( - file_path: &Bound<'_, PyAny>, - relative_path: &Bound<'_, PyAny>, -) -> PyResult { - let file_text: String = file_path.str()?.to_str()?.to_owned(); - Ok(ModelFileTexts { - name: native_field(file_path, STEM_ATTRIBUTE)?.extract()?, - posix_path: relative_path.call_method0(AS_POSIX_METHOD)?.extract()?, - relative_path: relative_path.str()?.to_str()?.to_owned(), - file_path: file_text, - }) +/// `text` with the platform's path separator spelled `/`, as `PurePath.as_posix` spells it. +fn posix(text: &str) -> String { + text.replace(std::path::MAIN_SEPARATOR, "/") } /// `text` as the Python string `cache` shares with every reader, built on first read. @@ -283,8 +288,8 @@ pub(crate) struct ModelFileParts { /// How discovery names a model file: natively, or by Python paths for names Python escapes. pub(crate) enum DiscoveredPaths { - /// The project directory, the `/`-separated relative path and `str(project_dir / relative)`. - Native(Py, String, String), + /// The `/`-separated relative path and `str(project_dir / relative_path)`. + Native(String, String), /// `(file_path, relative_path)` built by Python. Python(Py, Py), } @@ -302,8 +307,8 @@ impl PendingModelFile { parts: ModelFileParts, ) -> PyResult { let paths: ModelFilePaths = match paths { - DiscoveredPaths::Native(project_dir, relative_path, file_path) => { - ModelFilePaths::discovered(project_dir, &relative_path, file_path) + DiscoveredPaths::Native(relative_path, file_path) => { + ModelFilePaths::discovered(&relative_path, file_path) } DiscoveredPaths::Python(file_path, relative_path) => { ModelFilePaths::given(py, file_path, relative_path)? @@ -316,7 +321,7 @@ impl PendingModelFile { pub(crate) fn name(&self, py: Python<'_>) -> PyResult { match &self.paths.texts { Some(texts) => Ok(texts.name.clone()), - None => native_field(&self.paths.objects(py)?.0, STEM_ATTRIBUTE)?.extract(), + None => field(&self.paths.objects(py)?.0, STEM_ATTRIBUTE)?.extract(), } } @@ -408,53 +413,29 @@ impl NativeModelFile { self.paths.native_file_path.clone() } - /// The file's names, read through Python only for paths Python escapes. + /// The file's names, or the error Python raises reading a name that is not UTF-8 text. pub(crate) fn path_texts(&self, py: Python<'_>) -> PyResult { - match &self.paths.texts { - Some(texts) => Ok(texts.clone()), - None => { - let (file_path, relative_path) = self.paths.objects(py)?; - python_texts(&file_path, &relative_path) - } - } + self.paths.texts(py).cloned() } /// `relative_path.as_posix()`. pub(crate) fn posix_path(&self, py: Python<'_>) -> PyResult { - match &self.paths.texts { - Some(texts) => Ok(texts.posix_path.clone()), - None => self - .paths - .objects(py)? - .1 - .call_method0(AS_POSIX_METHOD)? - .extract(), - } + Ok(self.paths.texts(py)?.posix_path.clone()) } /// `str(relative_path)`. pub(crate) fn relative_text(&self, py: Python<'_>) -> PyResult { - match &self.paths.texts { - Some(texts) => Ok(texts.relative_path.clone()), - None => Ok(self.paths.objects(py)?.1.str()?.to_string()), - } + Ok(self.paths.texts(py)?.relative_path.clone()) } /// `str(relative_path.parent)`, the folder project graphs group the model by. pub(crate) fn folder(&self, py: Python<'_>) -> PyResult { - match &self.paths.texts { - Some(texts) => Ok(texts - .relative_path - .rsplit_once(std::path::MAIN_SEPARATOR) - .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned())), - None => self - .paths - .objects(py)? - .1 - .getattr("parent")? - .str()? - .extract(), - } + Ok(self + .paths + .texts(py)? + .relative_path + .rsplit_once(std::path::MAIN_SEPARATOR) + .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned())) } /// `(file_path, relative_path)` as Python path objects. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index af4529c9e3..1515583bcb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -2,11 +2,12 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; +use std::path::Path; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::discovery::constants; @@ -22,12 +23,8 @@ use crate::bindings::models::NativeProjectTree; const PATHLIB_MODULE: &str = "pathlib"; const STEM_ATTRIBUTE: &str = "stem"; -/// One discovered model file: its name and record, or the error building it raises. -struct ModelItem<'py> { - name: Option, - relative_path: RelativePath<'py>, - built: PyResult>, -} +/// One discovered model file: its relative path and record, or the error building it raises. +type ModelItem<'py> = (RelativePath<'py>, PyResult>); /// A discovered file's relative path: native text, or the Python path already built for it. enum RelativePath<'py> { @@ -38,10 +35,18 @@ enum RelativePath<'py> { impl<'py> RelativePath<'py> { fn object(&self, py: Python<'py>) -> PyResult> { match self { - Self::Native(relative) => imported(py, PATHLIB_MODULE, "Path")?.call1((relative,)), + Self::Native(relative) => Path::new(relative).into_pyobject(py), Self::Python(relative) => Ok(relative.clone()), } } + + /// The model's name, its file stem. + fn name(&self, py: Python<'py>) -> PyResult> { + match self { + Self::Native(relative) => Ok(PyString::new(py, model_file_stem(relative)).into_any()), + Self::Python(relative) => native_field(relative, STEM_ATTRIBUTE), + } + } } /// Discover every model file natively, each as its record or the error it raises when built. @@ -64,20 +69,11 @@ fn model_items<'py>( for file in files { let raw: bool = tree.get().inner.raw_relative(&file.relative_path).is_some(); if !raw && let FileOutcome::Parsed(model) = file.outcome { - let name: String = model_file_stem(&file.relative_path).to_owned(); let file_path: String = root.display_path(&file.relative_path); - let paths = DiscoveredPaths::Native( - project_dir.clone().unbind(), - file.relative_path.clone(), - file_path, - ); + let paths = DiscoveredPaths::Native(file.relative_path.clone(), file_path); let built = model_record(py, model, paths, options.extract_implicit_alias_columns)? .map(|record| record.into_bound(py)); - items.push(ModelItem { - name: Some(name), - relative_path: RelativePath::Native(file.relative_path), - built, - }); + items.push((RelativePath::Native(file.relative_path), built)); continue; } let relative_path: Bound<'py, PyAny> = @@ -94,11 +90,7 @@ fn model_items<'py>( FileOutcome::Unreadable(failure) => Err(read_error(failure, &file_path)), FileOutcome::Failed(failure) => Err(failure_error(py, failure)), }; - items.push(ModelItem { - name: None, - relative_path: RelativePath::Python(relative_path), - built, - }); + items.push((RelativePath::Python(relative_path), built)); } Ok(items) }) @@ -133,21 +125,16 @@ pub(crate) fn discovered_model_files<'py>( .selected_model_names .filter(|selected| !selected.is_none()); let mut records: Vec> = Vec::with_capacity(items.len()); - for item in items { - if let Some(selected) = &selected { - let name: Bound<'py, PyAny> = match &item.name { - Some(name) => PyString::new(py, name).into_any(), - None => native_field(&item.relative_path.object(py)?, STEM_ATTRIBUTE)?, - }; - if !selected.contains(name)? { - continue; - } + for (relative_path, built) in items { + if let Some(selected) = &selected + && !selected.contains(relative_path.name(py)?)? + { + continue; } - match item.built { + match built { Ok(record) => records.push(record), Err(error) => { - let relative_path: Bound<'py, PyAny> = item.relative_path.object(py)?; - records.extend(faults.kept(&relative_path, Err(error))?); + records.extend(faults.kept(&relative_path.object(py)?, Err(error))?); } } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index f1bdbdebbf..b99f80b25d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -1,6 +1,6 @@ //! Parsed MODEL header columns and audits built into the Python contract objects. -use std::sync::{Arc, OnceLock}; +use std::sync::OnceLock; use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; @@ -16,12 +16,50 @@ use sqlbuild_sqltext::compiler::models::AuthoredValue; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; use crate::bindings::_helpers::records::construct::{new_record, record_class}; use crate::bindings::_helpers::records::fields::all_specs; use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sqltext::authored_values::authored_value_to_python; +/// `SchemaColumn` fields in declaration order, the order its objects are built positionally in. +const SCHEMA_COLUMN_FIELDS: [&str; 8] = [ + "name", + "type", + "nullable", + "description", + "meta", + "audits", + "location", + "migrate_from", +]; +/// `SchemaAuditInstance` fields in declaration order. +const AUDIT_INSTANCE_FIELDS: [&str; 11] = [ + "definition_name", + "arguments", + "name", + "description", + "severity", + "run_scope", + "always_run", + "thresholds", + "minimum_samples", + "evidence_limit", + "location", +]; +/// `MeasurementThresholds` fields in declaration order. +const THRESHOLDS_FIELDS: [&str; 2] = ["warn", "error"]; +/// `MeasurementThresholdBound` fields in declaration order. +const THRESHOLD_BOUND_FIELDS: [&str; 4] = ["operator", "limit", "lower", "upper"]; +/// Each positionally built record with the field order its values are given in. +const POSITIONAL_RECORDS: [(&str, &[&str]); 4] = [ + ("SchemaColumn", &SCHEMA_COLUMN_FIELDS), + ("SchemaAuditInstance", &AUDIT_INSTANCE_FIELDS), + ("MeasurementThresholds", &THRESHOLDS_FIELDS), + ("MeasurementThresholdBound", &THRESHOLD_BOUND_FIELDS), +]; + /// Native column fields; a Python record is constructed only for public consumers. pub(crate) struct HeaderColumn { fields: ColumnFields, @@ -44,8 +82,6 @@ pub(crate) struct NativeColumnSpec { pub(crate) column: NativeColumn, /// The model file whose relative path the column's location names, and the location. pub(crate) location: Option<(Py, LineColumnSpan)>, - /// The contract classes the record and its audits are built with. - pub(crate) contracts: Arc>, } impl HeaderColumn { @@ -204,7 +240,7 @@ fn native_audits<'py>( spec: &NativeColumnSpec, location: &Bound<'py, PyAny>, ) -> PyResult> { - let classes: ContractClasses<'py> = spec.contracts.bind(py).extract()?; + let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; let audits: Vec> = spec .column .audits @@ -284,43 +320,6 @@ impl HeaderColumns { } } -/// `SchemaColumn` fields in declaration order, the order its objects are built positionally in. -const SCHEMA_COLUMN_FIELDS: [&str; 8] = [ - "name", - "type", - "nullable", - "description", - "meta", - "audits", - "location", - "migrate_from", -]; -/// `SchemaAuditInstance` fields in declaration order. -const AUDIT_INSTANCE_FIELDS: [&str; 11] = [ - "definition_name", - "arguments", - "name", - "description", - "severity", - "run_scope", - "always_run", - "thresholds", - "minimum_samples", - "evidence_limit", - "location", -]; -/// `MeasurementThresholds` fields in declaration order. -const THRESHOLDS_FIELDS: [&str; 2] = ["warn", "error"]; -/// `MeasurementThresholdBound` fields in declaration order. -const THRESHOLD_BOUND_FIELDS: [&str; 4] = ["operator", "limit", "lower", "upper"]; -/// Each positionally built record with the field order its values are given in. -const POSITIONAL_RECORDS: [(&str, &[&str]); 4] = [ - ("SchemaColumn", &SCHEMA_COLUMN_FIELDS), - ("SchemaAuditInstance", &AUDIT_INSTANCE_FIELDS), - ("MeasurementThresholds", &THRESHOLDS_FIELDS), - ("MeasurementThresholdBound", &THRESHOLD_BOUND_FIELDS), -]; - /// The Python classes the parsed metadata becomes, read from a Python mapping. #[derive(FromPyObject)] #[pyo3(from_item_all)] @@ -335,6 +334,27 @@ pub(crate) struct ContractClasses<'py> { threshold_operators: Bound<'py, PyDict>, } +impl ContractClasses<'_> { + /// Whether these are the classes native header parsing builds by default. + pub(crate) fn uses_default_classes(&self) -> PyResult { + let py: Python<'_> = self.schema_column.py(); + let default: ContractClasses<'_> = contract_classes(py)?.extract()?; + Ok(self.schema_column.is(&default.schema_column) + && self + .schema_audit_instance + .is(&default.schema_audit_instance) + && self.source_location.is(&default.source_location) + && self + .measurement_thresholds + .is(&default.measurement_thresholds) + && self + .measurement_threshold_bound + .is(&default.measurement_threshold_bound) + && self.severities.is(&default.severities) + && self.threshold_operators.is(&default.threshold_operators)) + } +} + pub(crate) fn column_tuple<'py>( py: Python<'py>, classes: &ContractClasses<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index 708d340421..cd1b534582 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -167,7 +167,7 @@ impl NativeModelConfigBuilder { let (matched_path_default, model_name) = model; let matched_path_default: Option<&str> = matched_path_default.as_deref(); compiler_guard(|| { - match self.built(py, &header, matched_path_default, &model_name, None)? { + match self.built(py, (&header, None), matched_path_default, &model_name)? { ConfigBuild::Built(built) => { Ok(built_config(&classes, &built, matched_path_default)?.unbind()) } @@ -194,16 +194,11 @@ impl NativeModelConfigBuilder { pub(crate) fn built<'py>( &self, py: Python<'py>, - header: &Bound<'py, PyDict>, + (header, facts): (&Bound<'py, PyDict>, Option<&HeaderFacts>), matched_path_default: Option<&str>, model_name: &str, - facts: Option<&HeaderFacts>, ) -> PyResult> { - let header: HeaderInput<'_, 'py> = HeaderInput { - values: header, - facts, - }; - match self.build_config(py, &header, matched_path_default, model_name) { + match self.build_config(py, (header, facts), matched_path_default, model_name) { Ok(built) => Ok(ConfigBuild::Built(built)), Err(Halt::Config(error)) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), Err(Halt::Error(error)) => Err(error), @@ -297,11 +292,11 @@ impl NativeModelConfigBuilder { fn build_config<'py>( &self, py: Python<'py>, - input: &HeaderInput<'_, 'py>, + input: HeaderInput<'_, 'py>, matched_path_default: Option<&str>, model_name: &str, ) -> Built> { - let header: &Bound<'py, PyDict> = input.values; + let header: &Bound<'py, PyDict> = input.0; check_header_tags(header, model_name)?; let path_values = matched_path_default .map(|key| self.path_default_values(py, key)) @@ -319,7 +314,7 @@ impl NativeModelConfigBuilder { layered.del_item(key)?; } } - let has_templates = input.contains_template(&layered)?; + let has_templates = templated(input, &layered)?; let mut expansion = Expansion { builder: self, py, @@ -388,7 +383,7 @@ impl NativeModelConfigBuilder { header.get_item(TABLE_TYPE_KEY)?.map(PyNode).as_ref(), model_name, )?; - input.check_no_config_macros(&values)?; + check_no_config_macros(input, &values)?; Ok(BuiltConfig { values, header_keys: sorted_keys(header)?, @@ -774,60 +769,57 @@ fn storage_free_values<'py>( } /// A model's authored header and, when it was read natively, the header's facts. -struct HeaderInput<'a, 'py> { - values: &'a Bound<'py, PyDict>, - facts: Option<&'a HeaderFacts>, -} +type HeaderInput<'a, 'py> = (&'a Bound<'py, PyDict>, Option<&'a HeaderFacts>); -impl<'py> HeaderInput<'_, 'py> { - /// The facts of a config entry whose value is the header's own. - fn own_entry( - &self, - key: &Bound<'py, PyAny>, - value: &Bound<'py, PyAny>, - ) -> PyResult> { - match self.facts { - Some(facts) => own_entry(facts, self.values, (key, value)), - None => Ok(None), - } +/// The facts of a config entry whose value is the header's own. +fn header_entry<'a, 'py>( + (header, facts): HeaderInput<'a, 'py>, + entry: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + match facts { + Some(facts) => own_entry(facts, header, entry), + None => Ok(None), } +} - /// `contains_template_data(values)`, reading the header's own entries from their facts. - fn contains_template(&self, values: &Bound<'py, PyDict>) -> Built { - for (key, value) in values.iter() { - let found: bool = match self.own_entry(&key, &value)? { - Some(facts) => facts.template, - None => contains_template(&PyNode(value)), - }; - if found { - return Ok(true); - } +/// `contains_template_data(values)`, reading the header's own entries from their facts. +fn templated<'py>(input: HeaderInput<'_, 'py>, values: &Bound<'py, PyDict>) -> Built { + for (key, value) in values.iter() { + let found: bool = match header_entry(input, (&key, &value))? { + Some(facts) => facts.template, + None => contains_template(&PyNode(value)), + }; + if found { + return Ok(true); } - Ok(false) } + Ok(false) +} - /// Raise for the first config field outside hooks holding a macro call, as Python's walk does. - fn check_no_config_macros(&self, values: &Bound<'py, PyDict>) -> Built<()> { - for (key, value) in values.iter() { - if HOOK_KEYS - .iter() - .any(|name| PyNode(key.clone()).is_text(name)) - { - continue; - } - let found: MacroPath = match self.own_entry(&key, &value)? { - Some(facts) => facts.macro_path.clone(), - None => entry_macro_path(&PyNode(key), &PyNode(value)), - }; - if let MacroPath::Found(path) = found { - return Err(Halt::Config(ConfigError::compile(format!( - "model config field '{}' does not allow macros", - path.join(".") - )))); - } +/// Raise for the first config field outside hooks holding a macro call, as Python's walk does. +fn check_no_config_macros<'py>( + input: HeaderInput<'_, 'py>, + values: &Bound<'py, PyDict>, +) -> Built<()> { + for (key, value) in values.iter() { + if HOOK_KEYS + .iter() + .any(|name| PyNode(key.clone()).is_text(name)) + { + continue; + } + let found: MacroPath = match header_entry(input, (&key, &value))? { + Some(facts) => facts.macro_path.clone(), + None => entry_macro_path(&PyNode(key), &PyNode(value)), + }; + if let MacroPath::Found(path) = found { + return Err(Halt::Config(ConfigError::compile(format!( + "model config field '{}' does not allow macros", + path.join(".") + )))); } - Ok(()) } + Ok(()) } fn sorted_keys(header: &Bound<'_, PyDict>) -> Built> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index 7ae7fa68db..fe8990db6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -645,12 +645,7 @@ pub(crate) fn lint( /// A file's bodies planned again against its shared compile expansion, read only for this file. fn shared_plan(request: &LintRequest<'_>, file: &LintFile) -> Option { - let Some(source) = request.compiled_expansions.get(&file.path) else { - return None; - }; - let Some(compiled) = source.shared() else { - return None; - }; + let compiled: CompiledExpansion = request.compiled_expansions.get(&file.path)?.shared()?; let Ok(relative) = file.path.strip_prefix(request.project_dir) else { return None; }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 70dc0e9b85..4be1bbc2fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -30,6 +30,8 @@ use crate::bindings::_helpers::semantic_checks::type_recovery::items; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use crate::bindings::types::CompilerDetach; +/// `SqlReferenceKind.UDF` and `SqlReferenceKind.TABLE_FUNCTION` values. +const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; const METADATA_CONFIG_KEYS: [&str; 4] = [ "unique_key", "cursor", @@ -446,13 +448,9 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes } let (name, query_sql, authored_sql, calls_functions) = match model.cast::() { - Ok(native) => { - let function_kinds: [String; 2] = [ - classes.udf_kind.getattr("value")?.extract()?, - classes.table_function_kind.getattr("value")?.extract()?, - ]; - native.get().metadata_texts(model.py(), &function_kinds)? - } + Ok(native) => native + .get() + .metadata_texts(model.py(), &FUNCTION_REFERENCE_KINDS)?, Err(_) => { let mut calls_functions: bool = false; for reference in native_field(model, "references")?.try_iter()? { diff --git a/fensu.toml b/fensu.toml index 1b60010ba2..ffd3236e52 100644 --- a/fensu.toml +++ b/fensu.toml @@ -603,7 +603,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=1", @@ -616,7 +616,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_named.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=5", @@ -999,7 +999,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs=1", @@ -1145,7 +1145,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=24", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=2", @@ -1193,7 +1193,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=23", + "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", @@ -1230,7 +1230,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 6e894a8e10..28b083bba4 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1167,3 +1167,13 @@ class HeaderColumnAuditBuildTestCase: accepted_values: str expected_exit_code: int expected_failed_models: tuple[str, ...] + + +@dataclass(frozen=True) +class TemplatedHeaderDiagnosticsTestCase: + """A second model's templated column and where the first model's diagnostic points.""" + + description: str + amount_type: str + amount_description: str + expected_location: dict[str, object] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 6ec216670e..5a2b39a3e8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -951,3 +951,33 @@ def write_provider_task_project(*, project_dir: Path, annotation: str) -> None: path: Path = project_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") + + +def write_header_project(project_dir: Path, models: dict[str, str], project_toml: str) -> None: + """Write a DuckDB project with one SQL model file per name.""" + + _ = (project_dir / "sqlbuild_project.toml").write_text(project_toml) + model_dir: Path = project_dir / "models" + model_dir.mkdir() + for name, contents in models.items(): + _ = (model_dir / f"{name}.sql").write_text(contents) + + +def compile_diagnostics( + project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[int, list[dict[str, object]]]: + """Run one real CLI JSON compile and return its exit code and diagnostics.""" + + exit_code: int = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + payload: dict[str, object] = json.loads(capsys.readouterr().out) + return exit_code, cast(list[dict[str, object]], payload["diagnostics"]) + + +def diagnostics_by_model(diagnostics: list[dict[str, object]]) -> dict[str, str]: + """Each model's diagnostic as JSON with its own name replaced, to compare across models.""" + + by_model: dict[str, str] = {} + for item in diagnostics: + name: str = cast(str, item["resource_name"]) + by_model[name] = json.dumps(item, sort_keys=True).replace(name, "") + return by_model diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py index 09dcd0a5ed..75be4255f7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py @@ -2,9 +2,7 @@ from __future__ import annotations -import json from pathlib import Path -from typing import cast import duckdb import pytest @@ -13,11 +11,17 @@ from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( FirstModelErrorTestCase, HeaderColumnAuditBuildTestCase, + TemplatedHeaderDiagnosticsTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + compile_diagnostics, + diagnostics_by_model, + write_header_project, ) PROJECT_TOML: str = ( 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nlabel = "Order amount"\n\n' - '[connection]\ndatabase = "orders.duckdb"\n' + 'amount_type = "VARCHAR"\n\n[connection]\ndatabase = "orders.duckdb"\n' ) HOOKED_MODEL: str = ( "MODEL (description 'Hooked orders.', pre_hooks [inline_sql('SELECT 1')], " @@ -48,22 +52,6 @@ ) -def write_project(project_dir: Path, models: dict[str, str]) -> None: - (project_dir / "sqlbuild_project.toml").write_text(PROJECT_TOML) - model_dir: Path = project_dir / "models" - model_dir.mkdir() - for name, contents in models.items(): - (model_dir / f"{name}.sql").write_text(contents) - - -def compile_diagnostics( - project_dir: Path, capsys: pytest.CaptureFixture[str] -) -> tuple[int, list[dict[str, object]]]: - exit_code: int = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) - payload: dict[str, object] = json.loads(capsys.readouterr().out) - return exit_code, cast(list[dict[str, object]], payload["diagnostics"]) - - @pytest.mark.parametrize( "test_case", ( @@ -106,7 +94,7 @@ def compile_diagnostics( def test_given_several_failing_models_when_compiling_then_the_first_in_model_order_is_reported( test_case: FirstModelErrorTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: - write_project(tmp_path, test_case.models) + write_header_project(tmp_path, test_case.models, PROJECT_TOML) exit_code, diagnostics = compile_diagnostics(tmp_path, capsys) @@ -114,35 +102,59 @@ def test_given_several_failing_models_when_compiling_then_the_first_in_model_ord assert [item["message"] for item in diagnostics] == [test_case.expected_message] +@pytest.mark.parametrize( + "test_case", + ( + TemplatedHeaderDiagnosticsTestCase( + description="a templated column description", + amount_type="VARCHAR", + amount_description="'${label}'", + expected_location={ + "path": "models/a_orders.sql", + "line": 3, + "column": 3, + "end_line": 3, + "end_column": 9, + }, + ), + TemplatedHeaderDiagnosticsTestCase( + description="a templated column type", + amount_type="'${amount_type}'", + amount_description="'Order amount'", + expected_location={ + "path": "models/a_orders.sql", + "line": 3, + "column": 3, + "end_line": 3, + "end_column": 9, + }, + ), + ), + ids=lambda case: case.description, +) def test_given_native_and_templated_header_columns_when_compiling_then_diagnostics_agree( - tmp_path: Path, capsys: pytest.CaptureFixture[str] + test_case: TemplatedHeaderDiagnosticsTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], ) -> None: - write_project( + write_header_project( tmp_path, { - "a_orders": CONTRACT_MODEL.format( - amount_type="VARCHAR", description="'Order amount'" + "a_orders": CONTRACT_MODEL.format(amount_type="VARCHAR", description="'Order amount'"), + "b_orders": CONTRACT_MODEL.format( + amount_type=test_case.amount_type, description=test_case.amount_description ), - "b_orders": CONTRACT_MODEL.format(amount_type="VARCHAR", description="'${label}'"), }, + PROJECT_TOML, ) exit_code, diagnostics = compile_diagnostics(tmp_path, capsys) - by_model: dict[str, str] = {} - for item in diagnostics: - name: str = cast(str, item["resource_name"]) - by_model[name] = json.dumps(item, sort_keys=True).replace(name, "") + by_model: dict[str, str] = diagnostics_by_model(diagnostics) assert exit_code == 1 assert sorted(by_model) == ["a_orders", "b_orders"] assert by_model["a_orders"] == by_model["b_orders"] - assert diagnostics[0]["location"] == { - "path": "models/a_orders.sql", - "line": 3, - "column": 3, - "end_line": 3, - "end_column": 9, - } + assert diagnostics[0]["location"] == test_case.expected_location @pytest.mark.parametrize( @@ -169,12 +181,13 @@ def test_given_native_header_column_audits_when_building_then_duckdb_runs_them( capsys: pytest.CaptureFixture[str], ) -> None: contract: str = CONTRACT_MODEL.format(amount_type="DOUBLE", description="'Order amount'") - write_project( + write_header_project( tmp_path, { "a_orders": contract, "b_orders": contract.replace("[1, 2]", test_case.accepted_values), }, + PROJECT_TOML, ) exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", "build"]) From c04111266fc47142cbe534d4c29e9f3dabeae46c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 21:31:24 +0100 Subject: [PATCH 24/28] fix: run the model header pre-pass in turn when its pool cannot start A failed pool creation was cached and failed every later compile in the process. Compute the header facts on the calling thread instead, so compiling never depends on spawning the pre-pass workers. --- .../_helpers/compact_batch.rs | 27 +++++++-------- .../_helpers/compiled_project/model_loop.rs | 2 +- .../compiled_project/model_loop_prepass.rs | 33 +++++++++---------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs index 974c90e824..9a274defc7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs @@ -617,20 +617,21 @@ impl SessionCatalog { query_indexes.push(query_index); let references: Map = lineage_resources(member, share.as_ref()); let declared: Value = json!(declared_column_order(member, batch.nullability)); - // Batch-wide fields are identical for every member, so they stay out of the dedup key. - let identity: String = + let member_template_identity: String = json!([query_index, references, declared, member.recover_cte_facts]).to_string(); - let template_index: usize = *template_keys.entry(identity).or_insert_with(|| { - templates.push(json!({ - "queryIndex": query_index, - "references": references, - "functionReturnTypes": function_return_types, - "declaredColumnOrder": declared, - "recoverCteFacts": member.recover_cte_facts, - "richTypeInference": batch.rich_type_inference, - })); - templates.len() - 1 - }); + let template_index: usize = *template_keys + .entry(member_template_identity) + .or_insert_with(|| { + templates.push(json!({ + "queryIndex": query_index, + "references": references, + "functionReturnTypes": function_return_types, + "declaredColumnOrder": declared, + "recoverCteFacts": member.recover_cte_facts, + "richTypeInference": batch.rich_type_inference, + })); + templates.len() - 1 + }); let resource_names: Map = member .lineage_references .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index 4b87d7c85e..bdea620787 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -393,7 +393,7 @@ impl NativeModelLoop { let mut models: Vec = prepared_rows(py, &classes, model_files)?; let headers: Vec> = models.iter().map(native_header).collect(); let facts: Vec> = py - .compiler_detach(|| all_header_facts(&headers)) + .compiler_detach(|| Ok(all_header_facts(&headers))) .map_err(compiler_error)?; for (row, facts) in models.iter_mut().zip(facts) { row.header = facts; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs index a0ff116bfd..d5c9b515c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, OnceLock}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; -use rayon::{ThreadPool, ThreadPoolBuilder}; +use rayon::{ThreadPool, ThreadPoolBuildError, ThreadPoolBuilder}; use sqlbuild_discovery::model_files::main::header_facts::header_facts; use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; use sqlbuild_sqltext::compiler::models::AuthoredValue; @@ -16,7 +16,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_host::ModelLoopRow; const PREPASS_WORKERS: usize = 4; const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; -static POOL: OnceLock> = OnceLock::new(); +static POOL: OnceLock> = OnceLock::new(); /// A model's native header values and the relative path parse messages name. pub(crate) type NativeHeader = (Arc<[(String, AuthoredValue)]>, String); @@ -42,22 +42,19 @@ pub(crate) fn own_entry<'a, 'py>( .map(|_| entry)) } -/// Each model's header facts, computed on the compile pool; `None` where no native header exists. -pub(crate) fn all_header_facts( - headers: &[Option], -) -> Result>, String> { - let pool: &ThreadPool = POOL - .get_or_init(|| { - ThreadPoolBuilder::new() - .num_threads(PREPASS_WORKERS) - .stack_size(WORKER_STACK_BYTES) - .thread_name(|index| format!("sqlbuild-model-prepass-{index}")) - .build() - .map_err(|error| error.to_string()) - }) - .as_ref() - .map_err(Clone::clone)?; - Ok(pool.install(|| headers.par_iter().map(facts_of).collect())) +/// Each model's header facts on the compile pool, or in turn when the pool cannot start. +pub(crate) fn all_header_facts(headers: &[Option]) -> Vec> { + let pool: &Result = POOL.get_or_init(|| { + ThreadPoolBuilder::new() + .num_threads(PREPASS_WORKERS) + .stack_size(WORKER_STACK_BYTES) + .thread_name(|index| format!("sqlbuild-model-prepass-{index}")) + .build() + }); + match pool { + Ok(pool) => pool.install(|| headers.par_iter().map(facts_of).collect()), + Err(_) => headers.iter().map(facts_of).collect(), + } } fn facts_of(header: &Option) -> Option { From 07f3a7c6e96b9c3f7b2404060f39d9ecb27ae5e0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 22:22:03 +0100 Subject: [PATCH 25/28] feat: evaluate built-in rules speculatively with deferred cache writes Add a speculative rows evaluation to the rules engine: it runs on its own thread, holds its bulk cache and response memo writes, and answers the real request only when that request has the same key, making the held writes in the original order. A stop flag ends waiting and model evaluation early. --- .../src/engine/_helpers/cache.rs | 28 ++++ .../src/engine/_helpers/evaluation.rs | 94 +++++++++--- .../src/engine/_helpers/memo.rs | 123 +++++++++++++++- .../sqlbuild-rules/src/engine/_helpers/mod.rs | 1 + .../src/engine/_helpers/speculation.rs | 137 ++++++++++++++++++ .../engine/main/answer_speculative_rows.rs | 12 ++ .../engine/main/discard_speculative_rows.rs | 6 + crates/sqlbuild-rules/src/engine/main/mod.rs | 3 + .../src/engine/main/start_speculative_rows.rs | 6 + crates/sqlbuild-rules/src/engine/tests.rs | 2 + .../src/engine/tests/helpers.rs | 108 ++++++++++++++ .../src/engine/tests/test_speculative_rows.rs | 136 +++++++++++++++++ .../src/engine/tests/test_types.rs | 13 ++ crates/sqlbuild-rules/src/models.rs | 26 ++++ 14 files changed, 671 insertions(+), 24 deletions(-) create mode 100644 crates/sqlbuild-rules/src/engine/_helpers/speculation.rs create mode 100644 crates/sqlbuild-rules/src/engine/main/answer_speculative_rows.rs create mode 100644 crates/sqlbuild-rules/src/engine/main/discard_speculative_rows.rs create mode 100644 crates/sqlbuild-rules/src/engine/main/start_speculative_rows.rs create mode 100644 crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs diff --git a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs index 80efa9b7dc..6430d93cea 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs @@ -43,6 +43,34 @@ impl Cache { ) -> Result<(), String> { write_bucket(&self.native_rules_path, ruleset_identity, bucket) } + + /// The write `put_native_rules_bucket` would make, held until it is published. + pub(crate) fn pending_native_rules_bucket( + &self, + ruleset_identity: &str, + bucket: RuleCacheBucket, + ) -> PendingBucketWrite { + PendingBucketWrite { + path: self.native_rules_path.clone(), + fingerprint: ruleset_identity.to_owned(), + bucket, + } + } +} + +/// One bulk rules cache write that has not been made yet. +#[derive(Debug)] +pub(crate) struct PendingBucketWrite { + path: PathBuf, + fingerprint: String, + bucket: RuleCacheBucket, +} + +impl PendingBucketWrite { + /// Make the write exactly as `Cache::put_native_rules_bucket` makes it. + pub(crate) fn publish(&self) -> Result<(), String> { + write_bucket(&self.path, &self.fingerprint, &self.bucket) + } } fn read_bucket(path: &Path, fingerprint: &str) -> Result { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 8878a16fa8..e99d55b938 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -1,6 +1,6 @@ use crate::configuration::main::validate as config; use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; -use crate::engine::_helpers::cache::{Cache, RuleCacheEntry}; +use crate::engine::_helpers::cache::{Cache, PendingBucketWrite, RuleCacheEntry}; use crate::models::{ EvaluateRequest, EvaluateResponse, Fault, Model, RuleMetadata, RuleScope, RulesCodeGrammar, RulesConfig, @@ -20,13 +20,39 @@ use fensu_policy::lifecycle::models::{ #[cfg(test)] use rayon::iter::IndexedParallelIterator; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; +use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; const NATIVE_RULE_WORKERS: usize = 4; const NATIVE_RULE_STACK_BYTES: usize = 16 * 1024 * 1024; const MODEL_RULES_IDENTITY_SEED: &[u8] = b"sqlbuild-rules-model-v2"; +const CANCELLED: &str = "native rules evaluation was cancelled"; + +/// How one evaluation publishes its bulk rules cache bucket and whether it may stop early. +pub(crate) struct EvaluationControl<'a> { + /// `None` writes the bucket at once; `Some` holds the write for the caller to publish. + pub(crate) deferred_bucket: Option<&'a RefCell>>, + /// Stops model evaluation between models once set. + pub(crate) cancel: Option<&'a AtomicBool>, +} + +impl EvaluationControl<'_> { + /// Write the cache at once and never stop early, as an ordinary evaluation does. + pub(crate) fn immediate() -> Self { + Self { + deferred_bucket: None, + cancel: None, + } + } + + fn cancelled(&self) -> bool { + self.cancel + .is_some_and(|cancel| cancel.load(Ordering::Acquire)) + } +} struct PendingModelRule<'a> { index: usize, @@ -44,6 +70,7 @@ struct ModelRuleBatchContext<'a> { request: &'a EvaluateRequest, selected: &'a BTreeMap, threshold_overrides: &'a [ResolvedThresholdOverride], + cancel: Option<&'a AtomicBool>, } struct NativeModelRuleRequest<'a> { @@ -61,6 +88,7 @@ struct NativeModelEvaluation { fn evaluate_native_models( input: NativeModelRuleRequest<'_>, + control: &EvaluationControl<'_>, ) -> Result { let NativeModelRuleRequest { context, @@ -145,7 +173,13 @@ fn evaluate_native_models( bucket .entries .retain(|path, _| current.contains(path.as_str()) || project_dir.join(path).is_file()); - cache.put_native_rules_bucket(model_rules_fingerprint, &bucket)?; + match control.deferred_bucket { + None => cache.put_native_rules_bucket(model_rules_fingerprint, &bucket)?, + Some(deferred) => { + *deferred.borrow_mut() = + Some(cache.pending_native_rules_bucket(model_rules_fingerprint, bucket)); + } + } } Ok(result) } @@ -157,14 +191,21 @@ fn evaluate_pending_models( let evaluate_pending = |pending: PendingModelRule<'_>| CompletedModelRule { index: pending.index, identity: pending.identity, - faults: rules::evaluate_model(ModelEvaluationRequest { - model: pending.model, - config: &context.request.config, - selected: context.selected, - request: context.request, - threshold_overrides: context.threshold_overrides, - }) - .and_then(|faults| scoped_faults(context.selected, faults, RuleScope::evaluates_models)), + faults: if context + .cancel + .is_some_and(|cancel| cancel.load(Ordering::Acquire)) + { + Err(CANCELLED.to_owned()) + } else { + rules::evaluate_model(ModelEvaluationRequest { + model: pending.model, + config: &context.request.config, + selected: context.selected, + request: context.request, + threshold_overrides: context.threshold_overrides, + }) + .and_then(|faults| scoped_faults(context.selected, faults, RuleScope::evaluates_models)) + }, }; if pending_models.len() <= 1 { return Ok(pending_models.into_iter().map(evaluate_pending).collect()); @@ -271,7 +312,16 @@ fn model_rules_fingerprint( fingerprint::fingerprint(selected, &model_config, dialect) } +#[cfg(test)] pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result { + evaluate_request_with(request, &EvaluationControl::immediate()) +} + +/// Evaluate `request`, publishing its cache bucket and stopping early as `control` says. +pub(crate) fn evaluate_request_with( + request: EvaluateRequest, + control: &EvaluationControl<'_>, +) -> Result { if request.version != API_VERSION { return Err(format!( "unsupported rules native API version {}; expected {API_VERSION}", @@ -334,15 +384,23 @@ pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result = control.cancel; + let native = evaluate_native_models( + NativeModelRuleRequest { + context: ModelRuleBatchContext { + request: &request, + selected: &selected_by_code, + threshold_overrides: &threshold_overrides, + cancel, + }, + cache: cache.as_ref(), + model_rules_fingerprint: &model_rules_fingerprint, }, - cache: cache.as_ref(), - model_rules_fingerprint: &model_rules_fingerprint, - })?; + control, + )?; let cache_hits = native.hits; let cache_misses = native.misses; raw_faults.extend(native.faults); diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index 3452bc99f8..f63d05f412 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -1,33 +1,76 @@ //! Evaluate a natively built rules request, reusing the last identical request's response. +use std::cell::RefCell; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; use serde_json::Value; use crate::constants::{API_VERSION, NATIVE_ROWS_MEMO_VERSION}; -use crate::engine::_helpers::evaluation::evaluate_request; +use crate::engine::_helpers::cache::PendingBucketWrite; +use crate::engine::_helpers::evaluation::{EvaluationControl, evaluate_request_with}; use crate::errors::RowsError; use crate::models::{EvaluateRequest, Model, RowsEvaluation, RowsRequest}; +/// A rows request whose models are digested, ready to evaluate once. +#[derive(Debug)] +pub(crate) struct PreparedRows { + request: EvaluateRequest, + /// The request's memo identity and memo path, when the rules cache is enabled. + memo: Option<(String, PathBuf)>, + /// The digest of every evaluation input the request carries, when it was asked for. + pub(crate) key: Option, +} + +/// What a speculative evaluation found, with every write it would have made still pending. +#[derive(Debug)] +pub(crate) enum SpeculativeOutcome { + /// The memo answered the request, so the evaluation would have written nothing. + Reused(String), + /// The request was evaluated; its bucket write precedes its memo write. + Evaluated { + bucket: Option, + response: Result, + memo: Option<(String, PathBuf)>, + }, +} + pub(crate) fn evaluate_rows(rows: RowsRequest) -> Result { + evaluate_prepared(prepare(rows, false)?) +} + +/// Digest `rows`' models and derive its memo identity and, when asked, its speculation key. +pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result { let RowsRequest { mut request, memo } = rows; let models: Vec = std::mem::take(&mut request.models) .into_iter() .map(digested) .collect::>()?; - let identity: Option<(String, PathBuf)> = match memo { + let memo: Option<(String, PathBuf)> = match memo { Some((code_identity, path)) => Some((identity(&request, &models, &code_identity)?, path)), None => None, }; - let reused: Option = identity + let key: Option = if keyed { + Some(speculation_key(&request, &models, memo.as_ref())?) + } else { + None + }; + request.models = models; + Ok(PreparedRows { request, memo, key }) +} + +/// Answer a prepared request from the memo or evaluate it, writing the cache and memo at once. +pub(crate) fn evaluate_prepared(prepared: PreparedRows) -> Result { + let PreparedRows { request, memo, .. } = prepared; + let reused: Option = memo .as_ref() .and_then(|(identity, path)| read_response(path, identity)); let (response, was_reused): (String, bool) = match reused { Some(response) => (response, true), None => { - request.models = models; - let response: String = evaluate_request(request).map_err(RowsError::Rules)?; - if let Some((identity, path)) = &identity { + let response: String = evaluate_request_with(request, &EvaluationControl::immediate()) + .map_err(RowsError::Rules)?; + if let Some((identity, path)) = &memo { write_response(path, identity, &response).map_err(RowsError::Io)?; } (response, false) @@ -36,6 +79,74 @@ pub(crate) fn evaluate_rows(rows: RowsRequest) -> Result SpeculativeOutcome { + let PreparedRows { request, memo, .. } = prepared; + if let Some(response) = memo + .as_ref() + .and_then(|(identity, path)| read_response(path, identity)) + { + return SpeculativeOutcome::Reused(response); + } + let bucket: RefCell> = RefCell::new(None); + let response: Result = evaluate_request_with( + request, + &EvaluationControl { + deferred_bucket: Some(&bucket), + cancel: Some(cancel), + }, + ); + let bucket: Option = bucket.into_inner(); + SpeculativeOutcome::Evaluated { + bucket, + response, + memo, + } +} + +/// Make a speculative evaluation's pending writes in order and answer as `evaluate_prepared` would. +pub(crate) fn commit(speculative: SpeculativeOutcome) -> Result { + match speculative { + SpeculativeOutcome::Reused(response) => decode(&response, true), + SpeculativeOutcome::Evaluated { + bucket, + response, + memo, + } => { + if let Some(bucket) = &bucket { + bucket.publish().map_err(RowsError::Rules)?; + } + let response: String = response.map_err(RowsError::Rules)?; + if let Some((identity, path)) = &memo { + write_response(path, identity, &response).map_err(RowsError::Io)?; + } + decode(&response, false) + } + } +} + +/// Every evaluation input a request carries: its memo identity inputs and memo location. +fn speculation_key( + request: &EvaluateRequest, + models: &[Model], + memo: Option<&(String, PathBuf)>, +) -> Result { + let mut hasher: blake3::Hasher = blake3::Hasher::new(); + match memo { + Some((identity, path)) => { + hasher.update(b"memo\0"); + hasher.update(identity.as_bytes()); + hasher.update(b"\0"); + hasher.update(path.as_os_str().as_encoded_bytes()); + } + None => { + hasher.update(b"unmemoized\0"); + hasher.update(identity(request, models, "")?.as_bytes()); + } + } + Ok(hasher.finalize().to_hex().to_string()) +} + /// `model` with the BLAKE3 digest of its exact facts, which keys its cached findings. fn digested(mut model: Model) -> Result { let mut hasher: blake3::Hasher = blake3::Hasher::new(); diff --git a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs index b82023ac5c..ec8f8aa27d 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs @@ -4,3 +4,4 @@ pub mod evaluation; pub(crate) mod memo; pub(crate) mod project_rows; pub(crate) mod rows; +pub(crate) mod speculation; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs b/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs new file mode 100644 index 0000000000..06bdf637c2 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs @@ -0,0 +1,137 @@ +//! Evaluate a built rules request speculatively and answer the real request from it when identical. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::engine::_helpers::memo::{ + PreparedRows, SpeculativeOutcome, commit, evaluate_prepared, prepare, speculate, +}; +use crate::models::{RowsRequest, SpeculativeAnswer, SpeculativeRows}; + +const SPECULATION_THREAD: &str = "sqlbuild-early-rules"; +/// How often a wait for the speculative thread checks its stop flag. +const WAIT_POLL: Duration = Duration::from_millis(5); + +/// What the speculative thread finished with: its held outcome, or `None` when preparing failed. +type Speculated = Result, String>; + +pub(crate) fn start(rows: RowsRequest) -> std::io::Result { + let cancel: Arc = Arc::new(AtomicBool::new(false)); + let (sender, key): (Sender>, Receiver>) = channel(); + let stop: Arc = Arc::clone(&cancel); + let thread = std::thread::Builder::new() + .name(SPECULATION_THREAD.to_owned()) + .spawn(move || -> Speculated { + catch_compiler_panic(|| { + let Ok(prepared) = prepare(rows, true) else { + let _ = sender.send(None); + return Ok(None); + }; + let _ = sender.send(prepared.key.clone()); + Ok(Some(speculate(prepared, &stop))) + }) + })?; + Ok(SpeculativeRows { + key: Mutex::new(key), + thread: Some(thread), + cancel, + }) +} + +pub(crate) fn answer( + mut speculative: SpeculativeRows, + rows: RowsRequest, + stop: &AtomicBool, +) -> SpeculativeAnswer { + let prepared: PreparedRows = match prepare(rows, true) { + Ok(prepared) => prepared, + Err(error) => { + drop(speculative); + return SpeculativeAnswer::Answered { + evaluation: Err(error), + used: false, + }; + } + }; + let Some(speculated) = speculated_key(&speculative, stop) else { + drop(speculative); + return SpeculativeAnswer::Stopped; + }; + let outcome: Option = + if prepared.key.is_some() && speculated == prepared.key { + if !finished(&speculative, stop) { + drop(speculative); + return SpeculativeAnswer::Stopped; + } + match speculative.thread.take().map(std::thread::JoinHandle::join) { + Some(Ok(Ok(outcome))) => outcome, + _ => None, + } + } else { + drop(speculative); + None + }; + match outcome { + Some(outcome) => SpeculativeAnswer::Answered { + evaluation: commit(outcome), + used: true, + }, + None => SpeculativeAnswer::Answered { + evaluation: evaluate_prepared(prepared), + used: false, + }, + } +} + +/// The speculation's key once known (`Some(None)` when it has none), or `None` once `stop` is set. +fn speculated_key(speculative: &SpeculativeRows, stop: &AtomicBool) -> Option> { + let receiver = speculative + .key + .lock() + .unwrap_or_else(PoisonError::into_inner); + loop { + if stop.load(Ordering::Acquire) { + return None; + } + match receiver.recv_timeout(WAIT_POLL) { + Ok(key) => return Some(key), + Err(RecvTimeoutError::Disconnected) => return Some(None), + Err(RecvTimeoutError::Timeout) => {} + } + } +} + +/// Whether the speculative thread finished before `stop` was set. +fn finished(speculative: &SpeculativeRows, stop: &AtomicBool) -> bool { + loop { + if stop.load(Ordering::Acquire) { + return false; + } + if speculative + .thread + .as_ref() + .is_none_or(std::thread::JoinHandle::is_finished) + { + return true; + } + std::thread::sleep(WAIT_POLL); + } +} + +pub(crate) fn discard(speculative: SpeculativeRows) { + drop(speculative); +} + +impl Drop for SpeculativeRows { + /// Cancel the speculation and wait for its thread. + fn drop(&mut self) { + self.cancel.store(true, Ordering::Release); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/sqlbuild-rules/src/engine/main/answer_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/main/answer_speculative_rows.rs new file mode 100644 index 0000000000..35a0a87af9 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/answer_speculative_rows.rs @@ -0,0 +1,12 @@ +use std::sync::atomic::AtomicBool; + +use crate::models::{RowsRequest, SpeculativeAnswer, SpeculativeRows}; + +/// Evaluate `request` as `evaluate_rows` would, from `speculative` only when its key matches. +pub fn answer_speculative_rows( + speculative: SpeculativeRows, + request: RowsRequest, + stop: &AtomicBool, +) -> SpeculativeAnswer { + crate::engine::_helpers::speculation::answer(speculative, request, stop) +} diff --git a/crates/sqlbuild-rules/src/engine/main/discard_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/main/discard_speculative_rows.rs new file mode 100644 index 0000000000..7b2f5ce3aa --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/discard_speculative_rows.rs @@ -0,0 +1,6 @@ +use crate::models::SpeculativeRows; + +/// Cancel a speculative evaluation and wait for its thread; nothing it held is written. +pub fn discard_speculative_rows(speculative: SpeculativeRows) { + crate::engine::_helpers::speculation::discard(speculative); +} diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index 8cae1cda6c..a20ecdbdd4 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -1,5 +1,8 @@ +pub mod answer_speculative_rows; pub mod build_models; pub mod custom_host; +pub mod discard_speculative_rows; pub mod evaluate_rows; pub mod finalize_rows; pub mod project_model_rows; +pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/start_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/main/start_speculative_rows.rs new file mode 100644 index 0000000000..ca090e0aee --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/start_speculative_rows.rs @@ -0,0 +1,6 @@ +use crate::models::{RowsRequest, SpeculativeRows}; + +/// Evaluate a built request on its own thread, holding every cache and memo write it would make. +pub fn start_speculative_rows(request: RowsRequest) -> std::io::Result { + crate::engine::_helpers::speculation::start(request) +} diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index 1dd70e7f7c..3190903ee8 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -10,6 +10,8 @@ mod helpers; mod incremental_rules; #[path = "tests/test_scope_facts.rs"] mod scope_facts; +#[path = "tests/test_speculative_rows.rs"] +mod speculative_rows; #[path = "tests/test_sql_scanning.rs"] mod sql_scanning; #[path = "tests/test_sql_test_rules.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index dd22e28f03..1300b5ab36 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -1,7 +1,17 @@ +use std::path::{Path, PathBuf}; + use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use tempfile::TempDir; +use std::sync::atomic::AtomicBool; + +use crate::engine::main::answer_speculative_rows::answer_speculative_rows; +use crate::engine::main::discard_speculative_rows::discard_speculative_rows; +use crate::models::{ + EvaluateRequest, Fault, RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows, +}; + use crate::engine::_helpers::evaluation::evaluate_json; use crate::engine::_helpers::evaluation::evaluate_request as evaluate_parsed; use crate::engine::_helpers::evaluation::parse_parts; @@ -480,3 +490,101 @@ pub(crate) fn cached_paths(project_dir: &TempDir) -> Result, String> .map(|entries| entries.keys().cloned().collect()) .unwrap_or_default()) } + +const SPECULATIVE_CODE_IDENTITY: &str = "speculative-rows-test-build"; +pub(crate) const SPECULATIVE_CACHE_FILE: &str = "target/rules-cache/bulk/native.json"; +const SPECULATIVE_MEMO_FILE: &str = "target/rules-cache/bulk/native-response.json"; + +pub(crate) fn speculative_rows_request( + project_dir: &Path, + cache_enabled: bool, + models: &[Value], +) -> Result { + let mut request: Value = speculative_request_json(project_dir, cache_enabled); + request["models"] = Value::Array(models.to_vec()); + let request: EvaluateRequest = + serde_json::from_value(request).map_err(|error| error.to_string())?; + Ok(RowsRequest { + request, + memo: cache_enabled.then(|| { + ( + SPECULATIVE_CODE_IDENTITY.to_owned(), + project_dir.join(SPECULATIVE_MEMO_FILE), + ) + }), + }) +} + +fn speculative_request_json(project_dir: &Path, cache_enabled: bool) -> Value { + json!({ + "version": 1, + "project_dir": project_dir, + "dialect": "duckdb", + "config": {"select": ["SQBR"], "domains": ["commerce"], "cache": {"enabled": cache_enabled}}, + "models": [], + "defer_suppressions": true + }) +} + +pub(crate) fn changed_orders_models(query: Option<&str>) -> Vec { + let mut models: Vec = orders_models(); + let query: Value = query.map_or_else(|| models[0]["query_sql"].clone(), |query| json!(query)); + models[0]["query_sql"] = query.clone(); + models[0]["authored_sql"] = query; + models +} + +pub(crate) fn evaluation_summary( + evaluation: &RowsEvaluation, +) -> (Vec, Vec, u64, u64, u64, bool) { + ( + evaluation.faults.clone(), + evaluation.selected_codes.clone(), + evaluation.evaluated_models, + evaluation.cache_hits, + evaluation.cache_misses, + evaluation.reused, + ) +} + +pub(crate) fn written_rules_files(project_dir: &Path) -> Vec { + [SPECULATIVE_CACHE_FILE, SPECULATIVE_MEMO_FILE] + .iter() + .map(|file| project_dir.join(file)) + .filter(|path| path.is_file()) + .collect() +} + +/// Discard a speculation after it ran a while; it never reports stopping. +pub(crate) fn discard_after_running( + speculative: SpeculativeRows, + _project_dir: &Path, +) -> Result { + std::thread::sleep(std::time::Duration::from_millis(200)); + discard_speculative_rows(speculative); + Ok(false) +} + +/// Answer beside a speculation once the rules run was cancelled; whether the answer stopped. +pub(crate) fn answer_after_stop( + speculative: SpeculativeRows, + project_dir: &Path, +) -> Result { + Ok(matches!( + answer_speculative_rows( + speculative, + speculative_rows_request(project_dir, true, &orders_models())?, + &AtomicBool::new(true), + ), + SpeculativeAnswer::Stopped + )) +} + +/// Drop a speculation at once; it never reports stopping. +pub(crate) fn drop_at_once( + speculative: SpeculativeRows, + _project_dir: &Path, +) -> Result { + drop(speculative); + Ok(false) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs new file mode 100644 index 0000000000..7459c22b5b --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs @@ -0,0 +1,136 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; + +use serde_json::Value; +use tempfile::TempDir; + +use crate::engine::main::answer_speculative_rows::answer_speculative_rows; +use crate::engine::main::evaluate_rows::evaluate_rows; +use crate::engine::main::start_speculative_rows::start_speculative_rows; +use crate::engine::tests::helpers::{ + SPECULATIVE_CACHE_FILE, answer_after_stop, changed_orders_models, discard_after_running, + drop_at_once, evaluation_summary, orders_models, speculative_rows_request, written_rules_files, +}; +use crate::engine::tests::test_types::{SpeculativeRowsTestCase, SpeculativeStopTestCase}; +use crate::models::SpeculativeAnswer; + +#[test] +fn given_speculative_rows_when_answering_then_result_and_writes_match_ordinary_evaluation() +-> Result<(), String> { + let test_cases = [ + SpeculativeRowsTestCase { + description: "identical cached request uses the speculation", + cache_enabled: true, + changed_query: None, + expected_used: true, + }, + SpeculativeRowsTestCase { + description: "identical uncached request uses the speculation", + cache_enabled: false, + changed_query: None, + expected_used: true, + }, + SpeculativeRowsTestCase { + description: "a model changed after the speculation started is evaluated again", + cache_enabled: true, + changed_query: Some("SELECT order_id FROM warehouse.raw.orders"), + expected_used: false, + }, + ]; + for test_case in &test_cases { + let speculated_dir = TempDir::new().map_err(|error| error.to_string())?; + let ordinary_dir = TempDir::new().map_err(|error| error.to_string())?; + let project_dir: &Path = speculated_dir.path(); + let actual_models: Vec = changed_orders_models(test_case.changed_query); + + let speculative = start_speculative_rows(speculative_rows_request( + project_dir, + test_case.cache_enabled, + &orders_models(), + )?) + .map_err(|error| error.to_string())?; + let answered = answer_speculative_rows( + speculative, + speculative_rows_request(project_dir, test_case.cache_enabled, &actual_models)?, + &AtomicBool::new(false), + ); + let expected = evaluate_rows(speculative_rows_request( + ordinary_dir.path(), + test_case.cache_enabled, + &actual_models, + )?) + .map_err(|_| "ordinary evaluation failed".to_owned())?; + + let SpeculativeAnswer::Answered { evaluation, used } = answered else { + return Err(format!("{}: answer stopped", test_case.description)); + }; + let evaluation = evaluation.map_err(|_| "speculative answer failed".to_owned())?; + let cache = |dir: &Path| std::fs::read(dir.join(SPECULATIVE_CACHE_FILE)).ok(); + assert_eq!(used, test_case.expected_used, "{}", test_case.description); + assert_eq!( + evaluation_summary(&evaluation), + evaluation_summary(&expected), + "{}", + test_case.description + ); + assert_eq!( + cache(project_dir), + cache(ordinary_dir.path()), + "{}", + test_case.description + ); + assert_eq!( + written_rules_files(project_dir).len(), + written_rules_files(ordinary_dir.path()).len(), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_speculative_rows_when_discarded_or_stopped_then_nothing_is_written() -> Result<(), String> +{ + let test_cases = [ + SpeculativeStopTestCase { + description: "discarding after it ran a while", + end: discard_after_running, + expected_stopped: false, + }, + SpeculativeStopTestCase { + description: "answering after the rules run was cancelled", + end: answer_after_stop, + expected_stopped: true, + }, + SpeculativeStopTestCase { + description: "dropping it at once", + end: drop_at_once, + expected_stopped: false, + }, + ]; + for test_case in &test_cases { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let speculative = start_speculative_rows(speculative_rows_request( + project_dir.path(), + true, + &orders_models(), + )?) + .map_err(|error| error.to_string())?; + + let stopped: bool = (test_case.end)(speculative, project_dir.path())?; + + assert_eq!( + stopped, test_case.expected_stopped, + "{}", + test_case.description + ); + assert_eq!( + written_rules_files(project_dir.path()), + Vec::::new(), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index a4e58fe74e..22e099d757 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -259,3 +259,16 @@ pub(crate) struct DependencyTokenTestCase { pub expected_messages: &'static [&'static str], pub expected_columns: &'static [usize], } + +pub(crate) struct SpeculativeRowsTestCase { + pub(crate) description: &'static str, + pub(crate) cache_enabled: bool, + pub(crate) changed_query: Option<&'static str>, + pub(crate) expected_used: bool, +} + +pub(crate) struct SpeculativeStopTestCase { + pub(crate) description: &'static str, + pub(crate) end: fn(crate::models::SpeculativeRows, &std::path::Path) -> Result, + pub(crate) expected_stopped: bool, +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 5b57a1a7d4..c98860e8ff 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -1,4 +1,6 @@ use serde::{Deserialize, Serialize}; + +use crate::errors::RowsError; use serde_json::Value; use std::collections::BTreeMap; @@ -926,6 +928,30 @@ pub struct RowsRequest { pub memo: Option<(String, std::path::PathBuf)>, } +/// A built-in rules evaluation running speculatively on its own thread; dropping it cancels it. +#[derive(Debug)] +pub struct SpeculativeRows { + pub(crate) key: std::sync::Mutex>>, + pub(crate) thread: Option< + std::thread::JoinHandle< + Result, String>, + >, + >, + pub(crate) cancel: std::sync::Arc, +} + +/// How a request was answered beside a speculative evaluation. +#[derive(Debug)] +pub enum SpeculativeAnswer { + /// The request's evaluation, and whether the speculation evaluated exactly this request. + Answered { + evaluation: Result, + used: bool, + }, + /// `stop` was set while waiting; nothing was evaluated or written. + Stopped, +} + /// A rows request's decoded response. #[derive(Debug, Deserialize)] pub struct RowsEvaluation { From 541d999aca9a4446cd116065cfc351105f59b83f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 22:22:04 +0100 Subject: [PATCH 26/28] feat: report early built-in rules timings in compile JSON Add early_rules_build_ms, early_rules_wait_ms, early_rules_hits and early_rules_misses to compile_timings. --- crates/sqlbuild-core/src/compile_timings/constants.rs | 8 ++++++-- .../tests/test_compile_timing_milliseconds.rs | 4 ++-- .../cli/commands/main/build/test_compile_json_behavior.py | 4 ++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-core/src/compile_timings/constants.rs b/crates/sqlbuild-core/src/compile_timings/constants.rs index 00f84cac18..785f37672c 100644 --- a/crates/sqlbuild-core/src/compile_timings/constants.rs +++ b/crates/sqlbuild-core/src/compile_timings/constants.rs @@ -1,7 +1,7 @@ //! The compile phases and metrics `--json` reports, in presentation order. /// Phases whose elapsed nanoseconds are reported as whole milliseconds. -pub const COMPILE_TIMING_PHASES: [&str; 13] = [ +pub const COMPILE_TIMING_PHASES: [&str; 15] = [ "attachment_ms", "model_analysis_ms", "model_analysis_cpu_ms", @@ -15,14 +15,18 @@ pub const COMPILE_TIMING_PHASES: [&str; 13] = [ "cache_publication_ms", "physical_write_ms", "stale_traversal_ms", + "early_rules_build_ms", + "early_rules_wait_ms", ]; /// Counters reported as recorded. -pub const COMPILE_METRICS: [&str; 6] = [ +pub const COMPILE_METRICS: [&str; 8] = [ "analysis_batch_cache_hits", "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", "sql_test_scan_cache_hits", "sql_test_scan_cache_misses", + "early_rules_hits", + "early_rules_misses", ]; diff --git a/crates/sqlbuild-core/src/compile_timings/tests/test_compile_timing_milliseconds.rs b/crates/sqlbuild-core/src/compile_timings/tests/test_compile_timing_milliseconds.rs index ea3cfee116..09eddb3a4b 100644 --- a/crates/sqlbuild-core/src/compile_timings/tests/test_compile_timing_milliseconds.rs +++ b/crates/sqlbuild-core/src/compile_timings/tests/test_compile_timing_milliseconds.rs @@ -26,9 +26,9 @@ fn given_recorded_timings_when_reported_then_lists_every_phase_and_metric_in_ord ]; for test_case in test_cases { let entries = compile_timing_milliseconds(&test_case.timings); - assert_eq!(entries.len(), 19, "{}", test_case.description); + assert_eq!(entries.len(), 23, "{}", test_case.description); assert_eq!( - vec![entries[0], entries[18]], + vec![entries[0], entries[20]], test_case.expected_entries, "{}", test_case.description diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index cbdbaa803f..0107e1a80e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -87,12 +87,16 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "cache_publication_ms", "physical_write_ms", "stale_traversal_ms", + "early_rules_build_ms", + "early_rules_wait_ms", "analysis_batch_cache_hits", "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", "sql_test_scan_cache_hits", "sql_test_scan_cache_misses", + "early_rules_hits", + "early_rules_misses", "project_reuse_hits", "project_reuse_misses", "project_reuse_bypasses", From 5162915bc6160112e3b45fad64c5b575ff6daac9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 22:22:04 +0100 Subject: [PATCH 27/28] perf: start built-in rules once compile recovers model types After type recovery the compile builds the built-in rules request from the recovered project and starts its evaluation on a background thread. The rules phase builds its request as before and uses the early result only when the request keys match. Projects with custom rules, model selections and failed graphs keep the existing path. --- .../_helpers/compile_driver/analysis.rs | 20 +- .../_helpers/compiled_project/project.rs | 8 + .../_helpers/rules/early_builtin_rules.rs | 266 ++++++++++++++++++ .../src/bindings/_helpers/rules/mod.rs | 1 + .../src/bindings/_helpers/rules/rows.rs | 83 ++++-- .../bindings/_helpers/rules/rule_dispatch.rs | 122 ++++++-- .../src/bindings/_helpers/rules/rule_plan.rs | 36 ++- .../_helpers/semantic_checks/stage.rs | 2 + .../cli/commands/main/compile/_test_types.py | 25 ++ .../cli/commands/main/compile/helpers.py | 25 ++ .../test_compile_early_builtin_rules.py | 169 +++++++++++ 11 files changed, 696 insertions(+), 61 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_builtin_rules.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 9384073940..23f91269da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::constants::RULE_FIX_AVAILABLE_NOTE; +use crate::bindings::_helpers::rules::early_builtin_rules::{EarlyRulesScope, EarlyRulesSettings}; use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::rules_progress; use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; @@ -150,7 +151,10 @@ pub(crate) fn analyze<'py>( let enabled: bool = !(native_field(selection, "select")?.is_truthy()? || native_field(selection, "exclude")?.is_truthy()?); let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; - let analysis: PyResult> = analyzed(py, &options, &early_lint); + let early_rules: EarlyRulesScope = EarlyRulesScope::new(); + let analysis: PyResult> = + analyzed(py, &options, (&early_lint, enabled.then_some(&early_rules))); + early_rules.stop(py); let _ = early_lint.borrow().stop(py); analysis } @@ -172,11 +176,11 @@ struct ContractPhase<'py> { cpu_ms: i64, } -/// `_analyze_compile_project`. +/// `_analyze_compile_project`; `early_rules` is armed once the dialect is known, when present. fn analyzed<'py>( py: Python<'py>, options: &AnalysisOptions<'py>, - early_lint: &Bound<'py, EarlySqlLint>, + (early_lint, early_rules): (&Bound<'py, EarlySqlLint>, Option<&EarlyRulesScope>), ) -> PyResult> { let driver: Driver<'_, 'py> = Driver { py, @@ -194,12 +198,22 @@ fn analyzed<'py>( } else { PyString::new(py, GENERIC_DIALECT).into_any() }; + if let Some(early_rules) = early_rules { + early_rules.arm(EarlyRulesSettings { + project_dir: options.project_dir.extract()?, + dialect: lint_dialect.extract()?, + no_sql_analysis: options.no_sql_validation.is_truthy()?, + }); + } let (graph, graph_ms): (Bound<'py, PyAny>, u64) = driver.graph((&discovered, &adapter), &lint_dialect)?; let project: Bound<'py, PyAny> = native_field(&graph, "project")?; let graph_failed: bool = any_error(&native_field(&project, "diagnostics")?)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { + if let Some(early_rules) = early_rules { + early_rules.stop(py); + } early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index 8552f316c5..ef803053de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -131,6 +131,14 @@ impl NativeCompiledProject { &self.facts } + /// A project holding only `facts`, which analyses can be recorded into apart from the compile's. + pub(crate) fn with_facts(facts: CompiledProjectFacts) -> Self { + Self { + facts, + ..Self::default() + } + } + /// Retain facts a producer finished natively. pub(crate) fn retain(&mut self, facts: CompiledModelFacts) { record_model(&mut self.facts, facts); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs new file mode 100644 index 0000000000..6dcf6ea299 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -0,0 +1,266 @@ +//! Built-in rule evaluation started speculatively once a compile's model types are recovered. + +use std::cell::RefCell; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +use pyo3::exceptions::{PyException, PyRuntimeError}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_rules::engine::main::answer_speculative_rows::answer_speculative_rows; +use sqlbuild_rules::engine::main::discard_speculative_rows::discard_speculative_rows; +use sqlbuild_rules::engine::main::start_speculative_rows::start_speculative_rows; +use sqlbuild_rules::errors::RowsError; +use sqlbuild_rules::models::{RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::compile_timings::{ + add_compile_timing, record_compile_metric, record_compile_timing, +}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::records::{ + compiled_project_facts, record_model_analyses, +}; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::rules::catalogue::references_custom_rules; +use crate::bindings::_helpers::rules::rule_dispatch::speculative_rules_request; +use crate::bindings::_helpers::rules::rule_plan::rule_run_plan; +use crate::bindings::_helpers::rules::rules_config::load_rules_config; +use crate::bindings::types::CompilerDetach; + +const CANCELLED: &str = "native rules evaluation was cancelled"; +const BUILD_PHASE: &str = "early_rules_build_ms"; +const WAIT_PHASE: &str = "early_rules_wait_ms"; +const HIT_METRIC: &str = "early_rules_hits"; +const MISS_METRIC: &str = "early_rules_misses"; + +/// What one compile's early built-in rules start from. +pub(crate) struct EarlyRulesSettings { + pub(crate) project_dir: PathBuf, + pub(crate) dialect: String, + pub(crate) no_sql_analysis: bool, +} + +/// One compile thread's early built-in rules, from arming to the rules phase taking them. +enum Slot { + Armed(EarlyRulesSettings), + Started(EarlyBuiltinRules), + Spent, +} + +thread_local! { + static SLOT: RefCell> = const { RefCell::new(None) }; +} + +/// Whether the rules phase used the early evaluation, and how long it waited for it. +#[derive(Default)] +pub(crate) struct EarlyRulesReport { + used: AtomicBool, + consulted: AtomicBool, + wait_ns: std::sync::atomic::AtomicU64, +} + +impl EarlyRulesReport { + /// Record the outcome in the compile timings of the calling thread. + pub(crate) fn record(&self) { + if !self.consulted.load(Ordering::Acquire) { + return; + } + add_compile_timing(WAIT_PHASE, self.wait_ns.load(Ordering::Acquire)); + let used: bool = self.used.load(Ordering::Acquire); + record_compile_metric(HIT_METRIC, i64::from(used)); + record_compile_metric(MISS_METRIC, i64::from(!used)); + } +} + +/// Arms early built-in rules for one compile analysis on this thread; dropping it stops them. +pub(crate) struct EarlyRulesScope { + owner: bool, +} + +impl EarlyRulesScope { + /// Claim this thread's slot unarmed; a nested compile analysis on this thread claims nothing. + pub(crate) fn new() -> Self { + let owner: bool = SLOT.with(|slot| { + let mut slot = slot.borrow_mut(); + if slot.is_some() { + return false; + } + *slot = Some(Slot::Spent); + true + }); + Self { owner } + } + + /// Start early rules with `settings` once the compile recovers its model types. + pub(crate) fn arm(&self, settings: EarlyRulesSettings) { + if !self.owner { + return; + } + SLOT.with(|slot| { + if let Some(current @ Slot::Spent) = slot.borrow_mut().as_mut() { + *current = Slot::Armed(settings); + } + }); + } + + /// Cancel and join early rules this scope armed that started and were not taken. + pub(crate) fn stop(&self, py: Python<'_>) { + if self.owner + && let Some(rules) = take_started() + { + rules.discard(py); + } + } +} + +impl Drop for EarlyRulesScope { + fn drop(&mut self) { + if !self.owner { + return; + } + let slot: Option = SLOT.with(|slot| slot.borrow_mut().take()); + drop(slot); + } +} + +/// The started early rules, which only the rules phase of the arming compile takes. +pub(crate) fn take_early_rules() -> Option { + take_started() +} + +fn take_started() -> Option { + SLOT.with(|slot| { + let mut slot = slot.borrow_mut(); + match slot.as_mut() { + Some(current) => match std::mem::replace(current, Slot::Spent) { + Slot::Started(rules) => Some(rules), + other => { + *current = other; + None + } + }, + None => None, + } + }) +} + +/// Start early built-in rules over `project` when armed; only non-`Exception` errors propagate. +pub(crate) fn start_early_rules(project: &Bound<'_, PyAny>) -> PyResult<()> { + let settings: Option = SLOT.with(|slot| { + let mut slot = slot.borrow_mut(); + match slot.as_mut() { + Some(current @ Slot::Armed(_)) => match std::mem::replace(current, Slot::Spent) { + Slot::Armed(settings) => Some(settings), + _ => None, + }, + _ => None, + } + }); + let Some(settings) = settings else { + return Ok(()); + }; + let py: Python<'_> = project.py(); + let started: Option = + match record_compile_timing(BUILD_PHASE, || early_request(project, &settings)) { + Ok(Some(rows)) => Some(EarlyBuiltinRules::spawn(rows)?), + Ok(None) => None, + Err(error) if error.is_instance_of::(py) => None, + Err(error) => return Err(error), + }; + if let Some(started) = started { + SLOT.with(|slot| { + if let Some(current) = slot.borrow_mut().as_mut() { + *current = Slot::Started(started); + } + }); + } + Ok(()) +} + +/// The built-in rules request the rules phase is expected to build, when it evaluates no custom rules. +fn early_request( + project: &Bound<'_, PyAny>, + settings: &EarlyRulesSettings, +) -> PyResult> { + let py: Python<'_> = project.py(); + let config: Bound<'_, PyAny> = + load_rules_config(py, settings.project_dir.clone())?.into_bound(py); + if references_custom_rules(&config)? { + return Ok(None); + } + let project_dir: Bound<'_, PyAny> = settings.project_dir.clone().into_pyobject(py)?.into_any(); + let plan: Bound<'_, PyAny> = rule_run_plan( + (project, &config, &project_dir), + None, + settings.no_sql_analysis, + )?; + if field(&plan, "selected")?.len()? == 0 || field(&plan, "custom_payloads")?.is_truthy()? { + return Ok(None); + } + let facts: Bound<'_, NativeCompiledProject> = Bound::new( + py, + NativeCompiledProject::with_facts( + compiled_project_facts(project)?.borrow().facts().clone(), + ), + )?; + let _ = record_model_analyses(&facts, &field(project, "models")?)?; + speculative_rules_request(&plan, &settings.dialect, &facts).map(Some) +} + +/// One speculative built-in rules evaluation and the report of how the rules phase used it. +pub(crate) struct EarlyBuiltinRules { + speculative: SpeculativeRows, + report: Arc, +} + +impl EarlyBuiltinRules { + fn spawn(rows: RowsRequest) -> PyResult { + Ok(Self { + speculative: start_speculative_rows(rows) + .map_err(|error| PyRuntimeError::new_err(error.to_string()))?, + report: Arc::new(EarlyRulesReport::default()), + }) + } + + /// The outcome report the rules phase records once it has used or discarded these rules. + pub(crate) fn report(&self) -> Arc { + Arc::clone(&self.report) + } + + /// Evaluate `rows`, from this speculation only when identical; `interrupted` stops the wait. + pub(crate) fn answer( + self, + py: Python<'_>, + rows: RowsRequest, + interrupted: &AtomicBool, + ) -> PyResult> { + let Self { + speculative, + report, + } = self; + report.consulted.store(true, Ordering::Release); + let waited: Instant = Instant::now(); + let answered: SpeculativeAnswer = py + .compiler_detach(|| Ok(answer_speculative_rows(speculative, rows, interrupted))) + .map_err(value_error)?; + report.wait_ns.store( + u64::try_from(waited.elapsed().as_nanos()).unwrap_or(u64::MAX), + Ordering::Release, + ); + match answered { + SpeculativeAnswer::Answered { evaluation, used } => { + report.used.store(used, Ordering::Release); + Ok(evaluation) + } + SpeculativeAnswer::Stopped => Ok(Err(RowsError::Rules(CANCELLED.to_owned()))), + } + } + + /// Cancel the speculation and wait for its thread without holding the interpreter. + pub(crate) fn discard(self, py: Python<'_>) { + let speculative: SpeculativeRows = self.speculative; + py.detach(|| discard_speculative_rows(speculative)); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index f39c6ee92c..4b2670b1b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod custom_hosts; pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; +pub(crate) mod early_builtin_rules; pub(crate) mod early_sql_lint; pub(crate) mod evaluation; pub(crate) mod fact_encoder; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 613df85f12..cef85ca350 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use std::path::PathBuf; use std::sync::Mutex; +use std::sync::atomic::AtomicBool; use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; use pyo3::prelude::{ @@ -38,6 +39,7 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::config_json::asdict_json; +use crate::bindings::_helpers::rules::early_builtin_rules::EarlyBuiltinRules; use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; use crate::bindings::_helpers::rules::selection::rules_error; @@ -107,7 +109,34 @@ pub(crate) fn build_rules_request( ) -> PyResult { let request = Request::new("build_rules_request", options)?; let py: Python<'_> = request.py(); - as_rules_error(py, built_request(&request)) + as_rules_error( + py, + built_request(&request, None).map(|request| NativeRulesRequest { + request: Mutex::new(Some(request)), + }), + ) +} + +/// `build_rules_request`'s request, reading model facts from `facts` instead of the project. +pub(crate) fn rules_request_over( + options: &Bound<'_, PyDict>, + facts: &Bound<'_, NativeCompiledProject>, +) -> PyResult { + built_request( + &Request::new("build_rules_request", Some(options))?, + Some(facts), + ) +} + +/// The built request `request` holds, which only one evaluation may take. +pub(crate) fn taken_rows(request: &Bound<'_, NativeRulesRequest>) -> PyResult { + request + .borrow() + .request + .lock() + .map_err(|_| value_error("rules request lock is poisoned"))? + .take() + .ok_or_else(|| value_error("rules request was already evaluated")) } /// The model-free request of a run selecting no rules, which only validates its configuration. @@ -167,7 +196,10 @@ fn request_header( Ok(header) } -fn built_request(options: &Request<'_, '_>) -> PyResult { +fn built_request( + options: &Request<'_, '_>, + facts: Option<&Bound<'_, NativeCompiledProject>>, +) -> PyResult { let py: Python<'_> = options.py(); let project: Bound<'_, PyAny> = options.get("project")?; let config: Bound<'_, PyAny> = options.get("config")?; @@ -227,7 +259,10 @@ fn built_request(options: &Request<'_, '_>) -> PyResult { .collect::>()?, dialect, }; - let facts: Bound<'_, NativeCompiledProject> = compiled_project_facts(&project)?; + let facts: Bound<'_, NativeCompiledProject> = match facts { + Some(facts) => facts.clone(), + None => compiled_project_facts(&project)?, + }; let names: Vec = project_models(&project)? .into_iter() .map(|model| model.name) @@ -262,9 +297,7 @@ fn built_request(options: &Request<'_, '_>) -> PyResult { .memo .take() .map(|(identity, path)| (identity, PathBuf::from(path))); - Ok(NativeRulesRequest { - request: Mutex::new(Some(RowsRequest { request, memo })), - }) + Ok(RowsRequest { request, memo }) }) } @@ -274,27 +307,35 @@ pub(crate) fn evaluate_rules_request<'py>( py: Python<'py>, request: &Bound<'py, NativeRulesRequest>, ) -> PyResult> { - as_rules_error(py, evaluated(py, request)) + as_rules_error(py, evaluated(py, request, None)) +} + +/// `evaluate_rules_request`, answered by `early` when identical; `interrupted` stops the wait. +pub(crate) fn evaluate_rules_request_with<'py>( + py: Python<'py>, + request: &Bound<'py, NativeRulesRequest>, + early: EarlyBuiltinRules, + interrupted: &AtomicBool, +) -> PyResult> { + as_rules_error(py, evaluated(py, request, Some((early, interrupted)))) } fn evaluated<'py>( py: Python<'py>, request: &Bound<'py, NativeRulesRequest>, + early: Option<(EarlyBuiltinRules, &AtomicBool)>, ) -> PyResult> { - let request: RowsRequest = request - .borrow() - .request - .lock() - .map_err(|_| value_error("rules request lock is poisoned"))? - .take() - .ok_or_else(|| value_error("rules request was already evaluated"))?; - let evaluation: RowsEvaluation = py - .compiler_detach(|| Ok(evaluate_rows(request))) - .map_err(value_error)? - .map_err(|error| match error { - RowsError::Rules(message) => value_error(message), - RowsError::Io(error) => PyOSError::new_err(error.to_string()), - })?; + let request: RowsRequest = taken_rows(request)?; + let evaluation: Result = match early { + None => py + .compiler_detach(|| Ok(evaluate_rows(request))) + .map_err(value_error)?, + Some((early, interrupted)) => early.answer(py, request, interrupted)?, + }; + let evaluation: RowsEvaluation = evaluation.map_err(|error| match error { + RowsError::Rules(message) => value_error(message), + RowsError::Io(error) => PyOSError::new_err(error.to_string()), + })?; let classes = FindingClasses::new(py)?; let findings: Vec> = evaluation .faults diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index c1a4cf807c..3d30794f32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, keywords, }; use crate::bindings::_helpers::compile_driver::stage_work::count_stage; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::construct::new_record_with; @@ -34,14 +35,20 @@ use crate::bindings::_helpers::rules::custom_evaluation::{ }; use crate::bindings::_helpers::rules::custom_hosts::HostCancel; use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules; +use crate::bindings::_helpers::rules::early_builtin_rules::{ + EarlyBuiltinRules, EarlyRulesReport, take_early_rules, +}; use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; use crate::bindings::_helpers::rules::rows::{ - NativeRulesRequest, build_rules_request, evaluate_rules_request, selectionless_rules_request, + NativeRulesRequest, build_rules_request, evaluate_rules_request, evaluate_rules_request_with, + rules_request_over, selectionless_rules_request, }; use crate::bindings::_helpers::rules::rule_plan::plan_rule_run; use crate::bindings::_helpers::rules::rule_run::finish_rule_run; use crate::bindings::_helpers::rules::selection::{rule_selected, selected_rule_codes}; use crate::bindings::_helpers::rules::sql_rules::run_planned_sql_rules; +use sqlbuild_rules::models::RowsRequest; +use std::sync::Arc; const CUSTOM_THREAD: &str = "sqlbuild-custom-rules"; const GENERIC_DIALECT: &str = "generic"; @@ -159,7 +166,7 @@ fn evaluate_project_rules<'py>( initial_findings: None, cancel: HostCancel::default(), }; - evaluate_native(&run, (false, None), CustomSource::Deferred) + evaluate_native(&run, (false, None), CustomSource::Deferred, None) } /// Evaluate built-in rules natively with the catalogue's selected custom rules beside them. @@ -183,7 +190,12 @@ fn evaluate_native_rules<'py>( initial_findings: (!initial_findings.is_none()).then_some(initial_findings), cancel: HostCancel::default(), }; - evaluate_native(&run, (defer_suppressions, None), CustomSource::Deferred) + evaluate_native( + &run, + (defer_suppressions, None), + CustomSource::Deferred, + None, + ) } /// Evaluate custom rules through their incremental cache. @@ -269,7 +281,12 @@ pub(crate) fn evaluate_rules<'py>( let native_dialect: String = run.dialect.clone(); let native_cancel: HostCancel = run.cancel.clone(); let selectionless: bool = custom.is_none() && field(&plan, "selected")?.len()? == 0; + let early: Option = take_early_rules(); + let early_report: Option> = early.as_ref().map(EarlyBuiltinRules::report); let native_thread: Option = if selectionless { + if let Some(early) = early { + early.discard(py); + } None } else { Some(PhaseThread::spawn(RULES_THREAD, move || { @@ -289,6 +306,7 @@ pub(crate) fn evaluate_rules<'py>( &run, (true, Some(payloads.into_bound(py))), CustomSource::Started(custom), + early, ) .map(Bound::unbind) }) @@ -309,16 +327,20 @@ pub(crate) fn evaluate_rules<'py>( Some(native_thread) => joined(py, native_thread, &run.cancel), None => selectionless_result(&run, &payloads), }; + if let Some(report) = early_report { + report.record(); + } let sql_result: Bound<'py, PyAny> = sql_result?; let result: Bound<'py, PyAny> = result?; finish_rule_run(&plan, &sql_result, &result) } -/// Evaluate built-in rules natively while custom rules run beside them. +/// Evaluate built-in rules natively, or from `early` when identical, beside the custom rules. fn evaluate_native<'py>( run: &NativeRun<'py>, (defer_suppressions, custom_payloads): (bool, Option>), custom: CustomSource, + early: Option, ) -> PyResult> { let py: Python<'py> = run.project.py(); let prepared: PyResult<(Bound<'py, PyAny>, Bound<'py, NativeRulesRequest>)> = @@ -334,8 +356,19 @@ fn evaluate_native<'py>( (CustomSource::Deferred, Ok((payloads, _))) => start_custom_rules(run, payloads)?, (CustomSource::Deferred, Err(_)) => None, }; - let (_, request): (Bound<'py, PyAny>, Bound<'py, NativeRulesRequest>) = prepared?; - let evaluated: PyResult> = evaluate_rules_request(py, &request); + let (_, request): (Bound<'py, PyAny>, Bound<'py, NativeRulesRequest>) = match prepared { + Ok(prepared) => prepared, + Err(error) => { + if let Some(early) = early { + early.discard(py); + } + return Err(error); + } + }; + let evaluated: PyResult> = match early { + Some(early) => evaluate_rules_request_with(py, &request, early, &run.cancel), + None => evaluate_rules_request(py, &request), + }; cancel_on_interrupt(py, &evaluated, &run.cancel); let custom: PyResult> = match custom { Some(handle) => joined(py, handle, &run.cancel), @@ -448,34 +481,65 @@ fn native_request<'py>( ], )?, }; + let request: Bound<'py, NativeRulesRequest> = Bound::new( + py, + build_rules_request(Some(&request_options(run, &payloads)?))?, + )?; + Ok((payloads, request)) +} + +/// The `build_rules_request` options of one native evaluation with its custom-rule payloads. +fn request_options<'py>( + run: &NativeRun<'py>, + payloads: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = run.project.py(); let mut include_type_proof: bool = false; for code in TYPE_PROOF_RULE_CODES { include_type_proof = include_type_proof || rule_selected(&run.config, code)?; } - let request: Bound<'py, NativeRulesRequest> = Bound::new( + keywords( py, - build_rules_request(Some(&keywords( - py, - &[ - ("project", run.project.clone()), - ("config", run.config.clone()), - ("project_dir", run.project_dir.clone()), - ("dialect", run.dialect.clone().into_bound_py_any(py)?), - ( - "initial_findings", - run.initial_findings - .clone() - .unwrap_or_else(|| PyTuple::empty(py).into_any()), - ), - ("custom_payloads", payloads.clone()), - ( - "include_type_proof", - include_type_proof.into_bound_py_any(py)?, - ), - ], - )?))?, - )?; - Ok((payloads, request)) + &[ + ("project", run.project.clone()), + ("config", run.config.clone()), + ("project_dir", run.project_dir.clone()), + ("dialect", run.dialect.clone().into_bound_py_any(py)?), + ( + "initial_findings", + run.initial_findings + .clone() + .unwrap_or_else(|| PyTuple::empty(py).into_any()), + ), + ("custom_payloads", payloads.clone()), + ( + "include_type_proof", + include_type_proof.into_bound_py_any(py)?, + ), + ], + ) +} + +/// The request `evaluate_rules` would build for `plan`, reading model facts from `facts`. +pub(crate) fn speculative_rules_request<'py>( + plan: &Bound<'py, PyAny>, + dialect: &str, + facts: &Bound<'py, NativeCompiledProject>, +) -> PyResult { + let run: NativeRun<'py> = NativeRun { + project: field(plan, "project")?, + config: field(plan, "config")?, + project_dir: field(plan, "project_dir")?, + catalogue: field(plan, "catalogue")?, + dialect: dialect.to_owned(), + verify_determinism: false, + initial_findings: None, + cancel: HostCancel::default(), + }; + rules_request_over( + &request_options(&run, &field(plan, "custom_payloads")?)?, + facts, + ) } /// Start the selected custom rules on their thread after digesting installed code here. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 60127a6820..5d2745fb88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -36,9 +36,27 @@ pub(crate) fn plan_rule_run<'py>( let graph: Bound<'py, PyAny> = options.get("graph")?; let project: Bound<'py, PyAny> = field(&graph, "project")?; freed_memory_released(py)?; - let effective_config: Bound<'py, PyAny> = - resolved_ignore_selectors(&options.get("config")?, &project)?; - let project_dir: Bound<'py, PyAny> = options.get("project_dir")?.call_method0("resolve")?; + let selected_keys: Bound<'py, PyAny> = options.get("selected_keys")?; + rule_run_plan( + ( + &project, + &options.get("config")?, + &options.get("project_dir")?, + ), + Some((&graph, &selected_keys)), + options.get("no_sql_analysis")?.is_truthy()?, + ) +} + +/// `plan_rule_run`'s plan of `project`, selected by `(graph, selected_keys)` when given. +pub(crate) fn rule_run_plan<'py>( + (project, config, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + selection: Option<(&Bound<'py, PyAny>, &Bound<'py, PyAny>)>, + no_sql_analysis: bool, +) -> PyResult> { + let py: Python<'py> = project.py(); + let effective_config: Bound<'py, PyAny> = resolved_ignore_selectors(config, project)?; + let project_dir: Bound<'py, PyAny> = project_dir.call_method0("resolve")?; let include_custom: bool = references_custom_rules(&effective_config)?; let catalogue: Vec> = rule_catalogue(&effective_config, &project_dir, include_custom)?; @@ -57,8 +75,8 @@ pub(crate) fn plan_rule_run<'py>( Some(custom_payloads.clone()), )?; let mut skipped_type_proof_rules: Vec> = Vec::new(); - if options.get("no_sql_analysis")?.is_truthy()? - || !field(&project_attribute(&project, "settings")?, "sql_analysis")?.is_truthy()? + if no_sql_analysis + || !field(&project_attribute(project, "settings")?, "sql_analysis")?.is_truthy()? { let mut kept: Vec> = Vec::with_capacity(selected.len()); for rule in selected { @@ -80,11 +98,13 @@ pub(crate) fn plan_rule_run<'py>( if !custom_rules.is_empty() { verify_custom_rules(&custom_rules, &project_dir)?; } - let model_paths: Option> = - selected_model_paths(&graph, &options.get("selected_keys")?)?; + let model_paths: Option> = match selection { + Some((graph, selected_keys)) => selected_model_paths(graph, selected_keys)?, + None => None, + }; let selected_project: Bound<'py, PyAny> = match &model_paths { None => project.clone(), - Some(paths) => selected_models_project(&project, paths)?, + Some(paths) => selected_models_project(project, paths)?, }; let mut codes: Vec> = Vec::with_capacity(selected.len()); for rule in &selected { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs index b89ec1db02..14e1670531 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSes use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::rules::early_builtin_rules::start_early_rules; use crate::bindings::_helpers::semantic_checks::completion::completed_diagnostics; use crate::bindings::_helpers::semantic_checks::metadata::metadata_diagnostics; use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; @@ -71,6 +72,7 @@ pub(crate) fn completed_semantic_project<'py>( }; let recovered: Bound<'py, PyAny> = recovered_types(&classes, &project, &binding_results, &shared)?; + start_early_rules(&recovered)?; let metadata: Vec> = metadata_diagnostics( &classes, (&recovered, &profile), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index f25c77f4dc..7ba3c3653d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1023,3 +1023,28 @@ class DollarQuotedProjectVarCompileCase: description: str model_sql: str expected_fragment: str + + +@dataclass(frozen=True) +class EarlyBuiltinRulesTestCase: + """A rules-enabled project, the compile arguments and the early rules outcome it must report.""" + + description: str + repo_files: dict[str, str] + compile_arguments: tuple[str, ...] + expected_exit_code: int + expected_diagnostics: tuple[tuple[str, str, int, int], ...] + expected_early_hits: int + expected_early_misses: int + + +@dataclass(frozen=True) +class EarlyBuiltinRulesEditTestCase: + """A model edit between compiles and the early rules and rule cache outcomes of each compile.""" + + description: str + edited_path: str + edited_sql: str + expected_diagnostics: tuple[tuple[str, str, int, int], ...] + expected_early_outcomes: tuple[tuple[int, int], ...] + expected_rule_cache_outcomes: tuple[tuple[int, int], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 489c5722af..284f856294 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2964,6 +2964,31 @@ def compile_json_payload(*, project_dir: Path) -> tuple[int, dict[str, Any]]: return result.returncode, json.loads(result.stdout) +def compile_json_with( + *, project_dir: Path, arguments: tuple[str, ...] +) -> tuple[int, dict[str, Any]]: + """Run one JSON compile with extra arguments through the CLI and return its exit and payload.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json", *arguments), project_dir=project_dir + ) + return result.returncode, json.loads(result.stdout) + + +def early_rules_outcome(payload: dict[str, Any]) -> tuple[int, int]: + """The early built-in rules hits and misses one compile payload reports.""" + + timings: dict[str, Any] = payload["compile_timings"] + return timings["early_rules_hits"], timings["early_rules_misses"] + + +def rule_cache_outcome(payload: dict[str, Any]) -> tuple[int, int]: + """The rule cache hits and misses one compile payload reports.""" + + timings: dict[str, Any] = payload["compile_timings"] + return timings["rule_cache_hits"], timings["rule_cache_misses"] + + def diagnostic_location_keys(payload: dict[str, Any]) -> tuple[tuple[str, str, int, int], ...]: """Sorted code, path, line, and column of every diagnostic in a compile payload.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_builtin_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_builtin_rules.py new file mode 100644 index 0000000000..3f62424a1f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_builtin_rules.py @@ -0,0 +1,169 @@ +"""E2E tests for built-in rules evaluated speculatively during compilation.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + EarlyBuiltinRulesEditTestCase, + EarlyBuiltinRulesTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + compile_json_with, + diagnostic_location_keys, + early_rules_outcome, + rule_cache_outcome, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project + +_BUILTIN_PROJECT_TOML: str = ( + 'name = "commerce"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRMODEL102"]\n' +) +_CUSTOM_PROJECT_TOML: str = ( + 'name = "commerce"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRMODEL102", "XSQBR"]\n' + "\n[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" +) +_ORDERS_SQL: str = 'MODEL (description "Orders");\nSELECT 1 AS order_id, 2 AS amount\n' +_STAR_TOTALS_SQL: str = 'MODEL (description "Order totals");\nSELECT * FROM __ref("orders")\n' +_BROKEN_TOTALS_SQL: str = ( + 'MODEL (description "Order totals");\nSELECT order_id, missing_column FROM __ref("orders")\n' +) +_LONG_NAME_RULE: str = ( + "from sqlbuild.rules import Finding, Model, RuleContext, rule\n\n\n" + '@rule(code="XSQBRNAME001", message="Model names stay short", ' + 'remediation="Rename the model.")\n' + "def short_names(*, model: Model, ctx: RuleContext) -> list[Finding]:\n" + " if len(model.name) <= 8:\n" + " return []\n" + " return [ctx.finding(subject=model)]\n" +) +_STAR_FINDING: tuple[str, str, int, int] = ("SQBRMODEL102", "models/order_totals.sql", 1, 8) + + +@pytest.mark.parametrize( + "test_case", + ( + EarlyBuiltinRulesTestCase( + description="clean graph answers built-in rules from the early evaluation", + repo_files={ + "sqlbuild_project.toml": _BUILTIN_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _STAR_TOTALS_SQL, + }, + compile_arguments=(), + expected_exit_code=1, + expected_diagnostics=(_STAR_FINDING,), + expected_early_hits=1, + expected_early_misses=0, + ), + EarlyBuiltinRulesTestCase( + description="graph error skips rules and discards the early evaluation", + repo_files={ + "sqlbuild_project.toml": _BUILTIN_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _BROKEN_TOTALS_SQL, + }, + compile_arguments=(), + expected_exit_code=1, + expected_diagnostics=(("B002", "models/order_totals.sql", 2, 18),), + expected_early_hits=0, + expected_early_misses=0, + ), + EarlyBuiltinRulesTestCase( + description="model selection evaluates rules without an early evaluation", + repo_files={ + "sqlbuild_project.toml": _BUILTIN_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _STAR_TOTALS_SQL, + }, + compile_arguments=("--select", "order_totals"), + expected_exit_code=1, + expected_diagnostics=(_STAR_FINDING,), + expected_early_hits=0, + expected_early_misses=0, + ), + EarlyBuiltinRulesTestCase( + description="custom rules keep built-in rules in the rules phase", + repo_files={ + "sqlbuild_project.toml": _CUSTOM_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _STAR_TOTALS_SQL, + "rules/naming.py": _LONG_NAME_RULE, + }, + compile_arguments=(), + expected_exit_code=1, + expected_diagnostics=( + _STAR_FINDING, + ("XSQBRNAME001", "models/order_totals.sql", 1, 1), + ), + expected_early_hits=0, + expected_early_misses=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_rules_project_when_compiling_cold_and_warm_then_early_rules_keep_findings( + test_case: EarlyBuiltinRulesTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="commerce", repo_files=test_case.repo_files + ) + + runs: list[tuple[int, dict[str, Any]]] = [ + compile_json_with(project_dir=project_dir, arguments=test_case.compile_arguments) + for _ in range(2) + ] + + for exit_code, payload in runs: + assert exit_code == test_case.expected_exit_code + assert diagnostic_location_keys(payload) == test_case.expected_diagnostics + assert early_rules_outcome(payload) == ( + test_case.expected_early_hits, + test_case.expected_early_misses, + ) + + +@pytest.mark.parametrize( + "test_case", + ( + EarlyBuiltinRulesEditTestCase( + description="editing one model reuses the other model's cached findings", + edited_path="models/orders.sql", + edited_sql='MODEL (description "Orders");\nSELECT 1 AS order_id, 3 AS amount\n', + expected_diagnostics=(_STAR_FINDING,), + expected_early_outcomes=((1, 0), (1, 0), (1, 0)), + expected_rule_cache_outcomes=((0, 2), (2, 0), (1, 1)), + ), + ), + ids=lambda case: case.description, +) +def test_given_early_rules_when_compiling_cold_warm_and_edited_then_rule_cache_still_hits( + test_case: EarlyBuiltinRulesEditTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="commerce", + repo_files={ + "sqlbuild_project.toml": _BUILTIN_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _STAR_TOTALS_SQL, + }, + ) + + payloads: list[dict[str, Any]] = [compile_json_with(project_dir=project_dir, arguments=())[1]] + payloads.append(compile_json_with(project_dir=project_dir, arguments=())[1]) + (project_dir / test_case.edited_path).write_text(test_case.edited_sql, encoding="utf-8") + payloads.append(compile_json_with(project_dir=project_dir, arguments=())[1]) + + assert diagnostic_location_keys(payloads[-1]) == test_case.expected_diagnostics + assert tuple(early_rules_outcome(payload) for payload in payloads) == ( + test_case.expected_early_outcomes + ) + assert tuple(rule_cache_outcome(payload) for payload in payloads) == ( + test_case.expected_rule_cache_outcomes + ) From 1ef47adf4abb448b7d0f60c3584f3658a66dd82d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 17 Oct 2026 23:32:08 +0100 Subject: [PATCH 28/28] fix: treat failed early built-in rules as a miss A thread that cannot start, or a speculative evaluation that fails, no longer fails the compile: the rules phase evaluates the request as before. --- .../_helpers/rules/early_builtin_rules.rs | 19 +++++---- .../src/engine/_helpers/memo.rs | 13 ++++++ .../src/engine/_helpers/speculation.rs | 2 +- .../src/engine/tests/test_speculative_rows.rs | 42 ++++++++++++++++++- .../src/engine/tests/test_types.rs | 6 +++ 5 files changed, 72 insertions(+), 10 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs index 6dcf6ea299..ad7205070d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; -use pyo3::exceptions::{PyException, PyRuntimeError}; +use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_rules::engine::main::answer_speculative_rows::answer_speculative_rows; use sqlbuild_rules::engine::main::discard_speculative_rows::discard_speculative_rows; @@ -164,7 +164,7 @@ pub(crate) fn start_early_rules(project: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = project.py(); let started: Option = match record_compile_timing(BUILD_PHASE, || early_request(project, &settings)) { - Ok(Some(rows)) => Some(EarlyBuiltinRules::spawn(rows)?), + Ok(Some(rows)) => EarlyBuiltinRules::spawn(rows), Ok(None) => None, Err(error) if error.is_instance_of::(py) => None, Err(error) => return Err(error), @@ -216,12 +216,15 @@ pub(crate) struct EarlyBuiltinRules { } impl EarlyBuiltinRules { - fn spawn(rows: RowsRequest) -> PyResult { - Ok(Self { - speculative: start_speculative_rows(rows) - .map_err(|error| PyRuntimeError::new_err(error.to_string()))?, - report: Arc::new(EarlyRulesReport::default()), - }) + /// The started speculation, or `None` when its thread cannot start. + fn spawn(rows: RowsRequest) -> Option { + match start_speculative_rows(rows) { + Ok(speculative) => Some(Self { + speculative, + report: Arc::new(EarlyRulesReport::default()), + }), + Err(_) => None, + } } /// The outcome report the rules phase records once it has used or discarded these rules. diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index f63d05f412..b49f3ff516 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -105,6 +105,19 @@ pub(crate) fn speculate(prepared: PreparedRows, cancel: &AtomicBool) -> Speculat } /// Make a speculative evaluation's pending writes in order and answer as `evaluate_prepared` would. +impl SpeculativeOutcome { + /// Whether the speculation produced a response, so a failure is retried by the real request. + pub(crate) fn evaluated_cleanly(&self) -> bool { + !matches!( + self, + Self::Evaluated { + response: Err(_), + .. + } + ) + } +} + pub(crate) fn commit(speculative: SpeculativeOutcome) -> Result { match speculative { SpeculativeOutcome::Reused(response) => decode(&response, true), diff --git a/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs b/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs index 06bdf637c2..4df1f3b38b 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs @@ -68,7 +68,7 @@ pub(crate) fn answer( return SpeculativeAnswer::Stopped; } match speculative.thread.take().map(std::thread::JoinHandle::join) { - Some(Ok(Ok(outcome))) => outcome, + Some(Ok(Ok(outcome))) => outcome.filter(SpeculativeOutcome::evaluated_cleanly), _ => None, } } else { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs index 7459c22b5b..f8c2c47933 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs @@ -4,6 +4,7 @@ use std::sync::atomic::AtomicBool; use serde_json::Value; use tempfile::TempDir; +use crate::engine::_helpers::memo::SpeculativeOutcome; use crate::engine::main::answer_speculative_rows::answer_speculative_rows; use crate::engine::main::evaluate_rows::evaluate_rows; use crate::engine::main::start_speculative_rows::start_speculative_rows; @@ -11,7 +12,9 @@ use crate::engine::tests::helpers::{ SPECULATIVE_CACHE_FILE, answer_after_stop, changed_orders_models, discard_after_running, drop_at_once, evaluation_summary, orders_models, speculative_rows_request, written_rules_files, }; -use crate::engine::tests::test_types::{SpeculativeRowsTestCase, SpeculativeStopTestCase}; +use crate::engine::tests::test_types::{ + SpeculativeOutcomeTestCase, SpeculativeRowsTestCase, SpeculativeStopTestCase, +}; use crate::models::SpeculativeAnswer; #[test] @@ -134,3 +137,40 @@ fn given_speculative_rows_when_discarded_or_stopped_then_nothing_is_written() -> } Ok(()) } + +#[test] +fn given_speculative_outcome_when_checked_then_only_failed_evaluations_are_retried() { + let test_cases = [ + SpeculativeOutcomeTestCase { + description: "memo reuse is kept", + outcome: || SpeculativeOutcome::Reused("{}".to_owned()), + expected_clean: true, + }, + SpeculativeOutcomeTestCase { + description: "evaluated response is kept", + outcome: || SpeculativeOutcome::Evaluated { + bucket: None, + response: Ok("{}".to_owned()), + memo: None, + }, + expected_clean: true, + }, + SpeculativeOutcomeTestCase { + description: "failed evaluation is retried by the real request", + outcome: || SpeculativeOutcome::Evaluated { + bucket: None, + response: Err("worker pool unavailable".to_owned()), + memo: None, + }, + expected_clean: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + (test_case.outcome)().evaluated_cleanly(), + test_case.expected_clean, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 22e099d757..6d69c0b50b 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -272,3 +272,9 @@ pub(crate) struct SpeculativeStopTestCase { pub(crate) end: fn(crate::models::SpeculativeRows, &std::path::Path) -> Result, pub(crate) expected_stopped: bool, } + +pub(crate) struct SpeculativeOutcomeTestCase { + pub(crate) description: &'static str, + pub(crate) outcome: fn() -> crate::engine::_helpers::memo::SpeculativeOutcome, + pub(crate) expected_clean: bool, +}