diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs index c99bb198be..a3423f5705 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/models.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -73,9 +73,13 @@ pub struct SchemaColumnFact { pub name: String, pub data_type: Option, pub nullable: Option, - pub audit_count: u32, - /// Whether the column is declared outside the model file, in a named schema. - pub declared_outside_model: bool, + /// Whether the column declares a description. + pub described: bool, + pub migrate_from: Option, + /// Each column audit's definition name, in declaration order. + pub audit_definitions: Vec, + /// The file of a column declared outside the model file, in a named schema. + pub location_path: Option, } /// One declared dynamic-column family; `key` is its name casefolded. diff --git a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs index 98262c5aab..3a3381bbd8 100644 --- a/crates/sqlbuild-cache/src/hosted_shapes/constants.rs +++ b/crates/sqlbuild-cache/src/hosted_shapes/constants.rs @@ -1,7 +1,7 @@ //! The Python shape names of the natively hosted compiler shapes. /// The Python model type a natively hosted compiler shape is captured as. -pub const HOSTED_SHAPE_TYPES: [(&str, &str); 492] = [ +pub const HOSTED_SHAPE_TYPES: [(&str, &str); 490] = [ ( "sqlbuild._native:AlignedInterval", "sqlbuild.cursor_algebra.models:AlignedInterval", @@ -1104,14 +1104,6 @@ pub const HOSTED_SHAPE_TYPES: [(&str, &str); 492] = [ "sqlbuild._native:NativeLintError", "sqlbuild.lint.exceptions:NativeLintError", ), - ( - "sqlbuild._native:NativeModelAnalyses", - "sqlbuild.compiler.analysis_session.models:NativeModelAnalyses", - ), - ( - "sqlbuild._native:NativeModelAnalysisRequest", - "sqlbuild.compiler.analysis_session.models:NativeModelAnalysisRequest", - ), ( "sqlbuild._native:NativeModelConfigInputs", "sqlbuild.compiler.compile.models:NativeModelConfigInputs", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index c15924e2b9..4b6db613c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -1,24 +1,28 @@ //! One compile's native model analysis session, with Python's catalog and metric bookkeeping. +use std::path::PathBuf; + use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use sqlbuild_analysis::assembly::analysis_session::types::Shapes; use crate::bindings::_helpers::analysis_session::profile_inputs::{ case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, }; +use crate::bindings::_helpers::analysis_session::session::{ + CacheStatsRow, NativeModelAnalysisSession, RequestRow, start_model_analysis_session, +}; use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; -use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::cache::native_store::current_code_identity; +use crate::bindings::_helpers::cache::native_store::{content_digest, current_code_identity}; use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; use crate::bindings::_helpers::compiled_project::model_input_view::{ - AttachedSchema, attached_schema, + AttachedSchema, attached_schema, input_model_name, }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::semantic_shapes::shapes::{VALID_FROM_DEFAULT, VALID_TO_DEFAULT}; @@ -29,186 +33,165 @@ const STORE_FAILURE_MESSAGE: &str = "native model analysis cache not read or sav const STORE_FILE_NAME: &str = "model-analyses.bin"; const STORE_VERSION: &str = "native-model-analysis-store-v1"; +/// What one compile's model analysis reads: the analysed models and the relation facts. +pub(crate) struct ModelAnalysisRequest<'py> { + pub(crate) model_inputs: Vec>, + pub(crate) profile: Bound<'py, PyAny>, + pub(crate) column_types: Bound<'py, PyAny>, + pub(crate) nullability: Bound<'py, PyAny>, + pub(crate) complete_schemas: Bound<'py, PyAny>, + pub(crate) dynamic_families: Bound<'py, PyAny>, + pub(crate) rich_type_inference: bool, + /// The `AnalysisCacheContext`, or `None` when the analysis store is bypassed. + pub(crate) cache: Bound<'py, PyAny>, +} + +/// Each analysed model's position in the finished session, those that succeeded, and the session. +pub(crate) struct ModelAnalyses<'py> { + pub(crate) positions: Bound<'py, PyDict>, + pub(crate) succeeded: Bound<'py, PyFrozenSet>, + /// The finished session, or `None` when no model was analysed. + pub(crate) session: Option>, +} + /// Each model's analysis by name and the finished session; a native internal failure raises. -#[pyfunction] -#[pyo3(signature = (**request))] -fn analyze_native_model_sql<'py>( - request: Option<&Bound<'py, PyDict>>, -) -> PyResult> { - let request = Request::new("analyze_native_model_sql", request)?; - let py: Python<'py> = request.py(); - let request: Bound<'py, PyAny> = request.get("request")?; - let analyses_class: Bound<'py, PyAny> = registry::class(py, RecordId::NativeModelAnalyses)?; - let model_inputs: Bound<'py, PyAny> = request.getattr("model_inputs")?; - if !model_inputs.is_truthy()? { - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("positions", PyDict::new(py))?; - arguments.set_item("succeeded", PyFrozenSet::empty(py)?)?; - arguments.set_item("session", py.None())?; - return analyses_class.call((), Some(&arguments)); +pub(crate) fn analyze_models<'py>( + request: &ModelAnalysisRequest<'py>, +) -> PyResult> { + let py: Python<'py> = request.profile.py(); + if request.model_inputs.is_empty() { + return Ok(ModelAnalyses { + positions: PyDict::new(py), + succeeded: PyFrozenSet::empty(py)?, + session: None, + }); } - let profile: Bound<'py, PyAny> = request.getattr("inference_profile")?; - let catalog: Bound<'py, PyAny> = profile.getattr("binding_catalog")?; - let python: Vec> = model_inputs.try_iter()?.collect::>()?; - let (rules, adapter_rules) = nullability_rules(&profile)?; - let session_request: Bound<'py, PyAny> = session_request(&request, &python, &catalog, rules)?; - let (cache, adapter): (Bound<'py, PyAny>, Bound<'py, PyAny>) = match adapter_rules { - Some((adapter, nullability)) => ( - py.None().into_bound(py), - PyTuple::new( - py, - [ - adapter.into_any().into_bound(py), - nullability.into_bound(py), - ], - )? - .into_any(), - ), - None => (analysis_store(&request)?, py.None().into_bound(py)), + let catalog: Bound<'py, PyAny> = request.profile.getattr("binding_catalog")?; + let (rules, adapter_rules) = nullability_rules(&request.profile)?; + let session_request: RequestRow = session_request(request, &catalog, rules)?.extract()?; + let (cache, adapter) = match adapter_rules { + Some((adapter, nullability)) => (None, Some((adapter, nullability))), + None => (analysis_store(request)?, None), }; - let session: Bound<'py, PyAny> = registry::native(py, NativeAttr::StartModelAnalysisSession)? - .call1(( - catalog.getattr("native")?, - session_request, - cache, - adapter, - ))?; - run_session(&request, (&python, &catalog), &session, &analyses_class) + let session: Bound<'py, NativeModelAnalysisSession> = Bound::new( + py, + start_model_analysis_session( + catalog.getattr("native")?.extract()?, + session_request, + cache, + adapter, + )?, + )?; + run_session(request, &catalog, &session) } /// The session request row, in the order the native session reads it. fn session_request<'py>( - request: &Bound<'py, PyAny>, - python: &[Bound<'py, PyAny>], + request: &ModelAnalysisRequest<'py>, catalog: &Bound<'py, PyAny>, rules: Vec<(String, String)>, ) -> PyResult> { - let py: Python<'py> = request.py(); - let profile: Bound<'py, PyAny> = request.getattr("inference_profile")?; + let py: Python<'py> = catalog.py(); + let profile: &Bound<'py, PyAny> = &request.profile; let mut families: Vec<(String, Vec<_>)> = Vec::new(); - for item in request - .getattr("dynamic_families_by_table")? - .call_method0("items")? - .try_iter()? - { + for item in request.dynamic_families.call_method0("items")?.try_iter()? { let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; families.push((name, family_rows(&declared)?)); } let items: Vec> = vec![ - profile_dialect(&profile)?.into_bound_py_any(py)?, - case_sensitive_shapes(&profile, &profile.getattr("sql_analysis_dialect")?)? + profile_dialect(profile)?.into_bound_py_any(py)?, + case_sensitive_shapes(profile, &profile.getattr("sql_analysis_dialect")?)? .into_bound_py_any(py)?, - return_types(&profile)?.into_bound_py_any(py)?, + return_types(profile)?.into_bound_py_any(py)?, rules.into_bound_py_any(py)?, - request.getattr("rich_type_inference")?, - shape_rows(&request.getattr("column_types_by_table")?)?.into_bound_py_any(py)?, - shape_rows(&request.getattr("column_nullability_by_table")?)?.into_bound_py_any(py)?, - shape_rows(&request.getattr("complete_binding_schemas")?)?.into_bound_py_any(py)?, + request.rich_type_inference.into_bound_py_any(py)?, + shape_rows(&request.column_types)?.into_bound_py_any(py)?, + shape_rows(&request.nullability)?.into_bound_py_any(py)?, + shape_rows(&request.complete_schemas)?.into_bound_py_any(py)?, shape_rows(&catalog.getattr("schemas")?)?.into_bound_py_any(py)?, families.into_bound_py_any(py)?, - model_rows(py, python)?, + model_rows(py, &request.model_inputs)?, ]; Ok(PyTuple::new(py, items)?.into_any()) } /// The analysis store's path and environment, or None where Python bypasses its cache. -fn analysis_store<'py>(request: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = request.py(); - let context: Bound<'py, PyAny> = request.getattr("analysis_cache")?; +fn analysis_store(request: &ModelAnalysisRequest<'_>) -> PyResult> { + let context: &Bound<'_, PyAny> = &request.cache; if context.is_none() { - return Ok(py.None().into_bound(py)); + return Ok(None); } - let path: Bound<'py, PyAny> = context - .getattr("root")? - .div(STORE_FILE_NAME)? - .str()? - .into_any(); - let identity: Bound<'py, PyAny> = PyString::new(py, ¤t_code_identity(py)?).into_any(); - let parts: Bound<'py, PyList> = PyList::new( - py, - [ - PyString::new(py, STORE_VERSION).into_any(), - identity, - context.getattr("shared_fingerprint")?, - context.getattr("signature_namespace")?, - ], - )?; - let environment: Bound<'py, PyAny> = - registry::native(py, NativeAttr::ContentDigest)?.call1((parts,))?; - Ok(PyTuple::new(py, [path, environment])?.into_any()) + let path: PathBuf = context.getattr("root")?.div(STORE_FILE_NAME)?.extract()?; + let environment: String = content_digest(vec![ + STORE_VERSION.to_owned(), + current_code_identity(context.py())?, + context.getattr("shared_fingerprint")?.extract()?, + context.getattr("signature_namespace")?.extract()?, + ]); + Ok(Some((path, environment))) } /// Run and finish the session, leaving Python's catalog and metrics as its analysis did. fn run_session<'py>( - request: &Bound<'py, PyAny>, - (python, catalog): (&[Bound<'py, PyAny>], &Bound<'py, PyAny>), - session: &Bound<'py, PyAny>, - analyses_class: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = request.py(); - let types: Bound<'py, PyDict> = copied(&request.getattr("column_types_by_table")?)?; - let nullability: Bound<'py, PyDict> = copied(&request.getattr("column_nullability_by_table")?)?; - let (publications, failures): (Bound<'py, PyAny>, Bound<'py, PyAny>) = - session.call_method0("run")?.extract()?; - for failure in failures.try_iter()? { - log_debug(py, ANALYSIS_FAILURE_MESSAGE, &failure?)?; + request: &ModelAnalysisRequest<'py>, + catalog: &Bound<'py, PyAny>, + session: &Bound<'py, NativeModelAnalysisSession>, +) -> PyResult> { + let py: Python<'py> = catalog.py(); + let python: &[Bound<'py, PyAny>] = &request.model_inputs; + let types: Bound<'py, PyDict> = copied(&request.column_types)?; + let nullability: Bound<'py, PyDict> = copied(&request.nullability)?; + let (publications, failures) = session.borrow_mut().run(py)?; + for failure in failures { + log_debug( + py, + ANALYSIS_FAILURE_MESSAGE, + PyString::new(py, &failure).as_any(), + )?; } let unknown: Bound<'py, PyAny> = registry::class(py, RecordId::InferredNullability)?.getattr("UNKNOWN")?; - for item in publications.try_iter()? { - let (name, shape): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - let published: Bound<'py, PyAny> = dict_of(&shape)?; + for (name, shape) in publications { + let published: Bound<'py, PyDict> = shape.into_py_dict(py)?; types.call_method1("setdefault", (&name, &published))?; - let defaults: Bound<'py, PyAny> = PyDict::new(py) - .into_any() - .getattr("fromkeys")? - .call1((&published, &unknown))?; + let defaults: Bound<'py, PyDict> = PyDict::new(py); + for column in published.keys() { + defaults.set_item(column, &unknown)?; + } nullability.call_method1("setdefault", (&name, defaults))?; } - let (additions, analysis_names, succeeded): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = session.call_method0("finish")?.extract()?; - record_catalog_changes(catalog, &additions, &analysis_names, (&types, &nullability))?; - record_cache(python, session)?; + let (additions, analysis_names, succeeded) = session.borrow_mut().finish(py)?; + record_catalog_changes(catalog, additions, &analysis_names, (&types, &nullability))?; + record_cache(py, python, session.borrow().cache_stats())?; let positions: Bound<'py, PyDict> = PyDict::new(py); - let mut names: Vec> = Vec::new(); - for model_input in python { - let name: Bound<'py, PyAny> = model_input - .getattr("model_file")? - .getattr("file_path")? - .getattr("stem")?; - positions.set_item(&name, names.len())?; - names.push(name); + let mut succeeded_names: Vec> = Vec::new(); + for (index, (model_input, success)) in python.iter().zip(&succeeded).enumerate() { + let name: Bound<'py, PyAny> = PyString::new(py, &input_model_name(model_input)?).into_any(); + positions.set_item(&name, index)?; + if *success { + succeeded_names.push(name); + } } - let succeeded: Vec = succeeded.extract()?; - let succeeded_names: Vec<&Bound<'py, PyAny>> = names - .iter() - .zip(&succeeded) - .filter_map(|(name, success)| success.then_some(name)) - .collect(); - let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("positions", positions)?; - arguments.set_item("succeeded", PyFrozenSet::new(py, succeeded_names)?)?; - arguments.set_item("session", session)?; - analyses_class.call((), Some(&arguments)) + Ok(ModelAnalyses { + positions, + succeeded: PyFrozenSet::new(py, succeeded_names)?, + session: Some(session.clone()), + }) } /// Leave Python's binding catalog as its own analysis would have. fn record_catalog_changes<'py>( catalog: &Bound<'py, PyAny>, - schema_additions: &Bound<'py, PyAny>, - analysis_names: &Bound<'py, PyAny>, + schema_additions: Shapes, + analysis_names: &[String], (types, nullability): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), ) -> PyResult<()> { let py: Python<'py> = catalog.py(); let schemas: Bound<'py, PyAny> = catalog.getattr("schemas")?; let additions: Bound<'py, PyDict> = PyDict::new(py); - for item in schema_additions.try_iter()? { - let (name, shape): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + for (name, shape) in schema_additions { if !schemas.contains(&name)? { - additions.set_item(name, dict_of(&shape)?)?; + additions.set_item(name, shape.into_py_dict(py)?)?; } } schemas.call_method1("update", (&additions,))?; @@ -219,13 +202,12 @@ fn record_catalog_changes<'py>( } let selected_types: Bound<'py, PyDict> = PyDict::new(py); let selected_nullability: Bound<'py, PyDict> = PyDict::new(py); - let names: Vec> = analysis_names.try_iter()?.collect::>()?; - for name in &names { + for name in analysis_names { if let Some(shape) = types.get_item(name)? { selected_types.set_item(name, shape)?; } } - for name in &names { + for name in analysis_names { if let Some(shape) = nullability.get_item(name)? { selected_nullability.set_item(name, shape)?; } @@ -239,27 +221,29 @@ fn record_catalog_changes<'py>( Ok(()) } -fn record_cache(python: &[Bound<'_, PyAny>], session: &Bound<'_, PyAny>) -> PyResult<()> { - let py: Python<'_> = session.py(); - let stats: Bound<'_, PyAny> = session.getattr("cache_stats")?; - if stats.is_none() { - return record_metrics(py, (0, 0, python.len())); - } - let (hits, misses, _, failure): (usize, usize, usize, Bound<'_, PyAny>) = stats.extract()?; - if !failure.is_none() { - log_debug(py, STORE_FAILURE_MESSAGE, &failure)?; +fn record_cache( + py: Python<'_>, + python: &[Bound<'_, PyAny>], + stats: Option, +) -> PyResult<()> { + let Some((hits, misses, _, failure)) = stats else { + return record_metrics((0, 0, python.len())); + }; + if let Some(failure) = failure { + log_debug( + py, + STORE_FAILURE_MESSAGE, + PyString::new(py, &failure).as_any(), + )?; } if hits + misses == 0 { - return record_metrics(py, (0, 0, python.len())); + return record_metrics((0, 0, python.len())); } - record_metrics(py, (hits, misses, 0)) + record_metrics((hits, misses, 0)) } /// Record the analysis cache entries a native session read, missed and bypassed. -fn record_metrics( - _py: Python<'_>, - (hits, misses, bypasses): (usize, usize, usize), -) -> PyResult<()> { +fn record_metrics((hits, misses, bypasses): (usize, usize, usize)) -> PyResult<()> { record_analysis_cache_metrics((0, hits, misses, bypasses)); Ok(()) } @@ -320,10 +304,7 @@ fn model_rows<'py>( None => Vec::new(), }; let row: Vec> = vec![ - model_input - .getattr("model_file")? - .getattr("file_path")? - .getattr("stem")?, + input_model_name(model_input)?.into_bound_py_any(py)?, query_sql, placeholders.into_bound_py_any(py)?, references.into_bound_py_any(py)?, @@ -364,17 +345,3 @@ fn copied<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { copy.update(mapping.downcast()?)?; Ok(copy) } - -/// `dict(pairs)`. -fn dict_of<'py>(pairs: &Bound<'py, PyAny>) -> PyResult> { - let dict: Bound<'py, PyDict> = PyDict::new(pairs.py()); - for item in pairs.try_iter()? { - let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; - dict.set_item(key, value)?; - } - Ok(dict.into_any()) -} - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(analyze_native_model_sql, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index 0c43aea023..aa62b2be6a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -3,15 +3,14 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use crate::bindings::_helpers::analysis_session::{ - fixture_columns, fixture_inferences, model_analyses, oracle, outcome_objects, query_columns, - session, stage_functions, + fixture_columns, fixture_inferences, oracle, outcome_objects, query_columns, session, + stage_functions, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { session::register(module)?; fixture_columns::register(module)?; module.add_class::()?; - model_analyses::register(module)?; outcome_objects::register(module)?; query_columns::register(module)?; stage_functions::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index df8973e549..836ef0433d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use pyo3::{pyclass, pyfunction, pymethods}; use sqlbuild_analysis::assembly::analysis_session::main::attach_analysis_cache::attach_analysis_cache; use sqlbuild_analysis::assembly::analysis_session::main::expression_shapes::expression_shapes; use sqlbuild_analysis::assembly::analysis_session::main::finish_analysis_session::finish_analysis_session; @@ -58,7 +58,7 @@ type ModelRow = ( Vec, ); /// The profile settings, relation shapes and models one session analyses. -type RequestRow = ( +pub(crate) type RequestRow = ( String, bool, Pairs, @@ -72,7 +72,7 @@ type RequestRow = ( Vec, ); /// `(publications, failures)`. -type StepRow = (Shapes, Vec); +pub(crate) type StepRow = (Shapes, Vec); /// One model's analysis, lineage source and cleaned SQL. type OutcomeRow = ( bool, @@ -108,9 +108,9 @@ type PivotRequestRow = ( /// `("absent" | "deferred" | "proof", proof)`. pub(crate) type ContractRow = (&'static str, Option); /// `(hits, misses, stored, why the store was not read or saved)`. -type CacheStatsRow = (usize, usize, usize, Option); +pub(crate) type CacheStatsRow = (usize, usize, usize, Option); /// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. -type FinishRow = (Shapes, Vec, Vec); +pub(crate) type FinishRow = (Shapes, Vec, Vec); /// One compile's native model analysis; an internal failure raises `NativeCompilerError`. #[pyclass(module = "sqlbuild._native")] @@ -160,19 +160,9 @@ fn internal_error(reason: &str) -> PyErr { compiler_error(native_failure(SESSION_CONTEXT, reason)) } -#[pymethods] impl NativeModelAnalysisSession { - /// Models whose output names and lineage the finished session keeps for later stages. - #[getter] - pub(crate) fn fact_models(&self) -> Vec { - self.finished - .as_ref() - .map(finished_fact_models) - .unwrap_or_default() - } - /// Analyse every model. - fn run(&mut self, py: Python<'_>) -> PyResult { + pub(crate) fn run(&mut self, py: Python<'_>) -> PyResult { let mut session: AnalysisSession = self.running()?; let result: Result<(AnalysisSession, SessionStep), String> = py.compiler_detach(move || { @@ -188,7 +178,7 @@ impl NativeModelAnalysisSession { } /// Every model's outcome; saves the cache best-effort. - fn finish(&mut self, py: Python<'_>) -> PyResult { + pub(crate) fn finish(&mut self, py: Python<'_>) -> PyResult { let mut session: AnalysisSession = self.running()?; let cache: Option<(NativeStore, AnalysisCacheStats)> = take_analysis_cache(&mut session); let (outcome, finished) = catch_compiler_panic(|| finish_analysis_session(session)) @@ -213,9 +203,21 @@ impl NativeModelAnalysisSession { self.outcomes = models.into_iter().zip(dynamic_contracts).collect(); Ok((schema_additions, analysis_names, succeeded)) } +} + +#[pymethods] +impl NativeModelAnalysisSession { + /// Models whose output names and lineage the finished session keeps for later stages. + #[getter] + pub(crate) fn fact_models(&self) -> Vec { + self.finished + .as_ref() + .map(finished_fact_models) + .unwrap_or_default() + } /// Each requested model's raw binding diagnostic rows, in request order. - fn binding_rows(&self) -> Vec> { + pub(crate) fn binding_rows(&self) -> Vec> { self.outcomes .iter() .map(|(outcome, _)| outcome.analysis.binding_diagnostics.clone()) @@ -236,7 +238,7 @@ impl NativeModelAnalysisSession { /// The analysis cache's `(hits, misses, stored, failure)`; None without a cache. #[getter] - fn cache_stats(&self) -> Option { + pub(crate) fn cache_stats(&self) -> Option { self.cache_stats.clone() } @@ -265,9 +267,7 @@ impl NativeModelAnalysisSession { } /// Start a session on `catalog`. -#[pyfunction] -#[pyo3(signature = (catalog, request, cache=None, adapter_rules=None))] -fn start_model_analysis_session( +pub(crate) fn start_model_analysis_session( catalog: PyRef<'_, ProjectCatalog>, request: RequestRow, cache: Option<(PathBuf, String)>, @@ -514,6 +514,5 @@ fn pivot_model(row: PivotModelRow) -> PivotModel { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - module.add_function(wrap_pyfunction!(start_model_analysis_session, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index df161e7c3c..f9b1c6463f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -6,6 +6,9 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple} use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignores_quoted_case}; +use crate::bindings::_helpers::analysis_session::model_analyses::{ + ModelAnalyses, ModelAnalysisRequest, analyze_models, +}; use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; use crate::bindings::_helpers::compile_driver::assembly::AssemblyRequest; use crate::bindings::_helpers::compile_driver::compile_timings::{ @@ -30,8 +33,8 @@ pub(crate) struct AnalysisContext<'py> { pub(crate) nullability: Bound<'py, PyAny>, pub(crate) complete_schemas: Bound<'py, PyAny>, pub(crate) dynamic_families: Bound<'py, PyAny>, - /// The `NativeModelAnalyses`, or `None` when no model was analysed. - pub(crate) analyses: Bound<'py, PyAny>, + /// The model analyses, or `None` when SQL analysis does not run. + pub(crate) analyses: Option>, } /// Whether `model_input` is selected for analysis by `names` (`None` selects every model). @@ -90,7 +93,7 @@ pub(crate) fn analysis_context<'py>( nullability, complete_schemas, dynamic_families, - analyses: py.None().into_bound(py), + analyses: None, }; if context.sql_analysis_enabled.is_truthy()? { record_expression_shapes(inputs, &context)?; @@ -324,10 +327,10 @@ fn record_expression_shapes<'py>( fn model_analyses<'py>( request: &AssemblyRequest, context: &AnalysisContext<'py>, -) -> PyResult> { +) -> PyResult>> { let py: Python<'py> = context.profile.py(); if !context.sql_analysis_enabled.is_truthy()? { - return Ok(py.None().into_bound(py)); + return Ok(None); } let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); @@ -346,8 +349,6 @@ fn model_analyses<'py>( }; record_compile_timing("model_analysis_ms", || { record_compile_cpu_timing(py, "model_analysis_cpu_ms", || { - let (known_functions, known_types): (Bound<'py, PyAny>, Bound<'py, PyAny>) = - declared_names(inputs, &context.column_types)?; let mut model_inputs: Vec> = Vec::new(); for model_input in inputs.getattr("model_inputs")?.try_iter()? { let model_input: Bound<'py, PyAny> = model_input?; @@ -357,29 +358,17 @@ fn model_analyses<'py>( model_inputs.push(model_input); } } - let analysis_request: Bound<'py, PyAny> = call_with( - ®istry::class(py, RecordId::NativeModelAnalysisRequest)?, - &[ - ("known_functions", known_functions), - ("known_types", known_types), - ("model_inputs", PyTuple::new(py, model_inputs)?.into_any()), - ("column_nullability_by_table", context.nullability.clone()), - ("column_types_by_table", context.column_types.clone()), - ("inference_profile", context.profile.clone()), - ("allow_compact_analysis", flags.0.clone()), - ("rich_type_inference", flags.1.clone()), - ("analysis_cache", cache.clone()), - ("complete_binding_schemas", context.complete_schemas.clone()), - ( - "dynamic_families_by_table", - context.dynamic_families.clone(), - ), - ], - )?; - call_with( - ®istry::native(py, NativeAttr::AnalyzeNativeModelSql)?, - &[("request", analysis_request)], - ) + analyze_models(&ModelAnalysisRequest { + model_inputs, + profile: context.profile.clone(), + column_types: context.column_types.clone(), + nullability: context.nullability.clone(), + complete_schemas: context.complete_schemas.clone(), + dynamic_families: context.dynamic_families.clone(), + rich_type_inference: rich, + cache, + }) + .map(Some) }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs index aef011a80c..7f00d931ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_facts.rs @@ -24,16 +24,15 @@ pub(crate) fn native_project_facts<'py>( let py: Python<'py> = inputs.py(); let model_inputs: Bound<'py, PyAny> = inputs.getattr("model_inputs")?; let dialect: Bound<'py, PyAny> = context.profile.getattr("sql_analysis_dialect")?; - let analysed: bool = !context.analyses.is_none(); - let succeeded: Bound<'py, PyAny> = if analysed { - context.analyses.getattr("succeeded")? - } else { - PyFrozenSet::empty(py)?.into_any() - }; - let positions: Bound<'py, PyAny> = if analysed { - context.analyses.getattr("positions")? - } else { - PyDict::new(py).into_any() + let (succeeded, positions): (Bound<'py, PyAny>, Bound<'py, PyAny>) = match &context.analyses { + Some(analyses) => ( + analyses.succeeded.clone().into_any(), + analyses.positions.clone().into_any(), + ), + None => ( + PyFrozenSet::empty(py)?.into_any(), + PyDict::new(py).into_any(), + ), }; let syntax_checks: Bound<'py, PyAny> = model_syntax_checks(&model_inputs.clone(), &names.clone(), &succeeded) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 7975846def..c89ed3ab49 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -1,8 +1,10 @@ //! Each compiled model of an assembled project, with its syntax checks and analysis fallback. +use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyList, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSetMethods, PyList, PyString, PyTuple}; +use crate::bindings::_helpers::analysis_session::model_analyses::ModelAnalyses; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compile_driver::assembly_analysis::{ @@ -114,23 +116,26 @@ fn compiled_model<'py>( let mut session_analysis: Bound<'py, PyAny> = py.None().into_bound(py); let mut fallback: Bound<'py, PyAny> = py.None().into_bound(py); if request.sql_analysis_enabled { - let analyses: &Bound<'py, PyAny> = &context.analyses; - let position: Bound<'py, PyAny> = if analyses.is_none() { - py.None().into_bound(py) - } else { - analyses - .getattr("positions")? - .call_method1("get", (&model_name,))? + let analysed = match &context.analyses { + Some(analyses) => analyses + .positions + .get_item(&model_name)? + .map(|position| (analyses, position)), + None => None, }; - if analyses.is_none() || position.is_none() { + let Some((analyses, position)) = analysed else { return Err(NativeCompilerError::new_err(format!( "{NATIVE_FAILURE_PREFIX}native model analysis: the session returned no \ analysis for model '{}'", model_name.str()? ))); - } - session_analysis = PyTuple::new(py, [analyses.getattr("session")?, position])?.into_any(); - if !analyses.getattr("succeeded")?.contains(&model_name)? { + }; + let session: Bound<'py, PyAny> = analyses.session.as_ref().map_or_else( + || py.None().into_bound(py), + |session| session.clone().into_any(), + ); + session_analysis = PyTuple::new(py, [session, position])?.into_any(); + if !analyses.succeeded.contains(&model_name)? { if validates { stages .validate_sql @@ -219,16 +224,15 @@ fn fallback_columns<'py>( /// `_project_binding_diagnostics`: every analysed model's located binding diagnostics. pub(crate) fn project_binding_diagnostics<'py>( models: &Bound<'py, PyTuple>, - analyses: &Bound<'py, PyAny>, + analyses: Option<&ModelAnalyses<'py>>, ) -> PyResult>> { let mut diagnostics: Vec> = Vec::new(); - if analyses.is_none() { + let Some(analyses) = analyses else { return Ok(diagnostics); - } - let positions: Bound<'py, PyAny> = analyses.getattr("positions")?; + }; for model in models.as_any().try_iter()? { let model: Bound<'py, PyAny> = model?; - if positions.contains(model.getattr("name")?)? { + if analyses.positions.contains(model.getattr("name")?)? { for diagnostic in model.getattr("binding_diagnostics")?.try_iter()? { diagnostics.push(diagnostic?); } @@ -238,24 +242,22 @@ pub(crate) fn project_binding_diagnostics<'py>( } /// `_binding_rows`: each analysed model's raw binding rows by model name. -pub(crate) fn binding_rows<'py>(analyses: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = analyses.py(); +pub(crate) fn binding_rows<'py>( + py: Python<'py>, + analyses: Option<&ModelAnalyses<'py>>, +) -> PyResult> { let rows_by_name: Bound<'py, PyDict> = PyDict::new(py); - if analyses.is_none() { + let Some(session) = analyses.and_then(|analyses| analyses.session.as_ref()) else { return Ok(rows_by_name); - } - let session: Bound<'py, PyAny> = analyses.getattr("session")?; - if session.is_none() { - return Ok(rows_by_name); - } - let rows: Bound<'py, PyAny> = session.call_method0("binding_rows")?; - for item in analyses - .getattr("positions")? - .call_method0("items")? - .try_iter()? + }; + let rows = session.borrow().binding_rows(); + for (name, position) in analyses + .map(|analyses| analyses.positions.iter()) + .into_iter() + .flatten() { - let item: Bound<'py, PyAny> = item?; - rows_by_name.set_item(item.get_item(0)?, rows.get_item(item.get_item(1)?)?)?; + let position: usize = position.extract()?; + rows_by_name.set_item(name, rows.get(position).into_pyobject(py)?)?; } Ok(rows_by_name) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 0e57c1791c..a86956db51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -76,11 +76,14 @@ pub(crate) fn assembled_project<'py>( let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); let names: &Bound<'py, PyAny> = request.analysis_model_names.bind(py); let context: AnalysisContext<'py> = analysis_context(py, request)?; - let native_session: Bound<'py, PyAny> = if context.analyses.is_none() { - py.None().into_bound(py) - } else { - context.analyses.getattr("session")? - }; + let native_session: Bound<'py, PyAny> = context + .analyses + .as_ref() + .and_then(|analyses| analyses.session.as_ref()) + .map_or_else( + || py.None().into_bound(py), + |session| session.clone().into_any(), + ); let native: Bound<'py, PyAny> = native_project_facts(inputs, &context, names, &native_session)?; let resources: Bound<'py, PyAny> = native.getattr("resources")?; let scope_index: Bound<'py, PyAny> = scope_index_with_compile_usages(inputs)?; @@ -95,7 +98,7 @@ pub(crate) fn assembled_project<'py>( &target_values, )?; let project: Bound<'py, PyAny> = project_view(inputs, &fields, &models)?; - let binding_results: Bound<'py, PyDict> = binding_rows(&context.analyses)?; + let binding_results: Bound<'py, PyDict> = binding_rows(py, context.analyses.as_ref())?; call_with( ®istry::native(py, NativeAttr::CompleteSemanticProject)?, &[ @@ -265,7 +268,10 @@ fn project_fields<'py>( .getattr("diagnostics")? .try_iter()? .collect::>()?; - diagnostics.extend(project_binding_diagnostics(models, &context.analyses)?); + diagnostics.extend(project_binding_diagnostics( + models, + context.analyses.as_ref(), + )?); fields.set_item("diagnostics", PyTuple::new(py, diagnostics)?)?; fields.set_item( "external_sql_reference_resolver", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs index 4db5465b77..1a953d6662 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -6,6 +6,7 @@ use pyo3::FromPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; use sqlbuild_analysis::compiled_project::main::merge_call_site_references::merge_call_site_references; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use sqlbuild_attachments::audits::main::attached_model_reference_error::attached_model_reference_error; use sqlbuild_attachments::audits::main::audit_reference_error::audit_reference_error; use sqlbuild_attachments::audits::main::render_attached_audit::render_attached_audit; @@ -668,20 +669,26 @@ fn header_text(header: &Bound<'_, PyDict>, key: &str) -> PyResult pub(crate) fn model_owner<'py>( name: String, path: String, - schema: (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>), + schema: ( + &Bound<'py, PyTuple>, + (&Bound<'py, PyTuple>, &[SchemaColumnFact]), + ), ) -> PyResult> { - let (audits, columns) = schema; + let (audits, (columns, facts)) = schema; let mut instances: Vec<(String, Option, Bound<'py, PyAny>)> = Vec::new(); for audit in audits.iter() { instances.push((path.clone(), explicit_column(&audit)?, audit)); } - for column in columns.iter() { - let column_owner: String = location_path(&column)?.unwrap_or_else(|| path.clone()); - let column_name: String = column.getattr("name")?.extract()?; - for audit in column.getattr("audits")?.try_iter()? { + for (index, fact) in facts.iter().enumerate() { + if fact.audit_definitions.is_empty() { + continue; + } + let column_owner: &str = fact.location_path.as_deref().unwrap_or(&path); + for audit in columns.get_item(index)?.getattr("audits")?.try_iter()? { let audit: Bound<'py, PyAny> = audit?; - let owner_file: String = location_path(&audit)?.unwrap_or_else(|| column_owner.clone()); - instances.push((owner_file, Some(column_name.clone()), audit)); + let owner_file: String = + location_path(&audit)?.unwrap_or_else(|| column_owner.to_owned()); + instances.push((owner_file, Some(fact.name.clone()), audit)); } } Ok(AttachmentOwner { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs index 0429bd3606..0abb35a97b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs @@ -4,7 +4,7 @@ use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; -use pyo3::{FromPyObject, pyclass, pymethods}; +use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; use crate::bindings::_helpers::compiled_project::compiled_model_view::object_key; use crate::bindings::_helpers::compiled_project::function_views::ObjectKeyRow; @@ -410,3 +410,35 @@ pub(crate) fn audit_reference_rows( Ok(Some(record.attached.references.clone())) }) } + +impl NativeCompiledAudit { + /// The `CompiledAudit` field values, in field order, without building the record. + pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.scope_deps(py)?.into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.definition_name(py)?.into_bound_py_any(py)?, + self.audit_file(py)?.into_bound(py), + self.audit_block(py)?.into_bound(py), + self.sql_body(py)?.into_bound_py_any(py)?, + self.evaluation_mode(py)?.into_bound(py), + self.measurement_contract(py)?.into_bound(py), + self.thresholds(py)?.into_bound(py), + self.minimum_samples(py)?.into_bound(py), + self.measure_sql(py)?.into_bound_py_any(py)?, + self.evidence_sql(py)?.into_bound_py_any(py)?, + self.evidence_limit(py)?.into_bound(py), + self.references(py)?.into_bound(py).into_any(), + self.attached_target_kind(py)?, + self.attached_target_name(py)?.into_bound_py_any(py)?, + self.attached_column_name(py)?.into_bound_py_any(py)?, + self.severity(py)?, + self.run_scope(py)?.into_bound_py_any(py)?, + pyo3::types::PyBool::new(py, self.always_run(py)?) + .to_owned() + .into_any(), + self.description(py)?.into_bound_py_any(py)?, + ]) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index a6c7a57f8a..c4e94a2f27 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -6,14 +6,17 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTuple use pyo3::{FromPyObject, pyclass, pymethods}; use sqlbuild_analysis::assembly::analysis_session::models::LineageRow; -use sqlbuild_analysis::compiled_project::models::{DynamicProofFact, ModelAnalysisFact}; +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts; +use sqlbuild_analysis::compiled_project::models::{ + CompiledModelFacts, DynamicProofFact, ModelAnalysisFact, +}; use crate::bindings::_helpers::compiled_project::compiled_model_analysis::{ AnalysisRecord, ProofArgument, SessionArgument, analysis_record, }; use crate::bindings::_helpers::compiled_project::model_input_view::{ - InputViewClasses, ModelInputRecord, model_relation_key_row, read_input, references, - rejected_opt_out, schema_entry, view_classes, + InputViewClasses, ModelInputRecord, model_relation_key_row, read_input, read_input_with_owner, + references, rejected_opt_out, schema_entry, view_classes, }; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; @@ -30,6 +33,9 @@ const MODEL_RESOURCE: &str = "model"; const TAGS_KEY: &str = "tags"; const NULLABLE: &str = "nullable"; +/// A schema's entry audits, column specs and each audited column's index and name. +pub(crate) type AuditedSchema = (Py, Py, Vec<(usize, String)>); + /// `(name, type, nullability)` of one inferred output column. pub(crate) type InferredRow = (String, Option, String); @@ -72,8 +78,6 @@ pub(crate) struct DestinationFact { /// What assembly and the semantic checks finished for one compiled model. pub(crate) struct CompiledRecord { - /// The model's name, its file stem. - name: String, /// `(resource type, name)` of each dependency. deps: Vec<(String, String)>, destination: DestinationFact, @@ -158,7 +162,6 @@ pub(crate) fn compiled_record( let config: &Bound<'_, PyAny> = input.config.bind(py); let values: Bound<'_, PyDict> = config.getattr(VALUES_ATTRIBUTE)?.downcast_into()?; Ok(CompiledRecord { - name: name.to_owned(), deps, destination: DestinationFact { database: text_value(&values, DATABASE_KEY)?, @@ -254,7 +257,7 @@ impl NativeCompiledModel { /// This view's typed analysis as the owner's model facts read it, and the model's name. pub(crate) fn analysis_fact(&self, py: Python<'_>) -> PyResult<(String, ModelAnalysisFact)> { - self.read(py, |record, py| { + self.read(py, |record, _| { let analysis: &AnalysisRecord = &compiled_of(record)?.analysis; let rows: &[InferredRow] = analysis.inferred.as_deref().unwrap_or_default(); let mut inferred_columns: Vec<(String, Option)> = @@ -282,7 +285,7 @@ impl NativeCompiledModel { fast_lineage_has_star: analysis.has_star, unchecked_output_columns: self.overlay.poisoned.clone(), }; - Ok((model_name(record, py)?, fact)) + Ok((model_name(record), fact)) }) } @@ -331,6 +334,51 @@ impl NativeCompiledModel { }) } + /// The attached schema's declared `SchemaColumn`s, or `None` without a schema. + pub(crate) fn declared_columns(&self, py: Python<'_>) -> PyResult>> { + self.read(py, |record, py| { + Ok(record + .schema + .as_ref() + .map(|schema| schema.columns.clone_ref(py))) + }) + } + + /// The attached schema's entry audits, its column specs and each audited column's index and name. + pub(crate) fn schema_audits_and_columns( + &self, + py: Python<'_>, + ) -> PyResult> { + read_input_with_owner(&self.owner, self.index, py, |owner, record| { + let (Some(schema), Some(fact)) = (&record.schema, owner.input_schema_fact(record)) + else { + return Ok(None); + }; + let audited: Vec<(usize, String)> = fact + .columns + .iter() + .enumerate() + .filter(|(_, column)| !column.audit_definitions.is_empty()) + .map(|(index, column)| (index, column.name.clone())) + .collect(); + Ok(Some(( + schema.audits.clone_ref(py), + schema.columns.clone_ref(py), + audited, + ))) + }) + } + + /// The model's authored file text and expanded query, from its recorded facts. + pub(crate) fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { + let name: String = self.name(py)?; + let owner = self.owner.bind(py).borrow(); + let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") + })?; + Ok((facts.authored_sql.clone(), facts.query_sql.clone())) + } + fn model_file_attribute(&self, py: Python<'_>, name: &str) -> PyResult> { self.read(py, |record, py| { Ok(record.model_file.bind(py).getattr(name)?.unbind()) @@ -374,7 +422,7 @@ impl NativeCompiledModel { .key .get_or_try_init(py, || { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; - object_key(&classes.compiled, MODEL_RESOURCE, &model_name(record, py)?) + object_key(&classes.compiled, MODEL_RESOURCE, &model_name(record)) }) .map(|key| key.clone_ref(py)) }) @@ -400,11 +448,11 @@ impl NativeCompiledModel { #[getter] pub(crate) fn name(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, py| model_name(record, py)) + self.read(py, |record, _| Ok(model_name(record))) } #[getter] - fn relative_path(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { self.model_file_attribute(py, "relative_path") } @@ -414,7 +462,7 @@ impl NativeCompiledModel { } #[getter] - fn config(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn config(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, py| Ok(record.config.clone_ref(py))) } @@ -450,21 +498,23 @@ impl NativeCompiledModel { /// The model's declared key facts, for deriving its unique keys. #[getter] - fn relation_key_row(&self, py: Python<'_>) -> PyResult { - self.read(py, model_relation_key_row) + pub(crate) fn relation_key_row(&self, py: Python<'_>) -> PyResult { + read_input_with_owner(&self.owner, self.index, py, |owner, record| { + model_relation_key_row(owner, record, py) + }) } /// `(resource type, name)` of the model's key. #[getter] - fn key_row(&self, py: Python<'_>) -> PyResult<(String, String)> { - self.read(py, |record, py| { - Ok((MODEL_RESOURCE.to_owned(), model_name(record, py)?)) + pub(crate) fn key_row(&self, py: Python<'_>) -> PyResult<(String, String)> { + self.read(py, |record, _| { + Ok((MODEL_RESOURCE.to_owned(), model_name(record))) }) } /// `(resource type, name)` of each dependency. #[getter] - fn dep_rows(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn dep_rows(&self, py: Python<'_>) -> PyResult> { self.compiled(py, |compiled, _| Ok(compiled.deps.clone())) } @@ -505,19 +555,21 @@ impl NativeCompiledModel { /// Each declared column of the attached schema as `(name, type)`, or `None` without one. #[getter] pub(crate) fn schema_column_rows(&self, py: Python<'_>) -> PyResult>> { - self.read(py, |record, py| { - let Some(schema) = &record.schema else { - return Ok(None); - }; - let mut rows: Vec = Vec::new(); - for column in schema.columns.bind(py).iter() { - rows.push(( - column.getattr("name")?.extract()?, - column.getattr("type")?.extract()?, - )); - } - Ok(Some(rows)) - }) + let name: String = self.name(py)?; + let owner = self.owner.bind(py).borrow(); + let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { + pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") + })?; + let Some(schema) = &facts.schema else { + return Ok(None); + }; + Ok(Some( + schema + .columns + .iter() + .map(|column| (column.name.clone(), column.data_type.clone())) + .collect(), + )) } /// Whether the attached schema declares dynamic columns. @@ -601,7 +653,7 @@ impl NativeCompiledModel { } #[getter] - fn authored_sql(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn authored_sql(&self, py: Python<'_>) -> PyResult> { self.model_file_attribute(py, "contents") } @@ -626,12 +678,12 @@ impl NativeCompiledModel { } #[getter] - fn enum_declarations(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn enum_declarations(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, py| Ok(record.enum_declarations.clone_ref(py))) } #[getter] - fn constant_declarations(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn constant_declarations(&self, py: Python<'_>) -> PyResult> { self.read(py, |record, py| { Ok(record.constant_declarations.clone_ref(py)) }) @@ -649,7 +701,7 @@ impl NativeCompiledModel { } let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; self.read(py, |record, py| { - let name: String = model_name(record, py)?; + let name: String = model_name(record); let path: Bound<'_, PyAny> = record.model_file.bind(py).getattr("relative_path")?; compiled_of(record)? .analysis @@ -744,7 +796,7 @@ pub(crate) fn model_graph_row( .str()? .extract()?; Ok(( - (MODEL_RESOURCE.to_owned(), model_name(record, py)?), + (MODEL_RESOURCE.to_owned(), model_name(record)), compiled.deps.clone(), tags, Some(folder), @@ -752,16 +804,8 @@ pub(crate) fn model_graph_row( }) } -fn model_name(record: &ModelInputRecord, py: Python<'_>) -> PyResult { - if let Some(compiled) = &record.compiled { - return Ok(compiled.name.clone()); - } - record - .model_file - .bind(py) - .getattr("file_path")? - .getattr("stem")? - .extract() +fn model_name(record: &ModelInputRecord) -> String { + record.name.clone() } /// The `CompiledRelationLocation` of a destination. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs index 2c75f75807..d4544b9531 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs @@ -4,7 +4,7 @@ use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, pyclass, pymethods}; +use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; use crate::bindings::_helpers::compiled_project::compiled_model_view::{ DestinationFact, defaulted_destination, object_key, relation_location, @@ -562,3 +562,29 @@ function_getters!(NativeCompiledFunction, { )) } }); + +impl NativeCompiledFunction { + /// The `CompiledFunction` field values, in field order, without building the record. + pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py)?.into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.relative_path(py)?.into_bound(py), + self.arguments(py)?.into_bound(py).into_any(), + self.returns(py)?.into_bound_py_any(py)?, + self.body_sql(py)?.into_bound_py_any(py)?, + self.destination(py)?.into_bound(py), + self.fingerprint_destination(py)?.into_bound(py), + self.return_columns(py)?.into_bound(py).into_any(), + self.references(py)?.into_bound(py).into_any(), + self.language(py)?, + self.source_file_path(py)?.into_bound(py), + self.runtime_version(py)?.into_bound_py_any(py)?, + self.entry_point(py)?.into_bound_py_any(py)?, + self.packages(py)?.into_any(), + self.tags(py)?.into_any(), + self.description(py)?.into_bound_py_any(py)?, + ]) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 802f7dc128..63a4cce12f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -30,6 +30,7 @@ pub(crate) mod records; pub(crate) mod registration; pub(crate) mod resource_diagnostics; pub(crate) mod scenario_loop; +pub(crate) mod schema_facts; pub(crate) mod seed_views; pub(crate) mod source_loop; pub(crate) mod source_views; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index 3d70d29b34..ef2cfcc0f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -4,12 +4,14 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{FromPyObject, IntoPyObject, pyclass, pymethods}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::compiled_model_view::{ CompiledArguments, CompiledRecord, CompiledViewClasses, NativeCompiledModel, compiled_record, }; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::render::declaration_expansion::SpanRow; use crate::bindings::_helpers::rules::relation_keys::{RelationKeyRow, relation_key_row}; @@ -52,6 +54,8 @@ pub(crate) struct ExpansionRecord { /// What the model loop finished for one model beyond its facts; handles carry user values. pub(crate) struct ModelInputRecord { + /// The model's name, its file stem. + pub(crate) name: String, /// The discovered model file, owned by the discovery inputs. pub(crate) model_file: Py, /// The resolved config, whose values hold user values. @@ -116,6 +120,26 @@ impl NativeModelInput { ) -> PyResult { read_input(&self.owner, self.index, py, read) } + + /// Read the input record together with the project that owns it. + pub(crate) fn read_with_owner<'py, T>( + &self, + py: Python<'py>, + read: impl FnOnce(&NativeCompiledProject, &ModelInputRecord) -> PyResult, + ) -> PyResult { + read_input_with_owner(&self.owner, self.index, py, read) + } +} + +/// Read the input record at `index` of `owner` together with `owner`. +pub(crate) fn read_input_with_owner( + owner: &Py, + index: usize, + py: Python<'_>, + read: impl FnOnce(&NativeCompiledProject, &ModelInputRecord) -> PyResult, +) -> PyResult { + let owner = owner.bind(py).borrow(); + read(&owner, owner.input(index)?) } /// Read the input record at `index` of `owner`. @@ -180,13 +204,15 @@ impl NativeModelInput { /// The model's declared key facts, for deriving its unique keys. #[getter] fn relation_key_row(&self, py: Python<'_>) -> PyResult { - self.read(py, model_relation_key_row) + self.read_with_owner(py, |owner, record| { + model_relation_key_row(owner, record, py) + }) } /// The name of the schema entry attached to the model, or `None`. #[getter] fn attached_schema_name(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| attached_schema_name(record, py)) + self.read(py, |record, _| Ok(attached_schema_name(record))) } #[getter] @@ -244,12 +270,7 @@ impl NativeModelInput { ) -> PyResult { let mut owner = self.owner.bind(py).borrow_mut(); let record: &mut ModelInputRecord = owner.input_mut(self.index)?; - let name: String = record - .model_file - .bind(py) - .getattr("file_path")? - .getattr("stem")? - .extract()?; + let name: String = record.name.clone(); record.compiled = Some(compiled_record(py, record, &name, arguments)?); Ok(NativeCompiledModel::new( self.owner.clone_ref(py), @@ -353,44 +374,28 @@ pub(crate) fn reference_tuple( /// The model's declared key facts: its config and attached schema columns and audits. pub(crate) fn model_relation_key_row( + owner: &NativeCompiledProject, record: &ModelInputRecord, py: Python<'_>, ) -> PyResult { let values: Bound<'_, PyAny> = record.config.bind(py).getattr("values")?; + let declared = match (&record.schema, owner.input_schema_fact(record)) { + (Some(schema), Some(fact)) => { + Some((fact.columns.as_slice(), schema.audits.bind(py).as_any())) + } + _ => None, + }; relation_key_row( MODEL_RELATION_KIND, - record - .model_file - .bind(py) - .getattr("file_path")? - .getattr("stem")? - .extract()?, + record.name.clone(), values.downcast::()?, - record.schema.as_ref().map(|schema| { - ( - schema.columns.bind(py).as_any(), - schema.audits.bind(py).as_any(), - ) - }), + declared, ) } /// The name of the model's attached schema entry, its file stem, or `None`. -pub(crate) fn attached_schema_name( - record: &ModelInputRecord, - py: Python<'_>, -) -> PyResult> { - if record.schema.is_none() { - return Ok(None); - } - Ok(Some( - record - .model_file - .bind(py) - .getattr("file_path")? - .getattr("stem")? - .extract()?, - )) +pub(crate) fn attached_schema_name(record: &ModelInputRecord) -> Option { + record.schema.as_ref().map(|_| record.name.clone()) } /// The model's attached `SchemaModelEntry`, or `None`, built once. @@ -407,14 +412,7 @@ pub(crate) fn schema_entry( return Ok::<_, pyo3::PyErr>(py.None()); }; let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item( - "name", - record - .model_file - .bind(py) - .getattr("file_path")? - .getattr("stem")?, - )?; + fields.set_item("name", &record.name)?; fields.set_item("model_schema", &schema.model_schema)?; fields.set_item("description", &schema.description)?; fields.set_item("type_enforcement", schema.type_enforcement.then_some(true))?; @@ -448,10 +446,9 @@ fn expansion_object( Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) } -/// A model input's attached schema: its name, columns and dynamic column families. +/// A model input's attached schema: its name and dynamic column families. pub(crate) struct AttachedSchema<'py> { - pub(crate) name: Bound<'py, PyAny>, - pub(crate) columns: Bound<'py, PyAny>, + pub(crate) name: String, pub(crate) dynamic_columns: Bound<'py, PyAny>, pub(crate) type_enforcement: bool, } @@ -467,12 +464,7 @@ pub(crate) fn attached_schema<'py>( return Ok(None); }; Ok(Some(AttachedSchema { - name: record - .model_file - .bind(py) - .getattr("file_path")? - .getattr("stem")?, - columns: schema.columns.bind(py).clone().into_any(), + name: record.name.clone(), dynamic_columns: schema.dynamic_columns.bind(py).clone().into_any(), type_enforcement: schema.type_enforcement, })) @@ -483,13 +475,44 @@ pub(crate) fn attached_schema<'py>( return Ok(None); } Ok(Some(AttachedSchema { - name: entry.getattr("name")?, - columns: entry.getattr("columns")?, + name: entry.getattr("name")?.extract()?, dynamic_columns: entry.getattr("dynamic_columns")?, type_enforcement: entry.getattr("type_enforcement")?.is_truthy()?, })) } +/// The declared column facts of `model_input`'s attached schema, empty without one. +pub(crate) fn attached_columns(model_input: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model_input.py(); + if let Ok(native) = model_input.cast::() { + return native.get().read_with_owner(py, |owner, record| { + Ok(owner + .input_schema_fact(record) + .map(|fact| fact.columns.clone()) + .unwrap_or_default()) + }); + } + let entry: Bound<'_, PyAny> = model_input.getattr("schema_entry")?; + if entry.is_none() { + return Ok(Vec::new()); + } + python_column_facts(&entry.getattr("columns")?, None) +} + +/// The name of `model_input`, its file stem; a native input answers from its record. +pub(crate) fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult { + if let Ok(native) = model_input.cast::() { + return native + .get() + .read(model_input.py(), |record, _| Ok(record.name.clone())); + } + model_input + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem")? + .extract() +} + /// Each reference of `model_input` as `(kind, name, package, call argument count)`. pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = model_input.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs index 5ba537a17c..14a928a6a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs @@ -2,13 +2,12 @@ use pyo3::BoundObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use sqlbuild_model_config::header_metadata::models::DynamicFamily; use crate::bindings::_helpers::compiled_project::model_loop::LoopContext; -use crate::bindings::_helpers::compiled_project::project::{ - DynamicFamilyPy, SchemaColumnPy, SchemaPy, -}; +use crate::bindings::_helpers::compiled_project::model_loop_schema::DeclaredColumns; const CASES_ATTRIBUTE: &str = "cases"; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; @@ -18,7 +17,6 @@ const DEFINITION_NAME_FIELD: &str = "definition_name"; const ARGUMENTS_FIELD: &str = "arguments"; const NAME_FIELD: &str = "name"; const TYPE_FIELD: &str = "type"; -const NULLABLE_FIELD: &str = "nullable"; const AUDITS_FIELD: &str = "audits"; const MEMBERS_FIELD: &str = "members"; const VALUE_FIELD: &str = "value"; @@ -32,7 +30,6 @@ const KIND_FIELD: &str = "kind"; const PATH_FIELD: &str = "path"; const COLUMN_FIELD: &str = "column"; const LOCATION_FIELD: &str = "location"; -const CASEFOLD_METHOD: &str = "casefold"; const ACCEPTED_VALUES_AUDIT: &str = "accepted_values"; const CONTRACT_KEY: &str = "contract"; /// `AuditCase` fields copied onto each generated `SchemaAuditInstance` under the same name. @@ -63,6 +60,7 @@ const NAME_PATTERN_FIELD: &str = "name_pattern"; /// A model's enum-typed columns resolved to scalar types, and the usages they generate. pub(crate) struct EnumColumns<'py> { pub(crate) columns: Bound<'py, PyTuple>, + pub(crate) facts: Vec, /// Column name to its enum declaration, as `CompileModelInput.enum_columns` holds them. pub(crate) declarations: Bound<'py, PyDict>, pub(crate) usages: Vec>, @@ -165,28 +163,25 @@ pub(crate) fn enum_columns<'py>( (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ), values: &Bound<'py, PyDict>, - columns: &Bound<'py, PyTuple>, + columns: DeclaredColumns<'py>, ) -> PyResult> { - let py: Python<'py> = columns.py(); + let py: Python<'py> = values.py(); let (enums, (declarations, consumer)) = visible; let enforced: bool = match values.get_item(CONTRACT_KEY)? { Some(value) => value.eq(&context.classes.contract_enforced)?, None => false, }; let resolved: Bound<'py, PyDict> = PyDict::new(py); - let mut resolved_columns: Vec> = Vec::with_capacity(columns.len()); - for column in columns.iter() { - let column_type: Bound<'py, PyAny> = column.getattr(TYPE_FIELD)?; - let key: Bound<'py, PyAny> = if column_type.is_truthy()? { - column_type - } else { - "".into_pyobject_any(py)? - }; - let Some(declaration) = enums.get_item(&key)? else { + let mut resolved_columns: Vec> = Vec::with_capacity(columns.specs.len()); + let mut facts: Vec = Vec::with_capacity(columns.facts.len()); + for (column, mut fact) in columns.specs.into_iter().zip(columns.facts) { + let key: &str = fact.data_type.as_deref().unwrap_or_default(); + let Some(declaration) = enums.get_item(key)? else { resolved_columns.push(column); + facts.push(fact); continue; }; - resolved.set_item(column.getattr(NAME_FIELD)?, &declaration)?; + resolved.set_item(&fact.name, &declaration)?; let mut audits: Vec> = column .getattr(AUDITS_FIELD)? .try_iter()? @@ -202,68 +197,31 @@ pub(crate) fn enum_columns<'py>( } if !present { audits.push(generated); + fact.audit_definitions + .push(ACCEPTED_VALUES_AUDIT.to_owned()); } } + let scalar_type: Bound<'py, PyAny> = declaration.getattr(SCALAR_TYPE_ATTRIBUTE)?; + fact.data_type = scalar_type.extract()?; let fields: Bound<'py, PyDict> = PyDict::new(py); for field in KEPT_COLUMN_FIELDS { fields.set_item(field, column.getattr(field)?)?; } - fields.set_item(TYPE_FIELD, declaration.getattr(SCALAR_TYPE_ATTRIBUTE)?)?; + fields.set_item(TYPE_FIELD, scalar_type)?; fields.set_item(AUDITS_FIELD, PyTuple::new(py, audits)?)?; resolved_columns.push(context.contracts.schema_column.call((), Some(&fields))?); + facts.push(fact); } let usages: Vec> = generated_usages(context, declarations, consumer, &resolved)?; Ok(EnumColumns { columns: PyTuple::new(py, resolved_columns)?, + facts, declarations: resolved, usages, }) } -/// The schema row the retained facts are read from, as `_schema_row` builds it. -pub(crate) fn schema_row( - columns: &Bound<'_, PyTuple>, - (audit_count, model_path): (usize, &Bound<'_, PyAny>), - families: &[Bound<'_, PyAny>], - switches: (bool, bool), -) -> PyResult { - let mut column_rows: Vec = Vec::with_capacity(columns.len()); - for column in columns.iter() { - let location: Bound<'_, PyAny> = column.getattr(LOCATION_FIELD)?; - let declared_outside: bool = !location.is_none() && { - let path: Bound<'_, PyAny> = location.getattr(PATH_FIELD)?; - !path.is(model_path) && path.ne(model_path)? - }; - column_rows.push(( - column.getattr(NAME_FIELD)?.extract()?, - column.getattr(TYPE_FIELD)?.extract()?, - column.getattr(NULLABLE_FIELD)?.extract()?, - u32::try_from(column.getattr(AUDITS_FIELD)?.len()?).unwrap_or(u32::MAX), - declared_outside, - )); - } - let mut family_rows: Vec = Vec::with_capacity(families.len()); - for family in families { - let name: Bound<'_, PyAny> = family.getattr(NAME_FIELD)?; - family_rows.push(( - name.call_method0(CASEFOLD_METHOD)?.extract()?, - name.extract()?, - family.getattr(PIVOT_COLUMN_FIELD)?.extract()?, - family.getattr(VALUE_COLUMN_FIELD)?.extract()?, - family.getattr(AGGREGATE_FIELD)?.extract()?, - family.getattr(TYPE_FIELD)?.extract()?, - family.getattr(NAME_PATTERN_FIELD)?.extract()?, - )); - } - Ok(( - u32::try_from(audit_count).unwrap_or(u32::MAX), - column_rows, - family_rows, - switches, - )) -} - fn accepted_values<'py>( context: &LoopContext<'_, 'py>, declaration: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index 3b32a577ca..57fad6509a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -2,25 +2,27 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods}; -use sqlbuild_analysis::compiled_project::models::ModelSchemaFact; +use sqlbuild_analysis::compiled_project::models::{ + DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, +}; use sqlbuild_model_config::header_metadata::main::audit_factory_references::audit_factory_references; use sqlbuild_model_config::header_metadata::main::merged_audit_identities::merged_audit_identities; use sqlbuild_model_config::header_metadata::main::parse_dynamic_columns::parse_dynamic_columns; use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; -use sqlbuild_model_config::header_metadata::models::DynamicFamily; +use sqlbuild_model_config::header_metadata::models::{DynamicFamily, ParsedColumn}; use sqlbuild_model_config::model_validation::models::ColumnMigration; use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, compile_error, config_error, }; use crate::bindings::_helpers::compiled_project::model_loop_contracts::{ - EnumColumns, audit_identity, dynamic_family_objects, enum_columns, factory_audits, schema_row, + EnumColumns, audit_identity, dynamic_family_objects, enum_columns, factory_audits, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; use crate::bindings::_helpers::compiled_project::model_loop_schema::{ - ModelSchema, merged_columns, schema_enums, + DeclaredColumns, ModelSchema, merged_columns, schema_enums, }; -use crate::bindings::_helpers::compiled_project::project::schema_fact; +use crate::bindings::_helpers::compiled_project::schema_facts::family_fact; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; @@ -33,9 +35,6 @@ const AUDITS_KEY: &str = "audits"; const DYNAMIC_COLUMNS_KEY: &str = "dynamic_columns"; const AUDIT_FACTORIES_KEY: &str = "audit_factories"; const STRIP_METHOD: &str = "strip"; -const NAME_FIELD: &str = "name"; -const TYPE_FIELD: &str = "type"; -const MIGRATE_FROM_FIELD: &str = "migrate_from"; const ENUMS_FIELD: &str = "enums"; /// What a model's header schema entry is attached from. @@ -100,14 +99,21 @@ pub(crate) fn header_schema<'py>( let parsed_columns = metadata .columns .map_err(|error| raised_config_error(py, error))?; - let local_columns: Bound<'py, PyTuple> = column_tuple( - py, - context.contracts, - &parsed_columns, - &source.model_file.header_locations(py)?, - )?; - let columns: Bound<'py, PyTuple> = match source.schema { - Some(schema) => merged_columns(context, schema, &local_columns, model)?, + let locations: Bound<'py, PyDict> = source.model_file.header_locations(py)?; + let model_path: &Bound<'py, PyAny> = source.model_file.paths(py).1; + let local_columns: DeclaredColumns<'py> = DeclaredColumns { + specs: column_tuple(py, context.contracts, &parsed_columns, &locations)? + .iter() + .collect(), + facts: local_column_facts(&parsed_columns)?, + }; + let columns: DeclaredColumns<'py> = match source.schema { + Some(schema) => merged_columns( + context, + schema, + local_columns, + (source.name, source.relative_path, model_path), + )?, None => local_columns, }; let families = dynamic_families(py, &node(dynamic), model)?; @@ -126,17 +132,23 @@ pub(crate) fn header_schema<'py>( merged_audit_identities(&direct_identities, &generated_identities, model) .map_err(|error| raised_config_error(py, error))?; let migrations: Vec = columns + .facts .iter() - .map(|column| column_migration(&column)) - .collect::>()?; + .map(|column| ColumnMigration { + name: column.name.clone(), + migrate_from: column.migrate_from.clone(), + }) + .collect(); if let Some(error) = column_migration_error(source.name, source.values, &migrations)? { let error = native_config_error(py, error)?.into_bound(py).into_any(); return Err(config_error(&error, Some(source.values))); } - let mut type_enforcement: bool = !families.is_empty(); - for column in columns.iter() { - type_enforcement |= !column.getattr(TYPE_FIELD)?.is_none(); - } + let type_enforcement: bool = !families.is_empty() + || columns + .facts + .iter() + .any(|column| column.data_type.is_some()); + let family_facts: Vec = families.iter().map(family_fact).collect(); let audits: Bound<'py, PyTuple> = PyTuple::new( py, direct_audits.iter().chain(generated).collect::>(), @@ -152,13 +164,14 @@ pub(crate) fn header_schema<'py>( None => enums, }; let resolved: EnumColumns<'py> = - enum_columns(context, (&enums, source.visible), source.values, &columns)?; - let fact: ModelSchemaFact = schema_fact(schema_row( - &resolved.columns, - (audits.len(), source.model_file.paths(py).1), - &families, - (type_enforcement, source.schema.is_some()), - )?); + enum_columns(context, (&enums, source.visible), source.values, columns)?; + let fact: ModelSchemaFact = ModelSchemaFact { + audit_count: u32::try_from(audits.len()).unwrap_or(u32::MAX), + columns: resolved.facts, + dynamic_columns: family_facts, + type_enforcement, + named_schema: source.schema.is_some(), + }; Ok(AttachedHeader { description, schema: Some(HeaderSchema { @@ -213,9 +226,30 @@ fn raised_config_error( } } -fn column_migration(column: &Bound<'_, PyAny>) -> PyResult { - Ok(ColumnMigration { - name: column.getattr(NAME_FIELD)?.str()?.to_string(), - migrate_from: column.getattr(MIGRATE_FROM_FIELD)?.extract()?, - }) +/// The facts of the header's own columns, which are declared in the model file. +fn local_column_facts(parsed: &[ParsedColumn>]) -> PyResult> { + let mut facts: Vec = Vec::with_capacity(parsed.len()); + for column in parsed { + let text = |node: Option<&PyNode<'_>>| -> PyResult> { + node.map(|node| node.0.extract()).transpose() + }; + facts.push(SchemaColumnFact { + name: column.name.0.str()?.extract()?, + data_type: text(column.column_type.as_ref())?, + nullable: column + .nullable + .as_ref() + .map(|node| node.0.extract()) + .transpose()?, + described: column.description.is_some(), + migrate_from: text(column.migrate_from.as_ref())?, + audit_definitions: column + .audits + .iter() + .map(|audit| audit.definition_name.0.extract()) + .collect::>()?, + location_path: None, + }); + } + Ok(facts) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 55a54a0bcb..cb761c9cee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -234,6 +234,7 @@ pub(crate) fn finished_record<'py>( &[&authored, ¯o_source, &expanded], )?; let record: ModelInputRecord = ModelInputRecord { + name: name.clone(), model_file: model_file.clone_ref(py).into_any(), config: config.unbind(), query_sql: query, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs index 39cdd7bfef..7c6d83eb43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs @@ -1,10 +1,12 @@ //! A model's `model_schema`: resolved, checked for visibility, and merged into its header columns. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compile_error}; use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; +use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fact; const MODEL_SCHEMA_KEY: &str = "model_schema"; const NAME_FIELD: &str = "name"; @@ -17,7 +19,6 @@ const LOCATION_FIELD: &str = "location"; const PATH_FIELD: &str = "path"; const LINE_FIELD: &str = "line"; const COLUMNS_FIELD: &str = "columns"; -const LOWER_METHOD: &str = "lower"; const STRIP_METHOD: &str = "strip"; /// `SchemaColumn` fields a named-schema column keeps when a header column augments it. const KEPT_NAMED_FIELDS: [&str; 6] = [ @@ -100,54 +101,82 @@ pub(crate) fn schema_enums<'py>( Ok(enums) } +/// Schema columns as Python specs paired with their native facts, in declaration order. +pub(crate) struct DeclaredColumns<'py> { + pub(crate) specs: Vec>, + pub(crate) facts: Vec, +} + /// `_merge_model_schema_columns`: named columns first, augmented by header audits, then the rest. pub(crate) fn merged_columns<'py>( context: &LoopContext<'_, 'py>, schema: &ModelSchema<'py>, - local: &Bound<'py, PyTuple>, - model: (&str, &str), -) -> PyResult> { - let py: Python<'py> = local.py(); - let mut merged: Vec> = schema - .declaration - .getattr(COLUMNS_FIELD)? - .try_iter()? - .collect::>()?; - let mut keys: Vec> = Vec::with_capacity(merged.len()); - for column in &merged { - keys.push(column.getattr(NAME_FIELD)?.call_method0(LOWER_METHOD)?); + local: DeclaredColumns<'py>, + model: (&str, &str, &Bound<'py, PyAny>), +) -> PyResult> { + let (model_name, relative_path, model_path) = model; + let mut merged: DeclaredColumns<'py> = DeclaredColumns { + specs: Vec::new(), + facts: Vec::new(), + }; + for column in schema.declaration.getattr(COLUMNS_FIELD)?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + merged + .facts + .push(python_column_fact(&column, Some(model_path))?); + merged.specs.push(column); } - let mut additional: Vec> = Vec::new(); - for column in local.iter() { - let key: Bound<'py, PyAny> = column.getattr(NAME_FIELD)?.call_method0(LOWER_METHOD)?; - let mut named_index: Option = None; - for (index, candidate) in keys.iter().enumerate() { - if candidate.eq(&key)? { - named_index = Some(index); - } - } - let Some(index) = named_index else { - additional.push(column); + let keys: Vec = merged + .facts + .iter() + .map(|fact| fact.name.to_lowercase()) + .collect(); + let mut additional: DeclaredColumns<'py> = DeclaredColumns { + specs: Vec::new(), + facts: Vec::new(), + }; + for (column, fact) in local.specs.into_iter().zip(local.facts) { + let key: String = fact.name.to_lowercase(); + let Some(index) = keys.iter().rposition(|candidate| *candidate == key) else { + additional.specs.push(column); + additional.facts.push(fact); continue; }; - check_augmentation(context, &column, &merged[index], model)?; - merged[index] = augmented(context, &merged[index], &column)?; + check_augmentation( + context, + &fact, + &merged.specs[index], + (model_name, relative_path), + )?; + let (spec, augmented_fact) = augmented( + context, + (&merged.specs[index], &merged.facts[index]), + (&column, fact), + )?; + merged.specs[index] = spec; + merged.facts[index] = augmented_fact; } - merged.extend(additional); - PyTuple::new(py, merged) + merged.specs.extend(additional.specs); + merged.facts.extend(additional.facts); + Ok(merged) } fn augmented<'py>( context: &LoopContext<'_, 'py>, - named: &Bound<'py, PyAny>, - local: &Bound<'py, PyAny>, -) -> PyResult> { + (named, named_fact): (&Bound<'py, PyAny>, &SchemaColumnFact), + (local, local_fact): (&Bound<'py, PyAny>, SchemaColumnFact), +) -> PyResult<(Bound<'py, PyAny>, SchemaColumnFact)> { let py: Python<'py> = named.py(); let mut audits: Vec> = named .getattr(AUDITS_FIELD)? .try_iter()? .collect::>()?; - for audit in local.getattr(AUDITS_FIELD)?.try_iter()? { + let mut fact: SchemaColumnFact = named_fact.clone(); + for (audit, definition) in local + .getattr(AUDITS_FIELD)? + .try_iter()? + .zip(local_fact.audit_definitions) + { let audit: Bound<'py, PyAny> = audit?; let mut present: bool = false; for existing in &audits { @@ -158,6 +187,7 @@ fn augmented<'py>( } if !present { audits.push(audit); + fact.audit_definitions.push(definition); } } let fields: Bound<'py, PyDict> = PyDict::new(py); @@ -165,54 +195,67 @@ fn augmented<'py>( fields.set_item(field, named.getattr(field)?)?; } fields.set_item(AUDITS_FIELD, PyTuple::new(py, audits)?)?; - fields.set_item(MIGRATE_FROM_FIELD, local.getattr(MIGRATE_FROM_FIELD)?)?; - context.contracts.schema_column.call((), Some(&fields)) + fields.set_item(MIGRATE_FROM_FIELD, &local_fact.migrate_from)?; + fact.migrate_from = local_fact.migrate_from; + Ok(( + context.contracts.schema_column.call((), Some(&fields))?, + fact, + )) } /// `_validate_model_schema_audit_augmentation`. fn check_augmentation( context: &LoopContext<'_, '_>, - local: &Bound<'_, PyAny>, + local: &SchemaColumnFact, named: &Bound<'_, PyAny>, model: (&str, &str), ) -> PyResult<()> { let (model_name, relative_path) = model; - let name: String = local.getattr(NAME_FIELD)?.str()?.to_string(); - let location: Bound<'_, PyAny> = named.getattr(LOCATION_FIELD)?; - let origin: String = if location.is_none() { - "the named schema".to_owned() - } else { - format!( - "{}:{}", - location.getattr(PATH_FIELD)?.str()?, - location.getattr(LINE_FIELD)?.str()? - ) - }; + let name: &str = &local.name; let mut overridden: Vec<&str> = Vec::new(); - for field in [TYPE_FIELD, NULLABLE_FIELD, DESCRIPTION_FIELD] { - if !local.getattr(field)?.is_none() { - overridden.push(field); - } + if local.data_type.is_some() { + overridden.push(TYPE_FIELD); + } + if local.nullable.is_some() { + overridden.push(NULLABLE_FIELD); + } + if local.described { + overridden.push(DESCRIPTION_FIELD); } if !overridden.is_empty() { let message: String = format!( "model '{model_name}' in {relative_path} cannot override {} for named-schema column \ - '{name}' from {origin}; only audit augmentation is supported", - overridden.join(", ") + '{name}' from {}; only audit augmentation is supported", + overridden.join(", "), + origin(named)? ); return Err(compile_error(context.classes, &message, None)); } - if !local.getattr(AUDITS_FIELD)?.is_truthy()? && local.getattr(MIGRATE_FROM_FIELD)?.is_none() { + if local.audit_definitions.is_empty() && local.migrate_from.is_none() { let message: String = format!( "model '{model_name}' in {relative_path} redeclares named-schema column '{name}' from \ - {origin} without audits or migrate_from; only audit augmentation and column \ - migrations are supported" + {} without audits or migrate_from; only audit augmentation and column \ + migrations are supported", + origin(named)? ); return Err(compile_error(context.classes, &message, None)); } Ok(()) } +/// Where a named-schema column is declared, as its `path:line` or the named schema. +fn origin(named: &Bound<'_, PyAny>) -> PyResult { + let location: Bound<'_, PyAny> = named.getattr(LOCATION_FIELD)?; + if location.is_none() { + return Ok("the named schema".to_owned()); + } + Ok(format!( + "{}:{}", + location.getattr(PATH_FIELD)?.str()?, + location.getattr(LINE_FIELD)?.str()? + )) +} + fn invalid_name(context: &LoopContext<'_, '_>, model_name: &str) -> pyo3::PyErr { let message: String = format!("model '{model_name}': model_schema must be a non-empty string"); compile_error(context.classes, &message, None) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index c02bcdf032..ab58515a7e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -18,15 +18,16 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{FromPyObject, pyclass, pymethods}; use serde_json::Value; +use sqlbuild_analysis::compiled_project::main::model_facts::model_facts as project_model; use sqlbuild_analysis::compiled_project::main::project_models::project_models; use sqlbuild_analysis::compiled_project::main::record_model::record_model; use sqlbuild_analysis::compiled_project::main::record_model_analysis::record_model_analysis; use sqlbuild_analysis::compiled_project::models::{ - CompiledModelFacts, CompiledProjectFacts, DeclarationFact, DynamicFamilyFact, DynamicProofFact, - ModelAnalysisFact, ModelConfigFacts, ModelReferenceFact, ModelSchemaFact, SchemaColumnFact, + CompiledModelFacts, CompiledProjectFacts, DeclarationFact, DynamicProofFact, ModelAnalysisFact, + ModelConfigFacts, ModelReferenceFact, ModelSchemaFact, }; -use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_guard}; +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::compiled_project::audit_loop::{ AttachmentOwner, AuditFileRow, AuditSettings, model_owner, resource_owner, }; @@ -69,20 +70,6 @@ const TABLE_MATERIALIZATION: &str = "table"; /// `(name, type)`. type NamedType = (String, Option); -/// `(name, type, nullable, audit count, declared outside the model)`. -pub(crate) type SchemaColumnPy = (String, Option, Option, u32, bool); -/// `(key, name, pivot column, value column, aggregate, type, name pattern)`. -pub(crate) type DynamicFamilyPy = ( - String, - String, - String, - String, - String, - String, - Option, -); -/// `(entry audit count, columns, dynamic families, (type enforcement, named schema))`. -pub(crate) type SchemaPy = (u32, Vec, Vec, (bool, bool)); /// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. type ProofPy = (bool, bool, Vec, Option, Vec); /// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. @@ -97,7 +84,6 @@ pub(crate) struct ModelInputPy<'py>( (String, String, String, String, String), (Bound<'py, PyDict>, Vec, Option), Vec<(String, String, Option)>, - Option, Vec, (Vec>, Vec>), ); @@ -169,20 +155,22 @@ impl NativeCompiledProject { fn attachment_owners<'py>(&self, py: Python<'py>) -> PyResult>> { let mut owners: Vec> = Vec::new(); for record in &self.inputs { - let Some(schema) = &record.schema else { + let (Some(schema), Some(fact)) = (&record.schema, self.input_schema_fact(record)) + else { continue; }; - let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); owners.push(model_owner( - model_file - .getattr("file_path")? - .getattr("stem")? - .extract()?, - model_file + record.name.clone(), + record + .model_file + .bind(py) .getattr(RELATIVE_PATH_ATTRIBUTE)? .str()? .to_string(), - (schema.audits.bind(py), schema.columns.bind(py)), + ( + schema.audits.bind(py), + (schema.columns.bind(py), &fact.columns), + ), )?); } for source in &self.sources { @@ -275,6 +263,11 @@ impl NativeCompiledProject { &self.seeds } + /// The schema fact recorded for an input record's model. + pub(crate) fn input_schema_fact(&self, record: &ModelInputRecord) -> Option<&ModelSchemaFact> { + project_model(&self.facts, &record.name).and_then(|facts| facts.schema.as_ref()) + } + /// Every model's input record, in loop order. pub(crate) fn inputs(&self) -> &[ModelInputRecord] { &self.inputs @@ -337,15 +330,6 @@ impl NativeCompiledProject { Self::default() } - /// Retain one model's compile-input facts; a name recorded again is replaced in place. - pub(crate) fn record_model(&mut self, model: ModelInputPy<'_>) -> PyResult<()> { - compiler_guard(|| { - let facts: CompiledModelFacts = model_facts(model)?; - record_model(&mut self.facts, facts); - Ok(()) - }) - } - /// Attach a recorded model's typed analysis; `False` when that model was never recorded. pub(crate) fn record_model_analysis( &mut self, @@ -646,7 +630,6 @@ pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult) -> PyResult, key: &str) -> PyResult ModelSchemaFact { - ModelSchemaFact { - audit_count, - columns: columns - .into_iter() - .map( - |(name, data_type, nullable, audit_count, declared_outside_model)| { - SchemaColumnFact { - name, - data_type, - nullable, - audit_count, - declared_outside_model, - } - }, - ) - .collect(), - dynamic_columns: dynamic_columns.into_iter().map(family_fact).collect(), - type_enforcement, - named_schema, - } -} - -fn family_fact( - (key, name, pivot_column, value_column, aggregate, data_type, name_pattern): DynamicFamilyPy, -) -> DynamicFamilyFact { - DynamicFamilyFact { - key, - name, - pivot_column, - value_column, - aggregate, - data_type, - name_pattern, - } -} - pub(crate) fn declarations(rows: Vec>) -> PyResult> { rows.into_iter().map(declaration).collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index 44bb8ad7ac..61c1012b24 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -250,6 +250,22 @@ impl NativeProjectView { )) } + /// The project field custom-rule facts read by `name`, or `None` for any other field. + pub(crate) fn fact_field(&self, py: Python<'_>, name: &str) -> Option> { + let field: &Py = match name { + "sources" => &self.fields.compiler.sources, + "seeds" => &self.fields.compiler.seeds, + "functions" => &self.fields.compiler.functions, + "audits" => &self.fields.compiler.audits, + "sql_tests" => &self.fields.compiler.sql_tests, + "public_enums" => &self.fields.boundary.public_enums, + "public_constants" => &self.fields.boundary.public_constants, + "models" => return Some(self.fields.models.clone_ref(py).into_any()), + _ => return None, + }; + Some(field.clone_ref(py)) + } + /// The compiled project this view projects. pub(crate) fn owner(&self) -> &Py { &self.owner diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index c99a29a70c..480081c418 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -5,13 +5,15 @@ use pyo3::prelude::{ }; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_project::inputs_view::NativeInputsView; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::project::{NativeCompiledProject, model_facts}; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; +use crate::bindings::_helpers::compiled_project::schema_facts::entry_schema_fact; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -445,9 +447,12 @@ fn hand_built_project<'py>( let models: Bound<'py, PyAny> = project.getattr("models")?; for model in models.try_iter()? { let model: Bound<'py, PyAny> = model?; - native - .borrow_mut() - .record_model(model_row(&model)?.as_any().extract()?)?; + let mut facts: CompiledModelFacts = model_facts(model_row(&model)?.as_any().extract()?)?; + facts.schema = entry_schema_fact( + &model.getattr("schema_entry")?, + &model.getattr("relative_path")?, + )?; + native.borrow_mut().retain(facts); } let _ = record_model_analyses(&native, &models)?; Ok(native) @@ -519,7 +524,6 @@ fn model_row<'py>(model: &Bound<'py, PyAny>) -> PyResult> { )? .into_any(), PyList::new(py, references)?.into_any(), - schema_row(&model.getattr("schema_entry")?, &relative_path)?, PyList::new( py, model @@ -533,69 +537,6 @@ fn model_row<'py>(model: &Bound<'py, PyAny>) -> PyResult> { ) } -fn schema_row<'py>( - entry: &Bound<'py, PyAny>, - relative_path: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = entry.py(); - if entry.is_none() { - return Ok(entry.clone()); - } - let columns: Vec> = entry - .getattr("columns")? - .try_iter()? - .map(|column| { - let column: Bound<'py, PyAny> = column?; - let location: Bound<'py, PyAny> = column.getattr("location")?; - let elsewhere: bool = - !location.is_none() && location.getattr("path")?.ne(relative_path)?; - PyTuple::new( - py, - [ - column.getattr("name")?, - column.getattr("type")?, - column.getattr("nullable")?, - column.getattr("audits")?.len()?.into_bound_py_any(py)?, - elsewhere.into_bound_py_any(py)?, - ], - ) - }) - .collect::>()?; - let families: Vec> = entry - .getattr("dynamic_columns")? - .try_iter()? - .map(|family| { - let family: Bound<'py, PyAny> = family?; - PyTuple::new( - py, - [ - family.getattr("name")?.call_method0("casefold")?, - family.getattr("name")?, - family.getattr("pivot_column")?, - family.getattr("value_column")?, - family.getattr("aggregate")?, - family.getattr("type")?, - family.getattr("name_pattern")?, - ], - ) - }) - .collect::>()?; - Ok(PyTuple::new( - py, - [ - entry.getattr("audits")?.len()?.into_bound_py_any(py)?, - PyList::new(py, columns)?.into_any(), - PyList::new(py, families)?.into_any(), - ( - entry.getattr("type_enforcement")?.is_truthy()?, - !entry.getattr("model_schema")?.is_none(), - ) - .into_bound_py_any(py)?, - ], - )? - .into_any()) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { for function in [ wrap_pyfunction!(compiled_model_record, module)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index 4fa99fa92f..81546134f1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -251,10 +251,7 @@ fn missing_descriptions<'py>( let mut found: Vec>> = Vec::new(); for record in project.inputs() { let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); - let name: String = model_file - .getattr("file_path")? - .getattr("stem")? - .extract()?; + let name: String = record.name.clone(); let path: String = model_file.getattr("relative_path")?.str()?.extract()?; let contents: String = model_file.getattr("contents")?.extract()?; let entry: String = format!("description \"What one row of {name} represents\""); @@ -484,10 +481,7 @@ fn cursor_without_inputs<'py>( let cursor: Option = config_text(&values, CURSOR_KEY)?; let strategy: Option = config_text(&values, "microbatch_strategy")?; let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); - let name: String = model_file - .getattr("file_path")? - .getattr("stem")? - .extract()?; + let name: String = record.name.clone(); let Some((message, help)) = cursor_lacks_inputs( &name, &CursorModel { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs new file mode 100644 index 0000000000..1124532b51 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs @@ -0,0 +1,128 @@ +//! Native facts of declared schema columns and families, read once from their Python specs. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::compiled_project::models::{ + DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, +}; +use sqlbuild_core::text::main::python_casefold::python_casefold; +use sqlbuild_model_config::header_metadata::models::DynamicFamily; + +const NAME_FIELD: &str = "name"; +const TYPE_FIELD: &str = "type"; +const NULLABLE_FIELD: &str = "nullable"; +const DESCRIPTION_FIELD: &str = "description"; +const MIGRATE_FROM_FIELD: &str = "migrate_from"; +const AUDITS_FIELD: &str = "audits"; +const LOCATION_FIELD: &str = "location"; +const PATH_FIELD: &str = "path"; +const DEFINITION_NAME_FIELD: &str = "definition_name"; +const COLUMNS_FIELD: &str = "columns"; +const DYNAMIC_COLUMNS_FIELD: &str = "dynamic_columns"; +const TYPE_ENFORCEMENT_FIELD: &str = "type_enforcement"; +const MODEL_SCHEMA_FIELD: &str = "model_schema"; +const PIVOT_COLUMN_FIELD: &str = "pivot_column"; +const VALUE_COLUMN_FIELD: &str = "value_column"; +const AGGREGATE_FIELD: &str = "aggregate"; +const NAME_PATTERN_FIELD: &str = "name_pattern"; + +/// A Python column spec's fact; its location is kept when it differs from `model_path`. +pub(crate) fn python_column_fact( + column: &Bound<'_, PyAny>, + model_path: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let mut audit_definitions: Vec = Vec::new(); + if let Some(audits) = present(column, AUDITS_FIELD)? { + for audit in audits.try_iter()? { + audit_definitions.push(audit?.getattr(DEFINITION_NAME_FIELD)?.extract()?); + } + } + let location_path: Option = match present(column, LOCATION_FIELD)? { + Some(location) => { + let path: Bound<'_, PyAny> = location.getattr(PATH_FIELD)?; + let outside: bool = match model_path { + Some(model_path) => !path.is(model_path) && path.ne(model_path)?, + None => true, + }; + outside + .then(|| path.str().map(|text| text.to_string())) + .transpose()? + } + None => None, + }; + Ok(SchemaColumnFact { + name: column.getattr(NAME_FIELD)?.str()?.extract()?, + data_type: optional(column, TYPE_FIELD)?, + nullable: optional(column, NULLABLE_FIELD)?, + described: present(column, DESCRIPTION_FIELD)?.is_some(), + migrate_from: optional(column, MIGRATE_FROM_FIELD)?, + audit_definitions, + location_path, + }) +} + +/// The facts of every column spec in `columns`. +pub(crate) fn python_column_facts( + columns: &Bound<'_, PyAny>, + model_path: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + columns + .try_iter()? + .map(|column| python_column_fact(&column?, model_path)) + .collect() +} + +/// The fact of a hand-built model's `SchemaModelEntry`, or `None` without one. +pub(crate) fn entry_schema_fact( + entry: &Bound<'_, PyAny>, + model_path: &Bound<'_, PyAny>, +) -> PyResult> { + if entry.is_none() { + return Ok(None); + } + let mut dynamic_columns: Vec = Vec::new(); + for family in entry.getattr(DYNAMIC_COLUMNS_FIELD)?.try_iter()? { + let family: Bound<'_, PyAny> = family?; + dynamic_columns.push(family_fact(&DynamicFamily { + name: family.getattr(NAME_FIELD)?.extract()?, + pivot_column: family.getattr(PIVOT_COLUMN_FIELD)?.extract()?, + value_column: family.getattr(VALUE_COLUMN_FIELD)?.extract()?, + aggregate: family.getattr(AGGREGATE_FIELD)?.extract()?, + data_type: family.getattr(TYPE_FIELD)?.extract()?, + name_pattern: family.getattr(NAME_PATTERN_FIELD)?.extract()?, + })); + } + Ok(Some(ModelSchemaFact { + audit_count: u32::try_from(entry.getattr(AUDITS_FIELD)?.len()?).unwrap_or(u32::MAX), + columns: python_column_facts(&entry.getattr(COLUMNS_FIELD)?, Some(model_path))?, + dynamic_columns, + type_enforcement: entry.getattr(TYPE_ENFORCEMENT_FIELD)?.is_truthy()?, + named_schema: !entry.getattr(MODEL_SCHEMA_FIELD)?.is_none(), + })) +} + +/// A parsed dynamic column family's fact, keyed by its casefolded name. +pub(crate) fn family_fact(family: &DynamicFamily) -> DynamicFamilyFact { + DynamicFamilyFact { + key: python_casefold(&family.name), + name: family.name.clone(), + pivot_column: family.pivot_column.clone(), + value_column: family.value_column.clone(), + aggregate: family.aggregate.clone(), + data_type: family.data_type.clone(), + name_pattern: family.name_pattern.clone(), + } +} + +/// An attribute's value, `None` when it is missing or `None`. +fn present<'py>(spec: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + Ok(spec.getattr_opt(name)?.filter(|value| !value.is_none())) +} + +fn optional<'py, T: pyo3::FromPyObject<'py>>( + spec: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + present(spec, name)? + .map(|value| value.extract()) + .transpose() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs index 5c786f48e3..327b430fe6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs @@ -4,7 +4,7 @@ use pyo3::exceptions::{PyIndexError, PyRuntimeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::PyTuple; -use pyo3::{pyclass, pymethods}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_attachments::seeds::main::pair_seed_declarations::pair_seed_declarations; use crate::bindings::_helpers::compiled_project::compiled_model_view::{ @@ -283,3 +283,21 @@ impl NativeCompiledSeed { false } } + +impl NativeCompiledSeed { + /// The `CompiledSeed` field values, in field order, without building the record. + pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py).into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.seed_file(py)?.into_bound(py), + self.schema_entry(py)?.into_bound(py), + self.schema_file(py)?.into_bound(py), + self.destination(py)?.into_bound(py), + pyo3::types::PyBool::new(py, self.is_external()) + .to_owned() + .into_any(), + ]) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index 83dfeff912..6451ce7dd2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -4,7 +4,7 @@ use pyo3::exceptions::PyIndexError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; -use pyo3::{pyclass, pymethods}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use crate::bindings::_helpers::compiled_project::compiled_model_view::object_key; use crate::bindings::_helpers::compiled_project::model_input_view::{ @@ -247,6 +247,19 @@ impl NativeCompiledSource { } } +impl NativeCompiledSource { + /// The `CompiledSource` field values, in field order, without building the record. + pub(crate) fn record_values<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.key(py)?.into_bound(py), + self.deps(py).into_any(), + self.name(py)?.into_bound_py_any(py)?, + self.source_entry(py)?.into_bound(py), + self.source_file(py)?.into_bound(py), + ]) + } +} + /// Keep `attached` in `owner`; return each source's view. pub(crate) fn retain_sources( owner: &Bound<'_, NativeCompiledProject>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs index 2493399577..b69fc3bbd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_errors/constants.rs @@ -491,15 +491,6 @@ pub(crate) const COMPILER_ERRORS: &[ErrorSpec] = &[ optional: &[], fixed: &[], }, - ErrorSpec { - name: "FactDigestError", - parent: "RulesError", - doc: "Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.", - class_attributes: &[], - required: &[], - optional: &[], - fixed: &[], - }, ErrorSpec { name: "OpaqueModuleStateError", parent: "RulesError", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 7d6bf00b85..f4d373bc63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -4,32 +4,6 @@ use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::models::{FieldDefault, FieldSpec, RecordSpec}; -const R_NATIVE_MODEL_ANALYSIS_REQUEST: RecordSpec = RecordSpec { - name: "NativeModelAnalysisRequest", - doc: "The Python model analysis's arguments plus the facts dynamic pivot proofs need.", - fields: &[ - FieldSpec::required("known_functions"), - FieldSpec::required("known_types"), - FieldSpec::required("model_inputs"), - FieldSpec::required("column_nullability_by_table"), - FieldSpec::required("column_types_by_table"), - FieldSpec::required("inference_profile"), - FieldSpec::required("allow_compact_analysis"), - FieldSpec::required("rich_type_inference"), - FieldSpec::required("analysis_cache"), - FieldSpec::required("complete_binding_schemas"), - FieldSpec::required("dynamic_families_by_table"), - ], -}; -const R_NATIVE_MODEL_ANALYSES: RecordSpec = RecordSpec { - name: "NativeModelAnalyses", - doc: "Each analysed model's position in the finished session that keeps its outcome.", - fields: &[ - FieldSpec::required("positions"), - FieldSpec::required("succeeded"), - FieldSpec::required("session"), - ], -}; const R_NATIVE_PIVOT_TABLES: RecordSpec = RecordSpec { name: "NativePivotTables", doc: "The relation facts Python's dynamic pivot proofs read, for proofs outside a session.", @@ -944,8 +918,6 @@ const R_COMPILE_PIPELINE_RESULT: RecordSpec = RecordSpec { /// The compiler, lint and config value records. pub(crate) const COMPILER_RECORDS: &[RecordSpec] = &[ - R_NATIVE_MODEL_ANALYSIS_REQUEST, - R_NATIVE_MODEL_ANALYSES, R_NATIVE_PIVOT_TABLES, R_NATIVE_COLUMN_QUERY, R_NATIVE_QUERY_COLUMNS, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs index 53b0a5fd8a..4b840c0e71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs @@ -89,7 +89,7 @@ fn fact_contract_schema(schema: &ModelSchemaFact) -> ContractSchema { name: column.name.clone(), declared_type: column.data_type.clone(), not_null: column.nullable == Some(false), - declared_in_named_schema: schema.named_schema && column.declared_outside_model, + declared_in_named_schema: schema.named_schema && column.location_path.is_some(), }); } let mut dynamic_columns: Vec = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/user_code.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/user_code.rs index 12156b2506..30ede99a40 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/user_code.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/user_code.rs @@ -52,11 +52,6 @@ pub(crate) enum UserCodeHost { SqlResourceRefKind, LoadCustomRules, CustomRulePayloads, - FactOutcomeDigest, - FullFactKeys, - IsTreeFact, - SharedFactEncoder, - BuildRuleFactViews, EventExporterInputError, EventExporterStateError, InvokeLifecycleEventSink, @@ -301,31 +296,6 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::UserCode, }, - HostEntry { - name: "fact_outcome_digest", - kind: HostKind::Function, - edge: HostEdge::UserCode, - }, - HostEntry { - name: "full_fact_keys", - kind: HostKind::Function, - edge: HostEdge::UserCode, - }, - HostEntry { - name: "is_tree_fact", - kind: HostKind::Function, - edge: HostEdge::UserCode, - }, - HostEntry { - name: "shared_fact_encoder", - kind: HostKind::Function, - edge: HostEdge::UserCode, - }, - HostEntry { - name: "build_rule_fact_views", - kind: HostKind::Function, - edge: HostEdge::UserCode, - }, HostEntry { name: "EventExporterInputError", kind: HostKind::Exception, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index 557af6ff50..494a823e2a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -1,10 +1,14 @@ //! What each model, seed, source and table function declares about its columns before analysis. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; -use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; +use crate::bindings::_helpers::compiled_project::model_input_view::{ + attached_columns, attached_schema, +}; +use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; @@ -20,15 +24,19 @@ const UNKNOWN_MEMBER: &str = "UNKNOWN"; type DeclaredRelations<'py> = (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); /// `{column.name: column.type or "UNKNOWN"}`, or the declared type unchanged when `exact`. -fn column_types<'py>(columns: &Bound<'py, PyAny>, exact: bool) -> PyResult> { - let types: Bound<'py, PyDict> = PyDict::new(columns.py()); - for column in columns.try_iter()? { - let column: Bound<'py, PyAny> = column?; - let declared: Bound<'py, PyAny> = column.getattr("type")?; - if exact || declared.is_truthy()? { - types.set_item(column.getattr("name")?, declared)?; - } else { - types.set_item(column.getattr("name")?, UNKNOWN_TYPE)?; +fn column_types<'py>( + py: Python<'py>, + columns: &[SchemaColumnFact], + exact: bool, +) -> PyResult> { + let types: Bound<'py, PyDict> = PyDict::new(py); + for column in columns { + match column.data_type.as_deref() { + Some(declared) if exact || !declared.is_empty() => { + types.set_item(&column.name, declared)?; + } + None if exact => types.set_item(&column.name, py.None())?, + _ => types.set_item(&column.name, UNKNOWN_TYPE)?, } } Ok(types) @@ -36,24 +44,24 @@ fn column_types<'py>(columns: &Bound<'py, PyAny>, exact: bool) -> PyResult( - columns: &Bound<'py, PyAny>, + columns: &[SchemaColumnFact], nullability: &Bound<'py, PyAny>, ) -> PyResult> { - let facts: Bound<'py, PyDict> = PyDict::new(columns.py()); - for column in columns.try_iter()? { - let column: Bound<'py, PyAny> = column?; - let mut non_null: bool = column - .getattr("nullable")? - .is(PyBool::new(columns.py(), false).as_any()); - for audit in column.getattr("audits")?.try_iter()? { - non_null = non_null || audit?.getattr("definition_name")?.eq(NOT_NULL_AUDIT_NAME)?; - } - let state: &str = if non_null { - NON_NULL_MEMBER + let facts: Bound<'py, PyDict> = PyDict::new(nullability.py()); + let non_null_member: Bound<'py, PyAny> = nullability.getattr(NON_NULL_MEMBER)?; + let unknown_member: Bound<'py, PyAny> = nullability.getattr(UNKNOWN_MEMBER)?; + for column in columns { + let non_null: bool = column.nullable == Some(false) + || column + .audit_definitions + .iter() + .any(|definition| definition == NOT_NULL_AUDIT_NAME); + let state: &Bound<'py, PyAny> = if non_null { + &non_null_member } else { - UNKNOWN_MEMBER + &unknown_member }; - facts.set_item(column.getattr("name")?, nullability.getattr(state)?)?; + facts.set_item(&column.name, state)?; } Ok(facts) } @@ -76,8 +84,9 @@ fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult, Bound<'py, PyAny>) = (schema.name, schema.columns); - types.set_item(&name, column_types(&columns, false)?)?; + let (name, columns): (String, Vec) = + (schema.name, attached_columns(&model)?); + types.set_item(&name, column_types(py, &columns, false)?)?; nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; let contract: Option> = model .getattr("config")? @@ -88,55 +97,49 @@ fn declared_relations<'py>(inputs: Bound<'py, PyAny>) -> PyResult contract.eq(ENFORCED_CONTRACT)?, None => false, }; - if enforced && columns.is_truthy()? && !schema.dynamic_columns.is_truthy()? { - let stem: Bound<'py, PyAny> = model - .getattr("model_file")? - .getattr("file_path")? - .getattr("stem")?; - complete.set_item(stem, column_types(&columns, false)?)?; + if enforced && !columns.is_empty() && !schema.dynamic_columns.is_truthy()? { + complete.set_item(&name, column_types(py, &columns, false)?)?; } } - let mut seeds: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + let mut seeds: Vec<(Bound<'py, PyAny>, Vec)> = Vec::new(); for seed in inputs.getattr("seed_inputs")?.try_iter()? { let entry: Bound<'py, PyAny> = seed?.getattr("schema_entry")?; let name: Bound<'py, PyAny> = entry.getattr("name")?; - let columns: Bound<'py, PyAny> = entry.getattr("columns")?; - types.set_item(&name, column_types(&columns, false)?)?; + let columns: Vec = python_column_facts(&entry.getattr("columns")?, None)?; + types.set_item(&name, column_types(py, &columns, false)?)?; nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; seeds.push((name, columns)); } for source in inputs.getattr("source_inputs")?.try_iter()? { let entry: Bound<'py, PyAny> = source?.getattr("source_entry")?; let name: Bound<'py, PyAny> = entry.getattr("name")?; - let columns: Bound<'py, PyAny> = entry.getattr("columns")?; - types.set_item(&name, column_types(&columns, false)?)?; + let columns: Vec = python_column_facts(&entry.getattr("columns")?, None)?; + types.set_item(&name, column_types(py, &columns, false)?)?; nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; - if entry.getattr(CONTRACT_KEY)?.eq(ENFORCED_CONTRACT)? && columns.is_truthy()? { - complete.set_item(&name, column_types(&columns, false)?)?; + if entry.getattr(CONTRACT_KEY)?.eq(ENFORCED_CONTRACT)? && !columns.is_empty() { + complete.set_item(&name, column_types(py, &columns, false)?)?; } } for (name, columns) in seeds { - if columns.is_truthy()? { - complete.set_item(name, column_types(&columns, false)?)?; + if !columns.is_empty() { + complete.set_item(name, column_types(py, &columns, false)?)?; } } for function in inputs.getattr("sql_function_inputs")?.try_iter()? { let function: Bound<'py, PyAny> = function?; - let columns: Bound<'py, PyAny> = function.getattr("return_columns")?; - if !columns.is_truthy()? { + let columns: Vec = + python_column_facts(&function.getattr("return_columns")?, None)?; + if columns.is_empty() { continue; } let name: String = table_function_name(&function)?; - types.set_item(&name, column_types(&columns, true)?)?; + types.set_item(&name, column_types(py, &columns, true)?)?; let unknown: Bound<'py, PyDict> = PyDict::new(py); - for column in columns.try_iter()? { - unknown.set_item( - column?.getattr("name")?, - nullability.getattr(UNKNOWN_MEMBER)?, - )?; + for column in &columns { + unknown.set_item(&column.name, nullability.getattr(UNKNOWN_MEMBER)?)?; } nulls.set_item(&name, unknown)?; - complete.set_item(&name, column_types(&columns, true)?)?; + complete.set_item(&name, column_types(py, &columns, true)?)?; } Ok((types, nulls, complete)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs index c6574dadcf..afdc0ce45c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/ids.rs @@ -343,8 +343,6 @@ native_ids! { MovePreview => "MovePreview", NamedSqlHookEntry => "NamedSqlHookEntry", NativeColumnQuery => "NativeColumnQuery", - NativeModelAnalyses => "NativeModelAnalyses", - NativeModelAnalysisRequest => "NativeModelAnalysisRequest", NativeModelConfigInputs => "NativeModelConfigInputs", NativeModelConfigSession => "NativeModelConfigSession", NativePivotTables => "NativePivotTables", @@ -604,7 +602,6 @@ native_ids! { DiscoveryConflict => "DiscoveryConflictError", Discovery => "DiscoveryError", EventExporterDiscovery => "EventExporterDiscoveryError", - FactDigest => "FactDigestError", FingerprintInput => "FingerprintInputError", FutureCursorSafety => "FutureCursorSafetyError", HostCancelled => "HostCancelledError", @@ -663,7 +660,6 @@ native_ids! { /// A native function, class or value native code reaches through the module. pub(crate) enum NativeAttr { AnalyzeCompileProject => "analyze_compile_project", - AnalyzeNativeModelSql => "analyze_native_model_sql", ApplyCommandWarehouse => "apply_command_warehouse", ApplyConfiguredMicrobatchLimits => "apply_configured_microbatch_limits", ApplyDeferredLocations => "apply_deferred_locations", @@ -1134,7 +1130,6 @@ native_ids! { SqlTestScanCache => "SqlTestScanCache", SqlValueValidationError => "SqlValueValidationError", StandalonePivotModels => "standalone_pivot_models", - StartModelAnalysisSession => "start_model_analysis_session", StaticSqlTestPlanningDiagnostics => "static_sql_test_planning_diagnostics", StoredOldNameViewColumns => "stored_old_name_view_columns", TerminalEventIndex => "TerminalEventIndex", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs index dc8359a764..5ff1bb5822 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/constants.rs @@ -25,4 +25,4 @@ pub(crate) const CUSTOM_HOST_CANCELLED_MARKER: &str = "cancelled"; /// Rules that need proven column types and are skipped without SQL analysis. pub(crate) const TYPE_PROOF_RULE_CODES: [&str; 1] = ["SQBRCONTRACT105"]; /// Runtime identity custom-rule hosts and their cache entries record. -pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v2"; +pub(crate) const CUSTOM_HOST_RUNTIME_VERSION: &str = "sqlbuild-rules-custom-v3"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs index 6d32501eee..d692334b08 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_evaluation.rs @@ -1,12 +1,10 @@ //! Incremental custom-rule evaluation keyed on the compiler facts each invocation read. use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Instant; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; @@ -19,9 +17,7 @@ use sqlbuild_rules::custom_rule_cache::types::ReadSet; use crate::bindings::_helpers::cache::native_store::current_code_identity; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; -use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; +use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::rules::constants::{ CUSTOM_HOST_RUNTIME_VERSION, CUSTOM_RULES_CACHE_FILE, @@ -30,6 +26,8 @@ use crate::bindings::_helpers::rules::custom_hosts::{ HostCancel, HostRequest, host_payload_project, run_custom_hosts, }; use crate::bindings::_helpers::rules::custom_rule_cache::CustomRuleCache; +use crate::bindings::_helpers::rules::fact_encoder::{Encoded, reproducible}; +use crate::bindings::_helpers::rules::fact_replay::{FactReplay, is_tree_fact}; const PROJECT_SUBJECT: &str = "project"; const MODEL_SUBJECT: &str = "model"; @@ -84,7 +82,6 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( if custom.is_empty() { return outcome(py, (Vec::new(), 0, 0), (0, 0)); } - let facts_project: Bound<'py, PyAny> = host_project(run.project)?; let enabled: bool = run .config .getattr("cache")? @@ -120,18 +117,21 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( } else { None }; + let project_dir: PathBuf = run.project_dir.extract()?; + let mut digests: FactDigests<'py> = FactDigests::new(run.project, &project_dir)?; + let models: Vec<(String, String, Option)> = digests.replay.model_rows()?; + let mut model_paths: Vec = models.iter().map(|(path, _, _)| path.clone()).collect(); + model_paths.sort(); let mut cache: CustomRuleCache = CustomRuleCache::new( cache_path, rows, run.project.getattr("effective_target_name")?.extract()?, - model_rows(run.project)?, + models, ); - let mut digests: FactDigests<'py> = - FactDigests::new(&facts_project, run.project_dir, run.dialect)?; let pending: Vec = cache.pending_readsets().to_vec(); let mut readset_digests: Vec> = Vec::new(); for readset in &pending { - readset_digests.push(digests.reproducible(|digests| digests.combined(readset))?); + readset_digests.push(reproducible(digests.combined(readset))?); } let plan: HostPlan = cache.plan(readset_digests); if !plan.is_empty() { @@ -139,12 +139,7 @@ pub(crate) fn evaluate_custom_rules_cached<'py>( .iter() .map(|(code, rule)| Ok((code.clone(), host_subject(rule)?))) .collect::>()?; - let mut model_paths: Vec = model_rows(run.project)? - .into_iter() - .map(|(path, _, _)| path) - .collect(); - model_paths.sort(); - let project_dir: PathBuf = run.project_dir.extract()?; + let facts_project: Bound<'py, PyAny> = host_project(run.project)?; let host_run: HostRun = run_custom_hosts( py, &HostRequest { @@ -206,7 +201,7 @@ fn evaluations( Some(reads) => reads, None => { if full_keys.is_none() { - full_keys = Some(digests.full_fact_keys()?); + full_keys = Some(digests.replay.full_fact_keys()?); } let mut reads: Vec = run .untracked_reads @@ -223,9 +218,8 @@ fn evaluations( .into_iter() .collect(); if !combined.contains_key(&readset) { - let digest: Option = digests - .reproducible(|digests| digests.observed(&readset, &run.observed))? - .flatten(); + let digest: Option = + reproducible(digests.observed(&readset, &run.observed))?.flatten(); combined.insert(readset.clone(), digest); } let digest: Option = combined.get(&readset).cloned().flatten(); @@ -243,69 +237,30 @@ fn evaluations( /// Memoized digests of recorded fact keys against one compiled project snapshot. struct FactDigests<'py> { - project: Bound<'py, PyAny>, - project_dir: Bound<'py, PyAny>, - dialect: String, - views: Option>, - shared: Bound<'py, PyAny>, + replay: FactReplay<'py>, digests: HashMap, } impl<'py> FactDigests<'py> { - fn new( - project: &Bound<'py, PyAny>, - project_dir: &Bound<'py, PyAny>, - dialect: &str, - ) -> PyResult { + fn new(project: &Bound<'py, PyAny>, project_dir: &Path) -> PyResult { Ok(Self { - project: project.clone(), - project_dir: project_dir.clone(), - dialect: dialect.to_owned(), - views: None, - shared: host::entry(project.py(), UserCodeHost::SharedFactEncoder)?.call0()?, + replay: FactReplay::new(project, project_dir)?, digests: HashMap::new(), }) } - fn views(&mut self) -> PyResult> { - if let Some(views) = &self.views { - return Ok(views.clone()); - } - let py: Python<'py> = self.project.py(); - let views: Bound<'py, PyAny> = call_with( - &host::entry(py, UserCodeHost::BuildRuleFactViews)?, - &[ - ("project", self.project.clone()), - ("project_dir", self.project_dir.clone()), - ("dialect", self.dialect.clone().into_bound_py_any(py)?), - ], - )?; - self.views = Some(views.clone()); - Ok(views) - } - - /// One fact digest; raises `FactDigestError` when it cannot be reproduced. - fn digest(&mut self, key: &FactKey) -> PyResult { + /// One fact digest; `Unreproducible` when it cannot be recomputed. + fn digest(&mut self, key: &FactKey) -> Encoded { if let Some(cached) = self.digests.get(key) { return Ok(cached.clone()); } - let py: Python<'py> = self.project.py(); - let views: Bound<'py, PyAny> = self.views()?; - let digest: String = call_with( - &host::entry(py, UserCodeHost::FactOutcomeDigest)?, - &[ - ("views", views), - ("key", PyTuple::new(py, key)?.into_any()), - ("shared", self.shared.clone()), - ], - )? - .extract()?; + let digest: String = self.replay.digest(key)?; self.digests.insert(key.clone(), digest.clone()); Ok(digest) } /// One read set's digest over its keys in canonical order. - fn combined(&mut self, keys: &[FactKey]) -> PyResult { + fn combined(&mut self, keys: &[FactKey]) -> Encoded { let mut sorted: Vec<&FactKey> = keys.iter().collect(); sorted.sort(); let mut bytes: Vec = Vec::new(); @@ -321,11 +276,9 @@ impl<'py> FactDigests<'py> { &mut self, readset: &[FactKey], observed: &BTreeMap>, - ) -> PyResult> { - let py: Python<'py> = self.project.py(); - let is_tree_fact: Bound<'py, PyAny> = host::entry(py, UserCodeHost::IsTreeFact)?; + ) -> Encoded> { for key in readset { - if is_tree_fact.call1((PyTuple::new(py, key)?,))?.is_truthy()? + if is_tree_fact(key) && let Some(seen) = observed.get(key) && seen.as_deref() != Some(self.digest(key)?.as_str()) { @@ -334,32 +287,6 @@ impl<'py> FactDigests<'py> { } self.combined(readset).map(Some) } - - /// Keys covering every fact a custom rule can observe through its context. - fn full_fact_keys(&mut self) -> PyResult> { - let py: Python<'py> = self.project.py(); - host::entry(py, UserCodeHost::FullFactKeys)? - .call1((self.views()?,))? - .extract() - } - - /// `body`'s value, or `None` when it raised `FactDigestError`. - fn reproducible( - &mut self, - body: impl FnOnce(&mut Self) -> PyResult, - ) -> PyResult> { - let py: Python<'py> = self.project.py(); - match body(self) { - Ok(value) => Ok(Some(value)), - Err(error) if fact_digest_error(py, &error)? => Ok(None), - Err(error) => Err(error), - } - } -} - -fn fact_digest_error(py: Python<'_>, error: &PyErr) -> PyResult { - let class: Bound<'_, PyAny> = registry::error(py, ErrorId::FactDigest)?; - error.value(py).is_instance(&class) } /// `orjson.dumps(key)` of one fact key. @@ -394,31 +321,6 @@ fn host_project<'py>(project: &Bound<'py, PyAny>) -> PyResult> imported(py, "dataclasses", "replace")?.call((record,), Some(&changes)) } -/// `(relative path, name, materialization)` of each compiled model, in project order. -fn model_rows(project: &Bound<'_, PyAny>) -> PyResult)>> { - let mut rows: Vec<(String, String, Option)> = Vec::new(); - for model in project.getattr("models")?.try_iter()? { - let model: Bound<'_, PyAny> = model?; - let materialized: Bound<'_, PyAny> = model - .getattr("config")? - .getattr("values")? - .call_method1("get", ("materialized",))?; - rows.push(( - model - .getattr("relative_path")? - .call_method0("as_posix")? - .extract()?, - model.getattr("name")?.extract()?, - if materialized.is_instance_of::() { - Some(materialized.extract::()?) - } else { - None - }, - )); - } - Ok(rows) -} - fn subject_value(rule: &Bound<'_, PyAny>) -> PyResult> { let subject: Bound<'_, PyAny> = rule.getattr("subject")?; if subject.is_none() { @@ -506,6 +408,18 @@ fn custom_host_payload<'py>(project: &Bound<'py, PyAny>) -> PyResult, + project_dir: PathBuf, + key: FactKey, +) -> PyResult> { + reproducible(FactReplay::new(project, &project_dir)?.digest(&key)) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(custom_host_payload, module)?) + module.add_function(wrap_pyfunction!(custom_host_payload, module)?)?; + module.add_function(wrap_pyfunction!(custom_rule_fact_digest, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 6ea9a18e7e..b43a2d45c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -30,7 +30,7 @@ use crate::bindings::_helpers::rules::lint_expansion::{ }; use crate::bindings::_helpers::rules::lint_inputs::{model_key_row, model_lint_expansion}; use crate::bindings::_helpers::rules::lint_pipeline::{ - CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, + CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, }; use crate::bindings::_helpers::rules::relation_keys::{ RelationKeyRow, RelationKeys, relation_keys, @@ -211,7 +211,7 @@ fn start_early_lint( let Some(plan) = early_lint_plan(inputs, dialect)? else { return Ok(None); }; - let mut compiled: HashMap = HashMap::new(); + let mut compiled: HashMap = HashMap::new(); for model in inputs.getattr("model_inputs")?.try_iter()? { if let Some((path, expansion)) = model_lint_expansion(&model?)? { compiled.insert(path, expansion); @@ -244,7 +244,7 @@ struct EarlyLintWork { settings: LintSettings, discovered: Py, static_scope: Py, - compiled: HashMap, + compiled: HashMap, stop: Arc, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs new file mode 100644 index 0000000000..e03d93ca64 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_encoder.rs @@ -0,0 +1,328 @@ +//! A simple deterministic encoding of custom-rule fact values, for fact digests. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::collections::hash_map::Entry; + +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, PyTypeMethods, + Python, +}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyFrozenSet, PyInt, PyList, PySet, + PyString, PyStringMethods, PyTuple, PyType, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, OrjsonOptions}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::records::fields::record_spec; +use crate::bindings::_helpers::records::models::NativeRecord; + +const VALUE_LABEL: &str = "value"; +const ERROR_LABEL: &str = "error"; +const ENUM_CLASS: (&str, &str) = ("enum", "Enum"); +const INT_TAG: &str = "int"; +const FLOAT_TAG: &str = "float"; + +static ENUM_TYPE: PyOnceLock> = PyOnceLock::new(); + +/// Why a fact cannot be digested: the value is not reproducible, or Python failed. +pub(crate) enum FactFailure { + Unreproducible, + Python(PyErr), +} + +impl From for FactFailure { + fn from(error: PyErr) -> Self { + Self::Python(error) + } +} + +pub(crate) type Encoded = Result; + +/// How the encoder writes values of one Python type. +#[derive(Clone)] +enum TypeKind { + Record(Vec<&'static str>), + Enum, + Set, + Other, +} + +/// One classified type and the qualified name its records are tagged with. +struct TypeInfo { + kind: TypeKind, + qualified_name: String, + _class: Py, +} + +/// Encode fact values; each record is written as its class and the digest of its fields. +pub(crate) struct FactEncoder { + types: RefCell>, + records: RefCell, JsonValue)>>, + enums: RefCell, JsonValue)>>, +} + +impl FactEncoder { + pub(crate) fn new() -> Self { + Self { + types: RefCell::new(HashMap::new()), + records: RefCell::new(HashMap::new()), + enums: RefCell::new(HashMap::new()), + } + } + + /// The JSON standing for `value`. + pub(crate) fn encode(&self, value: &Bound<'_, PyAny>) -> Encoded { + if value.is_none() { + return Ok(JsonValue::Null); + } + if let Ok(text) = value.downcast::() { + return Ok(JsonValue::String(text.to_string_lossy().into_owned())); + } + if let Ok(flag) = value.downcast::() { + return Ok(JsonValue::Bool(flag.is_true())); + } + if value.is_instance_of::() { + return tagged(INT_TAG, value); + } + if value.is_instance_of::() { + return tagged(FLOAT_TAG, value); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + items.push(self.encode(&item?)?); + } + return Ok(JsonValue::Array(items)); + } + if let Ok(mapping) = value.downcast::() { + let mut entries: Vec<(String, JsonValue)> = Vec::with_capacity(mapping.len()); + for (key, item) in mapping.iter() { + entries.push((text_of(&self.encode(&key)?)?, self.encode(&item)?)); + } + entries.sort_by(|left, right| left.0.cmp(&right.0)); + return Ok(JsonValue::Array( + entries + .into_iter() + .map(|(key, item)| JsonValue::Array(vec![JsonValue::String(key), item])) + .collect(), + )); + } + self.object(value) + } + + /// A record of `qualified_name` with these encoded field values. + pub(crate) fn dataclass( + &self, + qualified_name: &str, + fields: Vec, + ) -> Encoded { + Ok(JsonValue::Array(vec![ + JsonValue::String(qualified_name.to_owned()), + JsonValue::String(digest_of(&JsonValue::Array(fields))?), + ])) + } + + /// One record from its field values, memoised by identity. + pub(crate) fn record( + &self, + value: &Bound<'_, PyAny>, + qualified_name: &str, + fields: &[Bound<'_, PyAny>], + ) -> Encoded { + let identity: usize = value.as_ptr() as usize; + if let Some((_, encoded)) = self.records.borrow().get(&identity) { + return Ok(encoded.clone()); + } + let mut encoded: Vec = Vec::with_capacity(fields.len()); + for field in fields { + encoded.push(self.encode(field)?); + } + let encoded: JsonValue = self.dataclass(qualified_name, encoded)?; + self.records + .borrow_mut() + .insert(identity, (value.clone().unbind(), encoded.clone())); + Ok(encoded) + } + + /// The qualified name records of `class` are tagged with. + pub(crate) fn qualified_name(&self, class: &Bound<'_, PyType>) -> PyResult { + Ok(self.info(class)?.1) + } + + /// A native record's field by name, or `None` when `value` is not a record with that field. + pub(crate) fn field<'py>( + &self, + value: &Bound<'py, PyAny>, + name: &str, + ) -> PyResult>> { + let (TypeKind::Record(names), _) = self.info(&value.get_type())? else { + return Ok(None); + }; + let Some(index) = names.iter().position(|field| *field == name) else { + return Ok(None); + }; + let record: &NativeRecord = value.downcast::()?.get(); + Ok(record + .values + .get(index) + .map(|field| field.bind(value.py()).clone())) + } + + fn object(&self, value: &Bound<'_, PyAny>) -> Encoded { + let class: Bound<'_, PyType> = value.get_type(); + let (kind, qualified_name) = self.info(&class)?; + match kind { + TypeKind::Record(_) => { + let py: Python<'_> = value.py(); + let fields: Vec> = value + .downcast::() + .map_err(PyErr::from)? + .get() + .values + .iter() + .map(|field| field.bind(py).clone()) + .collect(); + self.record(value, &qualified_name, &fields) + } + TypeKind::Enum => { + let identity: usize = value.as_ptr() as usize; + if let Some((_, encoded)) = self.enums.borrow().get(&identity) { + return Ok(encoded.clone()); + } + let encoded: JsonValue = JsonValue::Array(vec![ + JsonValue::String(qualified_name), + self.encode(&value.getattr("value")?)?, + ]); + self.enums + .borrow_mut() + .insert(identity, (value.clone().unbind(), encoded.clone())); + Ok(encoded) + } + TypeKind::Set => { + let mut texts: Vec = Vec::new(); + for item in value.try_iter()? { + texts.push(text_of(&self.encode(&item?)?)?); + } + texts.sort(); + Ok(JsonValue::Array( + texts.into_iter().map(JsonValue::String).collect(), + )) + } + TypeKind::Other => Ok(JsonValue::Array(vec![ + JsonValue::String(qualified_name), + JsonValue::String(value.str()?.to_string_lossy().into_owned()), + ])), + } + } + + /// The kind and qualified name of `class`, classified once. + fn info(&self, class: &Bound<'_, PyType>) -> PyResult<(TypeKind, String)> { + let identity: usize = class.as_ptr() as usize; + if let Some(info) = self.types.borrow().get(&identity) { + return Ok((info.kind.clone(), info.qualified_name.clone())); + } + let info: TypeInfo = classify(class)?; + let found: (TypeKind, String) = (info.kind.clone(), info.qualified_name.clone()); + if let Entry::Vacant(vacant) = self.types.borrow_mut().entry(identity) { + vacant.insert(info); + } + Ok(found) + } +} + +fn classify(class: &Bound<'_, PyType>) -> PyResult { + let py: Python<'_> = class.py(); + let enum_type: &Py = ENUM_TYPE.get_or_try_init(py, || { + Ok::<_, PyErr>(imported(py, ENUM_CLASS.0, ENUM_CLASS.1)?.unbind()) + })?; + let kind: TypeKind = if class.is_subclass_of::()? { + TypeKind::Record( + record_spec(class)? + .fields + .iter() + .map(|field| field.name) + .collect(), + ) + } else if class.is_subclass(enum_type.bind(py))? { + TypeKind::Enum + } else if class.is_subclass_of::()? || class.is_subclass_of::()? { + TypeKind::Set + } else { + TypeKind::Other + }; + Ok(TypeInfo { + kind, + qualified_name: format!("{}.{}", class.module()?, class.qualname()?), + _class: class.clone().unbind(), + }) +} + +/// A number as its type tag and `repr`, so it never collides with a string of the same text. +fn tagged(tag: &str, value: &Bound<'_, PyAny>) -> Encoded { + Ok(JsonValue::Array(vec![ + JsonValue::String(tag.to_owned()), + JsonValue::String(value.repr()?.to_string_lossy().into_owned()), + ])) +} + +/// The compact JSON text of `value`. +pub(crate) fn text_of(value: &JsonValue) -> Encoded { + dumps(value, &JsonDialect::Orjson(OrjsonOptions::default())) + .map_err(|_unencodable| FactFailure::Unreproducible) +} + +/// The SHA-256 hex digest of `value`'s JSON text. +pub(crate) fn digest_of(value: &JsonValue) -> Encoded { + Ok(sha256_hex(text_of(value)?.as_bytes())) +} + +/// The digest of a fact that produced `value`, already encoded. +pub(crate) fn value_digest(value: JsonValue) -> Encoded { + digest_of(&JsonValue::Array(vec![ + JsonValue::String(VALUE_LABEL.to_owned()), + value, + ])) +} + +/// The digest of a fact that raised an error of `kind` with `message`. +pub(crate) fn error_digest(kind: &str, message: &str) -> Encoded { + digest_of(&JsonValue::Array(vec![ + JsonValue::String(ERROR_LABEL.to_owned()), + JsonValue::String(kind.to_owned()), + JsonValue::String(message.to_owned()), + ])) +} + +/// The digest a rule host records for a live filesystem fact's value, or `None`. +#[pyfunction] +fn fact_value_digest(value: &Bound<'_, PyAny>) -> PyResult> { + reproducible(FactEncoder::new().encode(value).and_then(value_digest)) +} + +/// The digest a rule host records for a live filesystem fact that raised `error`, or `None`. +#[pyfunction] +fn fact_error_digest(error: &Bound<'_, PyAny>) -> PyResult> { + let kind: String = error.get_type().name()?.to_string(); + let message: String = error.str()?.to_string_lossy().into_owned(); + reproducible(error_digest(&kind, &message)) +} + +/// The digest, `None` when the value is not reproducible, or the Python failure. +pub(crate) fn reproducible(encoded: Encoded) -> PyResult> { + match encoded { + Ok(value) => Ok(Some(value)), + Err(FactFailure::Unreproducible) => Ok(None), + Err(FactFailure::Python(error)) => Err(error), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(fact_value_digest, module)?)?; + module.add_function(wrap_pyfunction!(fact_error_digest, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs new file mode 100644 index 0000000000..97919bc7ca --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_replay.rs @@ -0,0 +1,822 @@ +//! Native replay of the compiler facts custom-rule evaluations recorded, for their digests. + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, PyTypeMethods, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyType}; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_rules::custom_host_pool::types::FactKey; +use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; +use sqlbuild_rules::project_tree::main::pure_path_text::pure_path_text; +use sqlbuild_rules::project_tree::main::resolve_project_path::resolve_project_path; +use sqlbuild_rules::project_tree::main::tracked_text_path::tracked_text_path; +use sqlbuild_rules::project_tree::models::ProjectTreeEntry; + +use crate::bindings::_helpers::compiled_project::audit_views::NativeCompiledAudit; +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + InferredRow, NativeCompiledModel, +}; +use crate::bindings::_helpers::compiled_project::function_views::NativeCompiledFunction; +use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; +use crate::bindings::_helpers::compiled_project::seed_views::NativeCompiledSeed; +use crate::bindings::_helpers::compiled_project::source_views::NativeCompiledSource; +use crate::bindings::_helpers::records::fields::record_spec; +use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::fact_encoder::{ + Encoded, FactEncoder, FactFailure, error_digest, value_digest, +}; +use crate::bindings::_helpers::rules::fact_tree::{ + Outcome, Raised, TreeQuery, nodes_value, path_string, read_text, relative_parts, + selected_nodes, tree_nodes, +}; + +const MODEL_RECORD: &str = "sqlbuild._native.Model"; +const MATERIALIZED: &str = "materialized"; +const CONTRACT: &str = "contract"; +const GRAIN: &str = "grain"; +const MISSING_MODEL: &str = "MissingModel"; +const POSIX_SEPARATOR: char = '/'; + +/// One fact a custom rule can read through its context. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Fact { + Models, + Sources, + Seeds, + Functions, + Tests, + Audits, + TreePaths, + TestsAll, + AuditsAll, + PublicEnums, + PublicConstants, + Enums, + Constants, + SqlForModel, + Dependencies, + Dependents, + Declared, + Inferred, + Names, + Enforced, + Grain, + TestsForModel, + AuditsForModel, + Children, + Descendants, + Glob, + RelativeParts, + ResourcesUnder, + ReadText, +} + +/// What a fact key carries after the fact's name. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Arguments { + None, + ModelPath, + ModelName, + Text(usize), +} + +/// Every fact by key name, globals first, in the order full read sets list them. +const FACTS: [(&str, Fact, Arguments); 29] = [ + ("project.models", Fact::Models, Arguments::None), + ("project.sources", Fact::Sources, Arguments::None), + ("project.seeds", Fact::Seeds, Arguments::None), + ("project.functions", Fact::Functions, Arguments::None), + ("project.tests", Fact::Tests, Arguments::None), + ("project.audits", Fact::Audits, Arguments::None), + ("tree.paths", Fact::TreePaths, Arguments::None), + ("tests.all", Fact::TestsAll, Arguments::None), + ("audits.all", Fact::AuditsAll, Arguments::None), + ( + "declarations.public_enums", + Fact::PublicEnums, + Arguments::None, + ), + ( + "declarations.public_constants", + Fact::PublicConstants, + Arguments::None, + ), + ("declarations.enums", Fact::Enums, Arguments::None), + ("declarations.constants", Fact::Constants, Arguments::None), + ("sql.for_model", Fact::SqlForModel, Arguments::ModelPath), + ( + "graph.dependencies", + Fact::Dependencies, + Arguments::ModelPath, + ), + ("graph.dependents", Fact::Dependents, Arguments::ModelPath), + ("columns.declared", Fact::Declared, Arguments::ModelPath), + ("columns.inferred", Fact::Inferred, Arguments::ModelPath), + ("columns.names", Fact::Names, Arguments::ModelPath), + ("contracts.enforced", Fact::Enforced, Arguments::ModelPath), + ("contracts.grain", Fact::Grain, Arguments::ModelPath), + ("tests.for_model", Fact::TestsForModel, Arguments::ModelName), + ( + "audits.for_model", + Fact::AuditsForModel, + Arguments::ModelName, + ), + ("tree.children", Fact::Children, Arguments::Text(1)), + ("tree.descendants", Fact::Descendants, Arguments::Text(1)), + ("tree.glob", Fact::Glob, Arguments::Text(1)), + ( + "tree.relative_parts", + Fact::RelativeParts, + Arguments::Text(2), + ), + ( + "tree.resources_under", + Fact::ResourcesUnder, + Arguments::Text(1), + ), + ("tree.read_text", Fact::ReadText, Arguments::Text(1)), +]; +const READ_TEXT_FACT: &str = "tree.read_text"; + +/// Whether the fact is read from the live project filesystem. +pub(crate) fn is_tree_fact(key: &FactKey) -> bool { + key.first().is_some_and(|name| { + FACTS.iter().any(|(fact_name, fact, arguments)| { + fact_name == name + && (*fact == Fact::TreePaths || matches!(arguments, Arguments::Text(_))) + }) + }) +} + +/// What the facts read of one compiled model. +pub(crate) struct ModelEntry<'py> { + pub(crate) model: Bound<'py, PyAny>, + pub(crate) name: String, + path: Bound<'py, PyAny>, + path_text: String, + pub(crate) posix: String, + pub(crate) config_values: Option>, + key: (String, String), + deps: Vec<(String, String)>, +} + +/// The models in project order with the indexes the model facts look them up by. +struct Models<'py> { + entries: Vec>, + sorted: Vec, + by_path: HashMap, + by_key: HashMap<(String, String), usize>, + dependents: HashMap<(String, String), Vec>, +} + +/// Recompute recorded facts from one compiled project snapshot and digest them. +pub(crate) struct FactReplay<'py> { + project: Bound<'py, PyAny>, + root: PathBuf, + encoder: FactEncoder, + models: Option>, + nodes: Option>, +} + +impl<'py> FactReplay<'py> { + pub(crate) fn new(project: &Bound<'py, PyAny>, project_dir: &Path) -> PyResult { + Ok(Self { + project: project.clone(), + root: resolve_project_path(project_dir)?, + encoder: FactEncoder::new(), + models: None, + nodes: None, + }) + } + + /// One fact's digest, or `Unreproducible` when it cannot be recomputed. + pub(crate) fn digest(&mut self, key: &FactKey) -> Encoded { + let py: Python<'py> = self.project.py(); + match self.outcome(key) { + Ok(Ok(value)) => value_digest(value), + Ok(Err(raised)) => error_digest(&raised.kind, &raised.message), + Err(FactFailure::Python(error)) => { + let (kind, message) = python_error(py, &error)?; + error_digest(&kind, &message) + } + Err(failure) => Err(failure), + } + } + + /// Keys covering every fact a custom rule can observe through its context. + pub(crate) fn full_fact_keys(&mut self) -> PyResult> { + let mut keys: Vec = FACTS + .iter() + .filter(|(_, _, arguments)| *arguments == Arguments::None) + .map(|(name, _, _)| vec![(*name).to_owned()]) + .collect(); + let models: &Models<'py> = self.models()?; + for index in &models.sorted { + let model: &ModelEntry<'py> = &models.entries[*index]; + for (name, _, arguments) in &FACTS { + match arguments { + Arguments::ModelPath => { + keys.push(vec![(*name).to_owned(), model.posix.clone()]) + } + Arguments::ModelName => keys.push(vec![(*name).to_owned(), model.name.clone()]), + Arguments::None | Arguments::Text(_) => {} + } + } + } + let root: PathBuf = self.root.clone(); + let nodes: &Vec = self.nodes.get_or_insert_with(|| tree_nodes(&root)); + for node in nodes { + if node.is_file && tracked_text_path(&node.relative) { + keys.push(vec![ + READ_TEXT_FACT.to_owned(), + node.relative.to_string_lossy().into_owned(), + ]); + } + } + Ok(keys) + } + + fn outcome(&mut self, key: &FactKey) -> Encoded { + let Some((name, arguments)) = key.split_first() else { + return Err(FactFailure::Unreproducible); + }; + let Some((_, fact, expected)) = FACTS.iter().find(|(fact_name, _, expected)| { + fact_name == name + && match expected { + Arguments::None => arguments.is_empty(), + Arguments::ModelPath | Arguments::ModelName => arguments.len() == 1, + Arguments::Text(count) => arguments.len() == *count, + } + }) else { + return Err(FactFailure::Unreproducible); + }; + match expected { + Arguments::None => self.global(*fact).map(Ok), + Arguments::ModelPath => self.model_fact(*fact, &arguments[0]), + Arguments::ModelName => self.named(*fact, &arguments[0]).map(Ok), + Arguments::Text(_) => self.tree_fact(*fact, arguments), + } + } + + fn global(&mut self, fact: Fact) -> Encoded { + let py: Python<'py> = self.project.py(); + match fact { + Fact::Models => { + let sorted: Vec = self.models()?.sorted.clone(); + self.model_values(&sorted) + } + Fact::Sources | Fact::Seeds | Fact::Functions | Fact::Audits | Fact::AuditsAll => { + let field: &str = match fact { + Fact::Sources => "sources", + Fact::Seeds => "seeds", + Fact::Functions => "functions", + _ => "audits", + }; + let mut values: Vec = Vec::new(); + for item in self.project_field(field)?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let (name, _, fields) = item_fields(&self.encoder, &item)?; + values.push(self.encoder.record(&item, &name, &fields)?); + } + Ok(JsonValue::Array(values)) + } + Fact::Tests | Fact::TestsAll => { + let tests: Bound<'py, PyAny> = self.project_field("sql_tests")?; + self.encoder.encode(&tests) + } + Fact::PublicEnums | Fact::PublicConstants => { + let field: &str = if fact == Fact::PublicEnums { + "public_enums" + } else { + "public_constants" + }; + let declarations: Bound<'py, PyAny> = self.project_field(field)?; + self.encoder.encode(&declarations) + } + Fact::Enums | Fact::Constants => self.declarations(py, fact == Fact::Enums), + Fact::TreePaths => { + let root: PathBuf = self.root.clone(); + let nodes: &Vec = + self.nodes.get_or_insert_with(|| tree_nodes(&root)); + let nodes: Vec<&ProjectTreeEntry> = nodes.iter().collect(); + nodes_value(&self.encoder, &nodes) + } + _ => Err(FactFailure::Unreproducible), + } + } + + /// Public declarations, then each model's own, in project order. + fn declarations(&mut self, py: Python<'py>, enums: bool) -> Encoded { + let public: Bound<'py, PyAny> = self.project_field(if enums { + "public_enums" + } else { + "public_constants" + })?; + let mut values: Vec = vec![self.encoder.encode(&public)?]; + let models: Vec> = self + .models()? + .entries + .iter() + .map(|entry| entry.model.clone()) + .collect(); + for model in models { + let declarations: Bound<'py, PyAny> = match model.downcast::() { + Ok(view) if enums => view.get().enum_declarations(py)?.into_bound(py).into_any(), + Ok(view) => view + .get() + .constant_declarations(py)? + .into_bound(py) + .into_any(), + Err(_) => record_field( + &self.encoder, + &model, + if enums { + "enum_declarations" + } else { + "constant_declarations" + }, + )?, + }; + values.push(self.encoder.encode(&declarations)?); + } + Ok(JsonValue::Array(values)) + } + + fn model_fact(&mut self, fact: Fact, path: &str) -> Encoded { + let py: Python<'py> = self.project.py(); + let text: String = pure_path_text(Path::new(path)) + .to_string_lossy() + .into_owned(); + let Some(index) = self.models()?.by_path.get(&path_key(&text)).copied() else { + return Ok(Err(Raised::new(MISSING_MODEL, text))); + }; + let model: Bound<'py, PyAny> = self.models()?.entries[index].model.clone(); + let view: Option<&NativeCompiledModel> = match model.downcast::() { + Ok(view) => Some(view.get()), + Err(_) => None, + }; + let value: JsonValue = match fact { + Fact::SqlForModel => match &view { + Some(view) => { + let (authored, expanded) = view.sql_texts(py)?; + JsonValue::Array(vec![ + JsonValue::String(authored), + JsonValue::String(expanded), + ]) + } + None => { + let authored: Bound<'py, PyAny> = + record_field(&self.encoder, &model, "authored_sql")?; + let expanded: Bound<'py, PyAny> = + record_field(&self.encoder, &model, "query_sql")?; + JsonValue::Array(vec![ + self.encoder.encode(&authored)?, + self.encoder.encode(&expanded)?, + ]) + } + }, + Fact::Dependencies => { + let models: &Models<'py> = self.models()?; + let indexes: Vec = models.entries[index] + .deps + .iter() + .filter_map(|key| models.by_key.get(key).copied()) + .collect(); + self.model_values(&indexes)? + } + Fact::Dependents => { + let models: &Models<'py> = self.models()?; + let mut indexes: Vec = models + .dependents + .get(&models.entries[index].key) + .cloned() + .unwrap_or_default(); + indexes.sort_by(|left, right| { + models.entries[*left].name.cmp(&models.entries[*right].name) + }); + self.model_values(&indexes)? + } + Fact::Declared => match &view { + Some(view) => match view.declared_columns(py)? { + Some(columns) => self.encoder.encode(columns.bind(py).as_any())?, + None => JsonValue::Array(Vec::new()), + }, + None => { + let entry: Bound<'py, PyAny> = + record_field(&self.encoder, &model, "schema_entry")?; + if entry.is_none() { + JsonValue::Array(Vec::new()) + } else { + let columns: Bound<'py, PyAny> = + record_field(&self.encoder, &entry, "columns")?; + self.encoder.encode(&columns)? + } + } + }, + Fact::Inferred | Fact::Names => { + let rows: Option> = match &view { + Some(view) => view.inferred_rows(py)?, + None => inferred_record_rows(&self.encoder, &model)?, + }; + match rows { + None => JsonValue::Null, + Some(rows) => JsonValue::Array( + rows.into_iter() + .map(|(name, column_type, nullability)| { + if fact == Fact::Names { + return JsonValue::String(name); + } + JsonValue::Array(vec![ + JsonValue::String(name), + column_type.map_or(JsonValue::Null, JsonValue::String), + JsonValue::String(nullability), + ]) + }) + .collect(), + ), + } + } + Fact::Enforced | Fact::Grain => { + let name: &str = if fact == Fact::Enforced { + CONTRACT + } else { + GRAIN + }; + match &self.models()?.entries[index].config_values { + Some(values) => match values.get_item(name)? { + Some(value) => self.encoder.encode(&value)?, + None => JsonValue::Null, + }, + None => JsonValue::Null, + } + } + _ => return Err(FactFailure::Unreproducible), + }; + Ok(Ok(value)) + } + + /// Tests targeting the model, or audits attached to it, by model name. + fn named(&mut self, fact: Fact, name: &str) -> Encoded { + let (field, attribute) = if fact == Fact::TestsForModel { + ("sql_tests", "target_model_names") + } else { + ("audits", "attached_target_name") + }; + let mut values: Vec = Vec::new(); + for item in self.project_field(field)?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let (qualified_name, names, fields) = item_fields(&self.encoder, &item)?; + let Some(position) = names.iter().position(|field| *field == attribute) else { + continue; + }; + let target: &Bound<'py, PyAny> = &fields[position]; + let matched: bool = match target.downcast::() { + Ok(text) => text.to_cow()? == name, + Err(_) => target.contains(name).unwrap_or(false), + }; + if matched { + values.push(self.encoder.record(&item, &qualified_name, &fields)?); + } + } + Ok(JsonValue::Array(values)) + } + + fn tree_fact(&mut self, fact: Fact, arguments: &[String]) -> Encoded { + let root: PathBuf = self.root.clone(); + let query: TreeQuery<'_> = match fact { + Fact::Children => TreeQuery::Children(&arguments[0]), + Fact::Descendants => TreeQuery::Descendants(&arguments[0]), + Fact::Glob => TreeQuery::Glob(&arguments[0]), + Fact::RelativeParts => { + return match relative_parts(&root, &arguments[0], &arguments[1]) { + Ok(parts) => Ok(Ok(JsonValue::Array( + parts + .iter() + .map(|part| path_string(part)) + .collect::>>()?, + ))), + Err(raised) => Ok(Err(raised)), + }; + } + Fact::ResourcesUnder => return self.resources_under(&arguments[0]).map(Ok), + _ => { + return Ok(read_text(&root, &arguments[0]).map(JsonValue::String)); + } + }; + let nodes: &Vec = self.nodes.get_or_insert_with(|| tree_nodes(&root)); + match selected_nodes(nodes, query) { + Ok(selected) => nodes_value(&self.encoder, &selected).map(Ok), + Err(raised) => Ok(Err(raised)), + } + } + + /// Models whose project path is beneath `path`, sorted by path. + fn resources_under(&mut self, path: &str) -> Encoded { + let posix: String = String::from_utf8_lossy(&pure_path_posix(Path::new(path))).into_owned(); + let prefix: String = format!( + "{}{POSIX_SEPARATOR}", + posix.trim_end_matches(POSIX_SEPARATOR) + ); + let models: &Models<'py> = self.models()?; + let by_posix: BTreeMap<&str, usize> = models + .sorted + .iter() + .map(|index| (models.entries[*index].posix.as_str(), *index)) + .collect(); + let indexes: Vec = by_posix + .into_iter() + .filter(|(relative, _)| relative.starts_with(&prefix)) + .map(|(_, index)| index) + .collect(); + self.model_values(&indexes) + } + + /// Each model as the public `Model` record the host builds for it. + fn model_values(&mut self, indexes: &[usize]) -> Encoded { + let mut values: Vec = Vec::with_capacity(indexes.len()); + for index in indexes { + let (name, path, materialization) = { + let entry: &ModelEntry<'py> = &self.models()?.entries[*index]; + let materialization: Option = materialization(entry)?; + (entry.name.clone(), entry.path.clone(), materialization) + }; + let path: JsonValue = self.encoder.encode(&path)?; + values.push(self.encoder.dataclass( + MODEL_RECORD, + vec![ + JsonValue::String(name), + path, + materialization.map_or(JsonValue::Null, JsonValue::String), + ], + )?); + } + Ok(JsonValue::Array(values)) + } + + /// `(posix path, name, materialization)` of each model, in project order. + pub(crate) fn model_rows(&mut self) -> PyResult)>> { + let models: &Models<'py> = self.models()?; + let mut rows: Vec<(String, String, Option)> = + Vec::with_capacity(models.entries.len()); + for entry in &models.entries { + rows.push(( + entry.posix.clone(), + entry.name.clone(), + materialization(entry)?, + )); + } + Ok(rows) + } + + /// One project field, from a native project view or a project record. + fn project_field(&mut self, name: &str) -> PyResult> { + project_field(&self.encoder, &self.project, name) + } + + fn models(&mut self) -> PyResult<&Models<'py>> { + if self.models.is_none() { + let models: Bound<'py, PyAny> = self.project_field("models")?; + let mut entries: Vec> = Vec::new(); + for model in models.try_iter()? { + entries.push(model_entry(&self.encoder, &model?)?); + } + self.models = Some(indexed(entries)); + } + self.models + .as_ref() + .ok_or_else(|| pyo3::exceptions::PyRuntimeError::new_err("models were not indexed")) + } +} + +/// One project field, from a native project view or a project record. +fn project_field<'py>( + encoder: &FactEncoder, + project: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + let py: Python<'py> = project.py(); + if let Ok(view) = project.downcast::() + && let Some(field) = view.get().fact_field(py, name) + { + return Ok(field.into_bound(py)); + } + record_field(encoder, project, name) +} + +/// Each model of a native project view or project record, read once without attribute lookups. +pub(crate) fn project_models<'py>(project: &Bound<'py, PyAny>) -> PyResult>> { + let encoder: FactEncoder = FactEncoder::new(); + let mut entries: Vec> = Vec::new(); + for model in project_field(&encoder, project, "models")?.try_iter()? { + entries.push(model_entry(&encoder, &model?)?); + } + Ok(entries) +} + +/// The model's `materialized` config value when it is a string. +fn materialization(entry: &ModelEntry<'_>) -> PyResult> { + let Some(values) = &entry.config_values else { + return Ok(None); + }; + Ok(match values.get_item(MATERIALIZED)? { + Some(value) => match value.downcast::() { + Ok(text) => Some(text.to_string_lossy().into_owned()), + Err(_) => None, + }, + None => None, + }) +} + +/// The models with their lookup indexes, as the Python fact views keyed them. +fn indexed(entries: Vec>) -> Models<'_> { + let mut sorted: Vec = (0..entries.len()).collect(); + sorted.sort_by(|left, right| entries[*left].posix.cmp(&entries[*right].posix)); + let mut by_path: HashMap = HashMap::new(); + let mut by_key: HashMap<(String, String), usize> = HashMap::new(); + for (index, entry) in entries.iter().enumerate() { + by_path.insert(path_key(&entry.path_text), index); + by_key.insert(entry.key.clone(), index); + } + let mut dependents: HashMap<(String, String), Vec> = HashMap::new(); + for entry in &entries { + let Some(subject) = by_key.get(&entry.key).copied() else { + continue; + }; + for dependency in &entry.deps { + dependents + .entry(dependency.clone()) + .or_default() + .push(subject); + } + } + Models { + entries, + sorted, + by_path, + by_key, + dependents, + } +} + +/// What the facts read of one model view or `CompiledModel` record. +fn model_entry<'py>(encoder: &FactEncoder, model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let (name, path, config, key, deps) = match model.downcast::() { + Ok(view) => { + let view: &NativeCompiledModel = view.get(); + ( + view.name(py)?, + view.relative_path(py)?.into_bound(py), + view.config(py)?.into_bound(py), + view.key_row(py)?, + view.dep_rows(py)?, + ) + } + Err(_) => { + let key: Bound<'py, PyAny> = record_field(encoder, model, "key")?; + let mut deps: Vec<(String, String)> = Vec::new(); + for dependency in record_field(encoder, model, "deps")?.try_iter()? { + deps.push(key_row(encoder, &dependency?)?); + } + ( + record_field(encoder, model, "name")?.extract()?, + record_field(encoder, model, "relative_path")?, + record_field(encoder, model, "config")?, + key_row(encoder, &key)?, + deps, + ) + } + }; + let path_text: String = path.str()?.to_string_lossy().into_owned(); + let posix: String = + String::from_utf8_lossy(&pure_path_posix(Path::new(&path_text))).into_owned(); + let config_values: Option> = match encoder.field(&config, "values")? { + Some(values) => Some(values.downcast_into::().map_err(PyErr::from)?), + None => None, + }; + Ok(ModelEntry { + model: model.clone(), + name, + path, + path_text: pure_path_text(Path::new(&path_text)) + .to_string_lossy() + .into_owned(), + posix, + config_values, + key, + deps, + }) +} + +/// `(name, type, nullability)` of each `InferredColumn` of a `CompiledModel` record. +fn inferred_record_rows( + encoder: &FactEncoder, + model: &Bound<'_, PyAny>, +) -> PyResult>> { + let columns: Bound<'_, PyAny> = record_field(encoder, model, "inferred_columns")?; + if columns.is_none() { + return Ok(None); + } + let mut rows: Vec = Vec::new(); + for column in columns.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let (nullability, _) = key_row_of(encoder, &column, ("nullability", "name"))?; + rows.push(( + record_field(encoder, &column, "name")?.extract()?, + record_field(encoder, &column, "type")?.extract()?, + nullability, + )); + } + Ok(Some(rows)) +} + +/// `(resource type, name)` of a `CompiledObjectKey` record. +fn key_row(encoder: &FactEncoder, key: &Bound<'_, PyAny>) -> PyResult<(String, String)> { + key_row_of(encoder, key, ("resource_type", "name")) +} + +/// A record's enum field by value, with its text field. +fn key_row_of( + encoder: &FactEncoder, + key: &Bound<'_, PyAny>, + (kind, name): (&str, &str), +) -> PyResult<(String, String)> { + let resource: Bound<'_, PyAny> = record_field(encoder, key, kind)?; + let resource: String = match encoder.encode(&resource) { + Ok(JsonValue::String(text)) => text, + Ok(_) | Err(FactFailure::Unreproducible) => resource.str()?.extract()?, + Err(FactFailure::Python(error)) => return Err(error), + }; + Ok((resource, record_field(encoder, key, name)?.extract()?)) +} + +/// A record's field by name; a missing field is a type error. +fn record_field<'py>( + encoder: &FactEncoder, + value: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + encoder.field(value, name)?.ok_or_else(|| { + pyo3::exceptions::PyTypeError::new_err(format!("fact source has no field {name}")) + }) +} + +/// A record's qualified name, field names and field values. +type ItemFields<'py> = (String, Vec<&'static str>, Vec>); + +/// A project item's record name, field names and field values, without building the record. +fn item_fields<'py>(encoder: &FactEncoder, item: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = item.py(); + let (record, values): (RecordId, Vec>) = + if let Ok(view) = item.downcast::() { + (RecordId::CompiledSource, view.get().record_values(py)?) + } else if let Ok(view) = item.downcast::() { + (RecordId::CompiledSeed, view.get().record_values(py)?) + } else if let Ok(view) = item.downcast::() { + (RecordId::CompiledFunction, view.get().record_values(py)?) + } else if let Ok(view) = item.downcast::() { + (RecordId::CompiledAudit, view.get().record_values(py)?) + } else { + let class: Bound<'py, PyType> = item.get_type(); + let names: Vec<&'static str> = record_spec(&class)? + .fields + .iter() + .map(|field| field.name) + .collect(); + let values: Vec> = names + .iter() + .map(|name| record_field(encoder, item, name)) + .collect::>()?; + return Ok((encoder.qualified_name(&class)?, names, values)); + }; + let class: Bound<'py, PyAny> = registry::class(py, record)?; + let class: &Bound<'py, PyType> = class.downcast::()?; + let names: Vec<&'static str> = record_spec(class)? + .fields + .iter() + .map(|field| field.name) + .collect(); + Ok((encoder.qualified_name(class)?, names, values)) +} + +/// The class name and `str()` of a Python exception a fact raised. +fn python_error(py: Python<'_>, error: &PyErr) -> PyResult<(String, String)> { + Ok(( + error.get_type(py).name()?.to_string(), + error.value(py).str()?.to_string_lossy().into_owned(), + )) +} + +/// A model path as the key `Path` equality compares by: case-insensitive on Windows. +#[cfg(windows)] +fn path_key(text: &str) -> String { + text.to_lowercase() +} + +/// A model path as the key `Path` equality compares by. +#[cfg(not(windows))] +fn path_key(text: &str) -> String { + text.to_owned() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_tree.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_tree.rs new file mode 100644 index 0000000000..fa3dec8b4c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fact_tree.rs @@ -0,0 +1,243 @@ +//! Project-tree facts as custom rules read them, with the errors Python spells for each failure. + +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; + +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; +use sqlbuild_rules::project_tree::errors::{ + ProjectTextError, ProjectTreeFileError, RelativePartsError, Utf8Failure, Utf8Reason, +}; +use sqlbuild_rules::project_tree::main::project_path_escapes::project_path_escapes; +use sqlbuild_rules::project_tree::main::project_relative_parts::project_relative_parts; +use sqlbuild_rules::project_tree::main::project_tree_children::project_tree_children; +use sqlbuild_rules::project_tree::main::project_tree_descendants::project_tree_descendants; +use sqlbuild_rules::project_tree::main::project_tree_entries::project_tree_entries; +use sqlbuild_rules::project_tree::main::project_tree_glob::project_tree_glob; +use sqlbuild_rules::project_tree::main::pure_path_text::pure_path_text; +use sqlbuild_rules::project_tree::main::read_project_text::read_project_text; +use sqlbuild_rules::project_tree::main::resolve_project_path::resolve_project_path; +use sqlbuild_rules::project_tree::main::tracked_text_suffixes::tracked_text_suffixes; +use sqlbuild_rules::project_tree::models::ProjectTreeEntry; + +use crate::bindings::_helpers::rules::fact_encoder::{Encoded, FactEncoder, FactFailure}; + +/// The qualified name of the `ProjectPath` record class. +const PROJECT_PATH_RECORD: &str = "sqlbuild._native.ProjectPath"; +pub(crate) const RULE_USAGE_ERROR: &str = "RuleUsageError"; +pub(crate) const VALUE_ERROR: &str = "ValueError"; +const ESCAPES: &str = "project path escapes the project: "; +const MISSING: &str = "project file does not exist: "; +const SYMLINK: &str = "project file must not be a symlink: "; +const UNREADABLE: &str = "could not read project file "; +/// The error `os.lstat` raises for a path holding a NUL byte. +const EMBEDDED_NUL_MESSAGE: &str = "lstat: embedded null character in path"; +/// `OSError` subclasses Python picks by `errno`, by the kind of error it names. +const ERRNO_CLASSES: [(ErrorKind, &str); 10] = [ + (ErrorKind::WouldBlock, "BlockingIOError"), + (ErrorKind::BrokenPipe, "BrokenPipeError"), + (ErrorKind::ConnectionAborted, "ConnectionAbortedError"), + (ErrorKind::ConnectionRefused, "ConnectionRefusedError"), + (ErrorKind::ConnectionReset, "ConnectionResetError"), + (ErrorKind::AlreadyExists, "FileExistsError"), + (ErrorKind::NotFound, "FileNotFoundError"), + (ErrorKind::Interrupted, "InterruptedError"), + (ErrorKind::PermissionDenied, "PermissionError"), + (ErrorKind::TimedOut, "TimeoutError"), +]; +const OS_ERROR: &str = "OSError"; + +/// An exception a fact raised: its class name and `str()`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Raised { + pub(crate) kind: String, + pub(crate) message: String, +} + +impl Raised { + pub(crate) fn new(kind: &str, message: String) -> Self { + Self { + kind: kind.to_owned(), + message, + } + } + + fn usage(message: String) -> Self { + Self::new(RULE_USAGE_ERROR, message) + } +} + +/// A recomputed fact: its encoded value, or the exception it raised. +pub(crate) type Outcome = Result; + +/// The project's sorted tree nodes, as `project_tree_paths` lists them. +pub(crate) fn tree_nodes(root: &Path) -> Vec { + let mut entries: Vec = project_tree_entries(root); + entries.sort_by(|left, right| { + left.relative + .as_os_str() + .as_encoded_bytes() + .cmp(right.relative.as_os_str().as_encoded_bytes()) + }); + entries +} + +/// `ProjectPath` nodes encoded as the host encodes the records. +pub(crate) fn nodes_value( + encoder: &FactEncoder, + nodes: &[&ProjectTreeEntry], +) -> Encoded { + let mut values: Vec = Vec::with_capacity(nodes.len()); + for node in nodes { + values.push(encoder.dataclass( + PROJECT_PATH_RECORD, + vec![ + path_string(&node.relative)?, + JsonValue::Bool(node.is_file), + JsonValue::Bool(!node.is_file), + ], + )?); + } + Ok(JsonValue::Array(values)) +} + +/// The tree nodes a text fact selects, or the error it raised. +pub(crate) fn selected_nodes<'a>( + nodes: &'a [ProjectTreeEntry], + fact: TreeQuery<'_>, +) -> Result, Raised> { + let values: Vec<&Path> = nodes.iter().map(|node| node.relative.as_path()).collect(); + let indexes: Vec = match fact { + TreeQuery::Children(path) => project_tree_children(&values, Path::new(path)), + TreeQuery::Descendants(path) => { + project_tree_descendants(&values, (!path.is_empty()).then(|| Path::new(path))) + } + TreeQuery::Glob(pattern) => { + if project_path_escapes(Path::new(pattern)) { + return Err(Raised::usage(format!("{ESCAPES}{pattern}"))); + } + project_tree_glob(&values, Path::new(pattern)) + } + }; + Ok(indexes.into_iter().map(|index| &nodes[index]).collect()) +} + +/// One node query over the tree. +#[derive(Clone, Copy)] +pub(crate) enum TreeQuery<'a> { + Children(&'a str), + Descendants(&'a str), + Glob(&'a str), +} + +/// `ProjectTree.relative_parts(path=path, under=under)` under the resolved `root`. +pub(crate) fn relative_parts(root: &Path, path: &str, under: &str) -> Result, Raised> { + let relative: PathBuf = PathBuf::from(pure_path_text(Path::new(path))); + let under_path: Option = + (!under.is_empty()).then(|| PathBuf::from(pure_path_text(Path::new(under)))); + match project_relative_parts(&relative, root, under_path.as_deref(), resolve_project_path) { + Ok(parts) => Ok(parts.into_iter().map(PathBuf::from).collect()), + Err(RelativePartsError::Resolve(_) | RelativePartsError::Escapes) => { + Err(Raised::usage(format!("{ESCAPES}{path}"))) + } + Err(RelativePartsError::NotBeneath) => Err(Raised::usage(format!( + "project path {path} is not beneath {under}" + ))), + } +} + +/// `ProjectTree.read_text(path)` under the resolved `root`. +pub(crate) fn read_text(root: &Path, path: &str) -> Result { + read_project_text(root, Path::new(path)).map_err(|error| text_error(&error, path)) +} + +/// The exception `ProjectTree.read_text(path)` raises for `error`. +pub(crate) fn text_error(error: &ProjectTextError, path: &str) -> Raised { + match error { + ProjectTextError::File(error) => file_error(error, path), + ProjectTextError::Read { path: file, errno } => Raised::usage(format!( + "{UNREADABLE}{path}: {}", + os_error_text(*errno, &file.to_string_lossy()) + )), + ProjectTextError::Decode(failure) => { + Raised::usage(format!("{UNREADABLE}{path}: {}", decode_text(failure))) + } + } +} + +/// The exception `project_tree_file_path(root, path)` raises for `error`. +pub(crate) fn file_error(error: &ProjectTreeFileError, path: &str) -> Raised { + let prefix: String = match error { + ProjectTreeFileError::Escapes => ESCAPES.to_owned(), + ProjectTreeFileError::EmbeddedNul => { + return Raised::new(VALUE_ERROR, EMBEDDED_NUL_MESSAGE.to_owned()); + } + ProjectTreeFileError::Missing => MISSING.to_owned(), + ProjectTreeFileError::Untracked => format!( + "compiler rules input must be a tracked file type ({}): ", + tracked_text_suffixes().join(", ") + ), + ProjectTreeFileError::Symlink => SYMLINK.to_owned(), + ProjectTreeFileError::Unreadable { path: file, errno } => { + return Raised::new( + os_error_class(*errno), + os_error_text(*errno, &file.to_string_lossy()), + ); + } + }; + Raised::usage(format!("{prefix}{path}")) +} + +/// The `OSError` subclass Python raises for `errno`. +pub(crate) fn os_error_class(errno: i32) -> &'static str { + let kind: ErrorKind = std::io::Error::from_raw_os_error(errno).kind(); + ERRNO_CLASSES + .iter() + .find(|(code, _)| *code == kind) + .map_or(OS_ERROR, |(_, class)| class) +} + +/// `os.strerror(errno)`. +pub(crate) fn strerror(errno: i32) -> String { + let text: String = std::io::Error::from_raw_os_error(errno).to_string(); + text.strip_suffix(&format!(" (os error {errno})")) + .unwrap_or(&text) + .to_owned() +} + +/// `str(OSError(errno, os.strerror(errno), file))`. +fn os_error_text(errno: i32, file: &str) -> String { + format!( + "[Errno {errno}] {}: {}", + strerror(errno), + python_str_repr(file) + ) +} + +/// `str(UnicodeDecodeError)` of Python's strict UTF-8 decoder. +fn decode_text(failure: &Utf8Failure) -> String { + let reason: &str = match failure.reason { + Utf8Reason::InvalidStart => "invalid start byte", + Utf8Reason::InvalidContinuation => "invalid continuation byte", + Utf8Reason::UnexpectedEnd => "unexpected end of data", + }; + if failure.end - failure.start == 1 { + return format!( + "'utf-8' codec can't decode byte 0x{:02x} in position {}: {reason}", + failure.byte, failure.start + ); + } + format!( + "'utf-8' codec can't decode bytes in position {}-{}: {reason}", + failure.start, + failure.end - 1 + ) +} + +/// A path as a JSON string; undecodable paths hold surrogates `orjson` refuses. +pub(crate) fn path_string(path: &Path) -> Encoded { + match path.to_str() { + Some(text) => Ok(JsonValue::String(text.to_owned())), + None => Err(FactFailure::Unreproducible), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 70589cf566..d8217b0402 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -4,14 +4,15 @@ use std::collections::HashMap; use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyStringMethods; use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_project::model_input_view::{ ExpansionRecord, ModelInputRecord, NativeModelInput, model_relation_key_row, }; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; -use crate::bindings::_helpers::rules::lint_pipeline::{CompiledExpansion, span_pass}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + CompiledExpansion, CompiledSource, span_pass, +}; use crate::bindings::_helpers::rules::relation_keys::RelationKeyRow; const FILE_PATH_ATTRIBUTE: &str = "file_path"; @@ -20,10 +21,12 @@ const FILE_PATH_ATTRIBUTE: &str = "file_path"; pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = model.py(); if let Ok(input) = model.downcast::() { - return input.get().read(py, model_relation_key_row); + return input.get().read_with_owner(py, |owner, record| { + model_relation_key_row(owner, record, py) + }); } if let Ok(compiled) = model.downcast::() { - return compiled.get().read(py, model_relation_key_row); + return compiled.get().relation_key_row(py); } model.getattr("relation_key_row")?.extract() } @@ -37,23 +40,23 @@ fn record_path(record: &ModelInputRecord, py: Python<'_>) -> PyResult { .extract() } -/// An expansion record with the given passes. +/// The record's shared texts with the given passes, read only when their file is linted. fn expansion( expansion: &ExpansionRecord, passes: &[&Vec<(usize, usize, usize, usize)>], py: Python<'_>, -) -> PyResult { - Ok(CompiledExpansion { - authored_sql: expansion.authored_sql.bind(py).to_str()?.to_owned(), - expanded_sql: expansion.expanded_sql.bind(py).to_str()?.to_owned(), +) -> CompiledSource { + CompiledSource::Shared { + authored_sql: expansion.authored_sql.clone_ref(py), + expanded_sql: expansion.expanded_sql.clone_ref(py), passes: passes.iter().map(|rows| span_pass(rows)).collect(), - }) + } } /// A model input's whole compile expansion, variables included, keyed by its file path. pub(crate) fn model_lint_expansion( model: &Bound<'_, PyAny>, -) -> PyResult> { +) -> PyResult> { let py: Python<'_> = model.py(); if let Ok(input) = model.downcast::() { return input.get().read(py, |record, py| { @@ -68,7 +71,7 @@ pub(crate) fn model_lint_expansion( &whole.macro_spans, ], py, - )?, + ), ))) }); } @@ -83,15 +86,18 @@ pub(crate) fn model_lint_expansion( .getattr("model_file")? .getattr(FILE_PATH_ATTRIBUTE)? .extract()?; - Ok(Some((path, CompiledExpansion::from_record(&record)?))) + Ok(Some(( + path, + CompiledSource::Owned(CompiledExpansion::from_record(&record)?), + ))) } /// A compiled project's authored expansions: models whose interpolation left the query unchanged. pub(crate) fn project_sql_expansions( project: &Bound<'_, PyAny>, -) -> PyResult> { +) -> PyResult> { let py: Python<'_> = project.py(); - let mut expansions: HashMap = HashMap::new(); + let mut expansions: HashMap = HashMap::new(); if let Ok(view) = project.downcast::() { let owner = view.get().owner().bind(py).borrow(); for record in owner.inputs() { @@ -103,7 +109,7 @@ pub(crate) fn project_sql_expansions( authored, &[&authored.declaration_spans, &authored.macro_spans], py, - )?, + ), ); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index e4a14efaad..0b7114cd39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -5,8 +5,8 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::text::main::python_isspace::python_isspace; use sqlbuild_core::text::main::python_str_repr::python_str_repr; @@ -171,6 +171,42 @@ impl CompiledExpansion { } } +/// A compile's expansion of one file: owned, or the compile's shared texts read when linted. +pub(crate) enum CompiledSource { + Owned(CompiledExpansion), + Shared { + authored_sql: Py, + expanded_sql: Py, + passes: Vec>, + }, +} + +impl CompiledSource { + /// The owned expansion, readable without the GIL. + fn owned(&self) -> Option<&CompiledExpansion> { + match self { + Self::Owned(expansion) => Some(expansion), + Self::Shared { .. } => None, + } + } + + /// The shared texts as an expansion of their own, built only while one file is linted. + fn shared(&self, py: Python<'_>) -> PyResult> { + match self { + Self::Owned(_) => Ok(None), + Self::Shared { + authored_sql, + expanded_sql, + passes, + } => Ok(Some(CompiledExpansion { + authored_sql: authored_sql.bind(py).to_str()?.to_owned(), + expanded_sql: expanded_sql.bind(py).to_str()?.to_owned(), + passes: passes.clone(), + })), + } + } +} + /// Expansion span rows as lint spans. pub(crate) fn span_pass(rows: &[SpanRow]) -> Vec { rows.iter() @@ -402,7 +438,7 @@ pub(crate) struct LintRequest<'a> { pub(crate) settings: &'a LintSettings, /// Resolved paths of models whose output columns a dynamic `*` proves. pub(crate) dynamic_output_paths: &'a HashSet, - pub(crate) compiled_expansions: &'a HashMap, + pub(crate) compiled_expansions: &'a HashMap, /// The expansion inputs; without them bodies are not expanded or SQL-linted. pub(crate) context: Option<&'a SqlExpansionInputs>, pub(crate) skip_unexpandable: bool, @@ -478,9 +514,12 @@ struct BodyTraits { struct FilePlan { file: usize, header_violations: Vec, - expansion: Option), HeaderFailure>>, + expansion: Option, } +/// A file's body traits and planned bodies, or why its headers could not be read. +type PlannedBodies = Result<(BodyTraits, Vec), HeaderFailure>; + /// Why a file could not be planned. enum PlanFailure { OutsideProject { path: PathBuf }, @@ -551,6 +590,14 @@ pub(crate) fn lint( } }; let file: &LintFile = &files[plan.file]; + let (traits, bodies) = match shared_plan(py, request, file)? { + Some(Ok(planned)) => planned, + Some(Err(failure)) => { + scan_failure = Some(failure.into_pyerr(py)); + break; + } + None => (traits, bodies), + }; let expanded: ExpandedFile = match expand_bodies(py, context, file, (plan.file, traits, bodies)) { Ok(expanded) => expanded, @@ -593,6 +640,32 @@ pub(crate) fn lint( }) } +/// A file's bodies planned again against its shared compile expansion, read only for this file. +fn shared_plan( + py: Python<'_>, + request: &LintRequest<'_>, + file: &LintFile, +) -> PyResult> { + let Some(source) = request.compiled_expansions.get(&file.path) else { + return Ok(None); + }; + let Some(compiled) = source.shared(py)? else { + return Ok(None); + }; + let Ok(relative) = file.path.strip_prefix(request.project_dir) else { + return Ok(None); + }; + let parts: Vec = path_parts(relative); + let text: Vec = file.contents.chars().collect(); + let headers: Vec = scan_headers(&text, first_header_only(&parts)); + Ok(Some(plan_bodies( + request, + file, + (&text, &headers), + (&parts, Some(&compiled)), + ))) +} + /// Each file's headers, header violations and bodies in path order, up to the first failure. fn plan_files( request: &LintRequest<'_>, @@ -633,7 +706,12 @@ fn plan_file( } else { Vec::new() }; - let expansion = expanding.then(|| plan_bodies(request, file, (&text, &headers), &parts)); + let compiled: Option<&CompiledExpansion> = request + .compiled_expansions + .get(&file.path) + .and_then(CompiledSource::owned); + let expansion = + expanding.then(|| plan_bodies(request, file, (&text, &headers), (&parts, compiled))); Ok(FilePlan { file: index, header_violations, @@ -646,13 +724,12 @@ fn plan_bodies( request: &LintRequest<'_>, file: &LintFile, (text, headers): (&[char], &[HeaderSpan]), - parts: &[String], + (parts, compiled): (&[String], Option<&CompiledExpansion>), ) -> Result<(BodyTraits, Vec), HeaderFailure> { let kind: Option<&str> = named_declaration_role_kind(parts); let in_hook_directory: bool = parts.first().is_some_and(|part| part == HOOKS_FOLDER); let dynamic_star: bool = !request.dynamic_output_paths.is_empty() && request.dynamic_output_paths.contains(&resolved(&file.path)); - let compiled: Option<&CompiledExpansion> = request.compiled_expansions.get(&file.path); let external: Vec = external_identifiers(text, headers, &NativeHeaderParser)?; let ceremonial: bool = headers .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 9232d48281..45f26037a7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -9,6 +9,9 @@ pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; pub(crate) mod early_sql_lint; pub(crate) mod evaluation; +pub(crate) mod fact_encoder; +pub(crate) mod fact_replay; +pub(crate) mod fact_tree; pub(crate) mod findings; pub(crate) mod fix_facts; pub(crate) mod fix_session; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs index 4fceef9822..7538f41b7c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/project_tree_queries.rs @@ -1,20 +1,25 @@ //! Project-tree queries custom rules run over the discovered paths. use std::ffi::OsString; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use crate::bindings::_helpers::rules::fact_tree::{Raised, VALUE_ERROR, strerror, text_error}; +use pyo3::exceptions::PyOSError; use pyo3::exceptions::PyValueError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyString, PyTuple}; +use pyo3::types::{PyString, PyStringMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; -use sqlbuild_rules::project_tree::errors::RelativePartsError; +use sqlbuild_rules::project_tree::errors::{ + ProjectTextError, ProjectTreeFileError, RelativePartsError, +}; use sqlbuild_rules::project_tree::main::project_relative_parts::project_relative_parts; use sqlbuild_rules::project_tree::main::project_tree_children::project_tree_children; use sqlbuild_rules::project_tree::main::project_tree_descendants::project_tree_descendants; use sqlbuild_rules::project_tree::main::project_tree_glob::project_tree_glob; -use sqlbuild_rules::project_tree::main::tracked_text_path::tracked_text_path; +use sqlbuild_rules::project_tree::main::read_project_text::read_project_text; +use sqlbuild_rules::project_tree::main::resolve_project_path::resolve_project_path; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::records::ids::ErrorId; @@ -97,23 +102,6 @@ fn project_tree_glob_nodes<'py>( selected(py, &nodes, project_tree_glob(&values, &pattern)) } -/// The values of the file nodes rules may read as tracked text, in order. -#[pyfunction] -fn tracked_text_file_values<'py>(nodes: &Bound<'py, PyAny>) -> PyResult> { - let mut values: Vec> = Vec::new(); - for node in nodes.try_iter()? { - let node: Bound<'py, PyAny> = node?; - if !node.getattr("is_file")?.is_truthy()? { - continue; - } - let value: Bound<'py, PyAny> = node.getattr("value")?; - if tracked_text_path(&value.extract::()?) { - values.push(value); - } - } - PyTuple::new(nodes.py(), values) -} - /// `RuleUsageError(f"project path {path} is not beneath {under}")`, spelled by Python. fn not_beneath(path: &Bound<'_, PyAny>, under: &Bound<'_, PyAny>) -> PyErr { let py: Python<'_> = path.py(); @@ -152,12 +140,9 @@ fn project_tree_relative_parts<'py>( &relative.extract::()?, &root.extract::()?, under_path.as_deref(), - |_| relative.call_method0("resolve")?.extract::(), + resolve_project_path, ) { Ok(parts) => parts, - Err(RelativePartsError::Resolve(error)) if !error.is_instance_of::(py) => { - return Err(error); - } Err(RelativePartsError::Resolve(_) | RelativePartsError::Escapes) => { return Err(usage_error_for("project path escapes the project: ", path)); } @@ -170,11 +155,79 @@ fn project_tree_relative_parts<'py>( PyTuple::new(py, parts) } +/// The UTF-8 text of the tracked project file at `path` under the resolved `root`. +#[pyfunction] +fn project_tree_read_text(root: PathBuf, path: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = path.py(); + let text: String = path.str()?.to_string_lossy().into_owned(); + let raised: Raised = match read_project_text(&root, Path::new(&text)) { + Ok(contents) => return Ok(contents), + Err(ProjectTextError::File(ProjectTreeFileError::Unreadable { path, errno })) => { + return Err(unreadable(py, path, errno)); + } + #[cfg(not(unix))] + Err(ProjectTextError::Read { path: file, .. }) => { + return Err(python_read_error(py, &file, &text)); + } + Err(error) => text_error(&error, &text), + }; + if raised.kind == VALUE_ERROR { + return Err(PyValueError::new_err(raised.message)); + } + Err(PyErr::from_value( + registry::error(py, ErrorId::RuleUsage)?.call1((raised.message,))?, + )) +} + +/// The `OSError` `Path.is_file()` raises for a stat failure it does not ignore. +#[cfg(unix)] +fn unreadable(py: Python<'_>, path: PathBuf, errno: i32) -> PyErr { + let Ok(path) = path.into_os_string().into_pyobject(py); + PyOSError::new_err((errno, strerror(errno), path.unbind())) +} + +/// The `OSError` `Path.is_file()` raises for a Windows stat failure it does not ignore. +#[cfg(not(unix))] +fn unreadable(py: Python<'_>, path: PathBuf, code: i32) -> PyErr { + let detail: String = strerror(code); + let detail: &str = + detail.trim_end_matches(|character: char| character <= ' ' || character == '.'); + let Ok(path) = path.into_os_string().into_pyobject(py); + PyOSError::new_err((code, detail.to_owned(), path.unbind(), code)) +} + +/// The usage error for a Windows read failure, worded by Python's own `Path.read_text`. +#[cfg(not(unix))] +fn python_read_error(py: Python<'_>, file: &Path, text: &str) -> PyErr { + let read = || -> PyResult { + let reading = imported(py, "pathlib", "Path")? + .call1((file.as_os_str(),))? + .call_method1("read_text", ("utf-8",)); + let error: PyErr = match reading { + Ok(_) => { + return Ok(PyOSError::new_err(format!( + "could not read project file {text}" + ))); + } + Err(error) => error, + }; + let message: String = format!( + "could not read project file {text}: {}", + error.value(py).str()? + ); + let usage: PyErr = + PyErr::from_value(registry::error(py, ErrorId::RuleUsage)?.call1((message,))?); + usage.set_cause(py, Some(error)); + Ok(usage) + }; + read().unwrap_or_else(|error| error) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(project_tree_child_nodes, module)?)?; module.add_function(wrap_pyfunction!(project_tree_descendant_nodes, module)?)?; module.add_function(wrap_pyfunction!(project_tree_glob_nodes, module)?)?; - module.add_function(wrap_pyfunction!(tracked_text_file_values, module)?)?; module.add_function(wrap_pyfunction!(project_tree_relative_parts, module)?)?; + module.add_function(wrap_pyfunction!(project_tree_read_text, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index 52ec1a67af..a47b614944 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -50,6 +50,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { results::register(module)?; guidance::register(module)?; crate::bindings::_helpers::rules::custom_evaluation::register(module)?; + crate::bindings::_helpers::rules::fact_encoder::register(module)?; crate::bindings::_helpers::rules::custom_host_observation::register(module)?; crate::bindings::_helpers::rules::rule_dispatch::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index 1bacb85af8..aacff66cec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -3,11 +3,13 @@ use std::collections::BTreeMap; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation_keys as derive_keys; use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; +use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::rules::lint_inputs::model_key_row; const MATERIALIZED_CONFIG: &str = "materialized"; @@ -37,18 +39,16 @@ pub(crate) fn relation_key_row( kind: &str, name: String, values: &Bound<'_, PyDict>, - declared: Option<(&Bound<'_, PyAny>, &Bound<'_, PyAny>)>, + declared: Option<(&[SchemaColumnFact], &Bound<'_, PyAny>)>, ) -> PyResult { let mut columns: Vec<(String, Vec, bool)> = Vec::new(); let mut audits: Vec<(String, Option)> = Vec::new(); if let Some((declared_columns, declared_audits)) = declared { - for column in declared_columns.try_iter()? { - let column: Bound<'_, PyAny> = column?; - let nullable: Option> = column.getattr_opt("nullable")?; + for column in declared_columns { columns.push(( - column.getattr("name")?.str()?.extract()?, - audit_names(column.getattr_opt("audits")?)?, - nullable.is_some_and(|value| value.is(PyBool::new(column.py(), false))), + column.name.clone(), + column.audit_definitions.clone(), + column.nullable == Some(false), )); } for audit in declared_audits.try_iter()? { @@ -81,17 +81,6 @@ pub(crate) fn relation_key_row( )) } -fn audit_names(audits: Option>) -> PyResult> { - let Some(audits) = audits.filter(|audits| !audits.is_none()) else { - return Ok(Vec::new()); - }; - let mut names: Vec = Vec::new(); - for audit in audits.try_iter()? { - names.push(audit?.getattr("definition_name")?.extract()?); - } - Ok(names) -} - /// `str(values.get(key, ""))`. fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult { match values.get_item(key)? { @@ -134,7 +123,8 @@ pub(crate) fn entry_relation_key_row( if entry.is_none() { return relation_key_row(kind, name, values, None); } - let columns: Bound<'_, PyAny> = declared_specs(entry, "columns")?; + let columns: Vec = + python_column_facts(&declared_specs(entry, "columns")?, None)?; let audits: Bound<'_, PyAny> = declared_specs(entry, "audits")?; relation_key_row(kind, name, values, Some((&columns, &audits))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index b922e1e5ab..0197843b48 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -34,6 +34,7 @@ use crate::bindings::_helpers::compiled_project::records::declaration_rows; use crate::bindings::_helpers::compiled_project::values::plain_value; use crate::bindings::_helpers::records::ids::{NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::fact_replay::project_models; use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; @@ -166,10 +167,10 @@ fn built_request(options: &Request<'_, '_>) -> PyResult { registry::native(py, NativeAttr::CompiledProjectFacts)? .call1((&project,))? .downcast_into()?; - let mut names: Vec = Vec::new(); - for model in project.getattr("models")?.try_iter()? { - names.push(model?.getattr("name")?.extract()?); - } + let names: Vec = project_models(&project)? + .into_iter() + .map(|model| model.name) + .collect(); let sql_tests: Vec> = project .getattr("sql_tests")? .try_iter()? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs index 04c5f89a14..0366b908fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_api.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; -use pyo3::exceptions::{PyAttributeError, PyOSError, PyValueError}; +use pyo3::exceptions::PyAttributeError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -12,11 +12,8 @@ use pyo3::types::{ use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_config::toml::main::toml_literal::toml_literal; use sqlbuild_config::toml::models::TomlLiteral; -use sqlbuild_rules::project_tree::errors::ProjectTreeFileError; use sqlbuild_rules::project_tree::main::project_path_escapes::project_path_escapes; use sqlbuild_rules::project_tree::main::project_tree_entries::project_tree_entries; -use sqlbuild_rules::project_tree::main::project_tree_file::project_tree_file; -use sqlbuild_rules::project_tree::main::tracked_text_suffixes::tracked_text_suffixes; use sqlbuild_rules::project_tree::models::ProjectTreeEntry; use crate::bindings::_helpers::compile_driver::python_calls::imported; @@ -25,10 +22,6 @@ use crate::bindings::_helpers::records::registry; const BUILTINS_MODULE: &str = "builtins"; const DEFAULT_POSITION: i64 = 1; -#[cfg(unix)] -const OS_MODULE: &str = "os"; -/// The error `os.lstat` raises for a path holding a NUL byte. -const EMBEDDED_NUL_MESSAGE: &str = "lstat: embedded null character in path"; /// `RuleUsageError(message)`. fn usage_error(py: Python<'_>, message: String) -> PyErr { @@ -154,33 +147,6 @@ fn escapes(path: &Bound<'_, PyAny>) -> PyResult { ))) } -/// The `OSError` `Path.is_file()` raises for a stat failure it does not ignore. -#[cfg(unix)] -fn unreadable(py: Python<'_>, path: PathBuf, errno: i32) -> PyResult { - let message: String = imported(py, OS_MODULE, "strerror")? - .call1((errno,))? - .extract()?; - let Ok(path) = path.into_os_string().into_pyobject(py); - Ok(PyOSError::new_err((errno, message, path.unbind()))) -} - -/// The `OSError` `Path.is_file()` raises for a Windows stat failure it does not ignore. -#[cfg(not(unix))] -fn unreadable(py: Python<'_>, path: PathBuf, code: i32) -> PyResult { - let detail: String = std::io::Error::from_raw_os_error(code).to_string(); - let detail: &str = detail - .strip_suffix(&format!(" (os error {code})")) - .unwrap_or(&detail) - .trim_end_matches(|character: char| character <= ' ' || character == '.'); - let Ok(path) = path.into_os_string().into_pyobject(py); - Ok(PyOSError::new_err(( - code, - detail.to_owned(), - path.unbind(), - code, - ))) -} - /// Every supported path under the resolved project `root`, as sorted `ProjectPath`s. #[pyfunction] fn project_tree_paths<'py>(root: &Bound<'py, PyAny>) -> PyResult> { @@ -219,38 +185,10 @@ fn project_tree_paths<'py>(root: &Bound<'py, PyAny>) -> PyResult( - root: &Bound<'py, PyAny>, - path: &Bound<'py, PyAny>, -) -> PyResult> { - let py: Python<'py> = root.py(); - let relative: PathBuf = path.str()?.extract()?; - let prefix: String = match project_tree_file(&root.extract::()?, &relative) { - Ok(file) => return Ok(file.into_os_string().into_pyobject(py)?.into_any()), - Err(ProjectTreeFileError::Escapes) => "project path escapes the project: ".to_owned(), - Err(ProjectTreeFileError::EmbeddedNul) => { - return Err(PyValueError::new_err(EMBEDDED_NUL_MESSAGE)); - } - Err(ProjectTreeFileError::Missing) => "project file does not exist: ".to_owned(), - Err(ProjectTreeFileError::Untracked) => format!( - "compiler rules input must be a tracked file type ({}): ", - tracked_text_suffixes().join(", ") - ), - Err(ProjectTreeFileError::Symlink) => "project file must not be a symlink: ".to_owned(), - Err(ProjectTreeFileError::Unreadable { path, errno }) => { - return Err(unreadable(py, path, errno)?); - } - }; - Err(usage_error_for(&prefix, path)) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(finding_position, module)?)?; module.add_function(wrap_pyfunction!(rule_option_toml_value, module)?)?; module.add_function(wrap_pyfunction!(validate_project_relative, module)?)?; module.add_function(wrap_pyfunction!(project_tree_paths, module)?)?; - module.add_function(wrap_pyfunction!(project_tree_file_path, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index a46162f049..9633425702 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -15,6 +15,7 @@ use crate::bindings::_helpers::rules::catalogue::{ verify_custom_rules, }; use crate::bindings::_helpers::rules::constants::TYPE_PROOF_RULE_CODES; +use crate::bindings::_helpers::rules::fact_replay::project_models; use crate::bindings::_helpers::rules::rule_run::freed_memory_released; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; @@ -140,10 +141,9 @@ fn selected_model_paths<'py>( } } let paths: Bound<'py, PySet> = PySet::empty(py)?; - for model in graph.getattr("project")?.getattr("models")?.try_iter()? { - let model: Bound<'py, PyAny> = model?; - if names.contains(model.getattr("name")?)? { - paths.add(model.getattr("relative_path")?.call_method0("as_posix")?)?; + for model in project_models(&graph.getattr("project")?)? { + if names.contains(&model.name)? { + paths.add(model.posix)?; } } Ok(Some(PyFrozenSet::new(py, paths.iter())?)) @@ -156,10 +156,9 @@ fn selected_models_project<'py>( ) -> PyResult> { let py: Python<'py> = project.py(); let mut models: Vec> = Vec::new(); - for model in project.getattr("models")?.try_iter()? { - let model: Bound<'py, PyAny> = model?; - if model_paths.contains(model.getattr("relative_path")?.call_method0("as_posix")?)? { - models.push(model); + for entry in project_models(project)? { + if model_paths.contains(&entry.posix)? { + models.push(entry.model); } } call_with( @@ -183,8 +182,8 @@ fn selection_rules_config<'py>( }; let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; let model_names: Bound<'py, PySet> = PySet::empty(py)?; - for model in project.getattr("models")?.try_iter()? { - model_names.add(model?.getattr("name")?)?; + for model in project_models(project)? { + model_names.add(model.name)?; } let mut rule_exceptions: Vec> = Vec::new(); for entry in config.getattr("rule_exceptions")?.try_iter()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index ba678efecb..230c255033 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -14,6 +14,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::records::registry; +use crate::bindings::_helpers::rules::fact_replay::project_models; use crate::bindings::_helpers::rules::results::rule_finding; use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; @@ -80,12 +81,8 @@ fn finish_rule_run<'py>( .map(|finding| finding.path.clone()) .collect(); let mut unevaluated_models: i64 = 0; - for model in plan.getattr("project")?.getattr("models")?.try_iter()? { - let path: String = model? - .getattr("relative_path")? - .call_method0("as_posix")? - .extract()?; - unevaluated_models += i64::from(unevaluated.contains(&path)); + for model in project_models(&plan.getattr("project")?)? { + unevaluated_models += i64::from(unevaluated.contains(&model.posix)); } let findings: Vec> = grouped .into_iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs index 2a46abc916..9885fb6da7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -15,7 +15,8 @@ use crate::bindings::_helpers::rules::lint_expansion::{ LintExpansionSources, lint_expansion_inputs, }; use crate::bindings::_helpers::rules::lint_pipeline::{ - CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, + CompiledExpansion, CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, lint, + lint_sources, }; /// The selected paths, or `None` for every file. @@ -51,14 +52,17 @@ pub(crate) fn given_files(files: &Bound<'_, PyAny>) -> PyResult, -) -> PyResult> { - let mut compiled: HashMap = HashMap::new(); +) -> PyResult> { + let mut compiled: HashMap = HashMap::new(); if expansions.is_none() { return Ok(compiled); } for (path, expansion) in expansions.downcast::()?.iter() { if !expansion.is_none() { - compiled.insert(path.extract()?, CompiledExpansion::from_record(&expansion)?); + compiled.insert( + path.extract()?, + CompiledSource::Owned(CompiledExpansion::from_record(&expansion)?), + ); } } Ok(compiled) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index a98ccc8576..3fe77192ba 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -30,7 +30,7 @@ use crate::bindings::_helpers::rules::lint_expansion::{ }; use crate::bindings::_helpers::rules::lint_inputs::project_sql_expansions; use crate::bindings::_helpers::rules::lint_pipeline::{ - CompiledExpansion, LintFile, LintOutcome, LintRequest, LintSettings, LintViolationRow, lint, + CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, LintViolationRow, lint, lint_sources, outside_project, resolved, }; use crate::bindings::_helpers::rules::relation_keys::project_relation_keys; @@ -312,7 +312,7 @@ fn evaluated_sql_rules( .collect(); let mut file_identities: Vec<(String, String)> = Vec::new(); let mut project_files: Option> = None; - let compiled: HashMap = project_sql_expansions(&run.project)?; + let compiled: HashMap = project_sql_expansions(&run.project)?; if !run.model_selection { let mut model_paths: HashSet = HashSet::new(); for model in run.project.getattr("models")?.try_iter()? { @@ -434,7 +434,7 @@ fn file_lines(py: Python<'_>, project_dir: &Path, path: &str) -> PyResult, (selected_paths, dynamic_output_paths): (&BTreeSet, &HashSet), - compiled: &HashMap, + compiled: &HashMap, source_files: Option>, ) -> PyResult> { let py: Python<'_> = run.project.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 8cfb954000..da04b93276 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -4,7 +4,9 @@ use std::sync::Arc; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple, PyTupleMethods, +}; use sqlbuild_analysis::semantic_checks::main::check_semantic_metadata::check_semantic_metadata; use sqlbuild_analysis::semantic_checks::models::{ AuditValue, MetadataAudit, MetadataFinding, MetadataFunction, MetadataModel, MetadataRequest, @@ -14,6 +16,7 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis::model_reads::binding_validated; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; use crate::bindings::_helpers::semantic_checks::inputs::{ column_names, resource_sql_diagnostics, sql_test_columns, @@ -471,20 +474,40 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes } fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { - let entry: Bound<'_, PyAny> = model.getattr("schema_entry")?; - if entry.is_none() { - return Ok(Vec::new()); - } let py: Python<'_> = model.py(); let mut attached: Vec<(Option, Bound<'_, PyAny>)> = Vec::new(); - for audit in entry.getattr("audits")?.try_iter()? { - attached.push((None, audit?)); - } - for column in entry.getattr("columns")?.try_iter()? { - let column: Bound<'_, PyAny> = column?; - let name: String = column.getattr("name")?.extract()?; - for audit in column.getattr("audits")?.try_iter()? { - attached.push((Some(name.clone()), audit?)); + match model.downcast::() { + Ok(view) => { + let Some((entry_audits, columns, audited)) = + view.get().schema_audits_and_columns(py)? + else { + return Ok(Vec::new()); + }; + for audit in entry_audits.bind(py).iter() { + attached.push((None, audit)); + } + for (index, name) in audited { + let column: Bound<'_, PyAny> = columns.bind(py).get_item(index)?; + for audit in column.getattr("audits")?.try_iter()? { + attached.push((Some(name.clone()), audit?)); + } + } + } + Err(_) => { + let entry: Bound<'_, PyAny> = model.getattr("schema_entry")?; + if entry.is_none() { + return Ok(Vec::new()); + } + for audit in entry.getattr("audits")?.try_iter()? { + attached.push((None, audit?)); + } + for column in entry.getattr("columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let name: String = column.getattr("name")?.extract()?; + for audit in column.getattr("audits")?.try_iter()? { + attached.push((Some(name.clone()), audit?)); + } + } } } let empty: Bound<'_, PyAny> = PyString::new(py, "").into_any(); diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index 0ed0d546a9..ece8718b45 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -63,7 +63,7 @@ fn schema_row(schema: &ModelSchemaFact) -> SchemaRow { name: column.name.clone(), data_type: column.data_type.clone(), nullable: column.nullable, - audit_count: column.audit_count, + audit_count: u32::try_from(column.audit_definitions.len()).unwrap_or(u32::MAX), }) .collect(), dynamic_columns: schema diff --git a/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs index ad40bd0d59..63b1a9e8e8 100644 --- a/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod entries; pub(crate) mod files; +pub(crate) mod project_tree_file; pub(crate) mod relative_paths; diff --git a/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs b/crates/sqlbuild-rules/src/project_tree/_helpers/project_tree_file.rs similarity index 92% rename from crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs rename to crates/sqlbuild-rules/src/project_tree/_helpers/project_tree_file.rs index 3f76fe8cc9..5994d8927a 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/project_tree_file.rs +++ b/crates/sqlbuild-rules/src/project_tree/_helpers/project_tree_file.rs @@ -10,7 +10,7 @@ use crate::project_tree::errors::ProjectTreeFileError; use crate::project_tree::models::PureParts; /// The tracked file inside the resolved `root` that the project-relative `path` names. -pub fn project_tree_file(root: &Path, path: &Path) -> Result { +pub(crate) fn project_tree_file(root: &Path, path: &Path) -> Result { if escapes(path) { return Err(ProjectTreeFileError::Escapes); } diff --git a/crates/sqlbuild-rules/src/project_tree/constants.rs b/crates/sqlbuild-rules/src/project_tree/constants.rs index 7759efbbb0..f42f47b436 100644 --- a/crates/sqlbuild-rules/src/project_tree/constants.rs +++ b/crates/sqlbuild-rules/src/project_tree/constants.rs @@ -25,3 +25,8 @@ pub(crate) const CURRENT: &str = "."; pub(crate) const PARENT: &str = ".."; /// Private-use characters standing for undecodable path bytes or unpaired surrogates. pub(crate) const ESCAPED_BASE: u32 = 0xF_0000; +/// The `ValueError` text `os.fspath` paths raise for a NUL byte. +pub(crate) const EMBEDDED_NUL_BYTE: &str = "embedded null byte"; +pub(crate) const LINE_FEED: &str = "\n"; +/// Bytes that can never start a UTF-8 sequence. +pub(crate) const UTF8_INVALID_START: [std::ops::RangeInclusive; 2] = [0x80..=0xC1, 0xF5..=0xFF]; diff --git a/crates/sqlbuild-rules/src/project_tree/errors.rs b/crates/sqlbuild-rules/src/project_tree/errors.rs index 94b63825bd..7d570f3d96 100644 --- a/crates/sqlbuild-rules/src/project_tree/errors.rs +++ b/crates/sqlbuild-rules/src/project_tree/errors.rs @@ -29,3 +29,31 @@ pub enum RelativePartsError { /// The path is not beneath the requested directory. NotBeneath, } + +/// Why a tracked project file's text cannot be read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ProjectTextError { + /// The path names no file rules may read. + File(ProjectTreeFileError), + /// Opening or reading the file at `path` failed with this `errno`. + Read { path: PathBuf, errno: i32 }, + /// The file is not valid UTF-8. + Decode(Utf8Failure), +} + +/// The first invalid UTF-8 sequence, as Python's strict decoder reports it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Utf8Failure { + pub byte: u8, + pub start: usize, + pub end: usize, + pub reason: Utf8Reason, +} + +/// The reason Python's UTF-8 decoder gives. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Utf8Reason { + InvalidStart, + InvalidContinuation, + UnexpectedEnd, +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/mod.rs b/crates/sqlbuild-rules/src/project_tree/main/mod.rs index db27d63ca8..2e0808bc3b 100644 --- a/crates/sqlbuild-rules/src/project_tree/main/mod.rs +++ b/crates/sqlbuild-rules/src/project_tree/main/mod.rs @@ -4,7 +4,10 @@ pub mod project_relative_parts; pub mod project_tree_children; pub mod project_tree_descendants; pub mod project_tree_entries; -pub mod project_tree_file; pub mod project_tree_glob; +pub mod pure_path_posix; +pub mod pure_path_text; +pub mod read_project_text; +pub mod resolve_project_path; pub mod tracked_text_path; pub mod tracked_text_suffixes; diff --git a/crates/sqlbuild-rules/src/project_tree/main/pure_path_posix.rs b/crates/sqlbuild-rules/src/project_tree/main/pure_path_posix.rs new file mode 100644 index 0000000000..ad107d0b28 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/pure_path_posix.rs @@ -0,0 +1,11 @@ +//! `Path(path).as_posix()` as `pathlib` spells it. + +use std::path::Path; + +use crate::project_tree::_helpers::relative_paths::{posix_bytes, pure_parts}; + +/// `Path(path).as_posix()` as encoded bytes. +#[must_use] +pub fn pure_path_posix(path: &Path) -> Vec { + posix_bytes(&pure_parts(path)) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/pure_path_text.rs b/crates/sqlbuild-rules/src/project_tree/main/pure_path_text.rs new file mode 100644 index 0000000000..0a71768cc8 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/pure_path_text.rs @@ -0,0 +1,12 @@ +//! `str(Path(path))` as `pathlib` spells it. + +use std::ffi::OsString; +use std::path::Path; + +use crate::project_tree::_helpers::relative_paths::{pure_parts, pure_text}; + +/// `str(Path(path))`: the anchor and components joined by the platform separator, or `.`. +#[must_use] +pub fn pure_path_text(path: &Path) -> OsString { + pure_text(&pure_parts(path)) +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/read_project_text.rs b/crates/sqlbuild-rules/src/project_tree/main/read_project_text.rs new file mode 100644 index 0000000000..e94232f2d4 --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/read_project_text.rs @@ -0,0 +1,64 @@ +//! The text of a tracked project file as `Path.read_text(encoding="utf-8")` reads it. + +use std::path::{Path, PathBuf}; +use std::str::Utf8Error; + +use crate::project_tree::_helpers::project_tree_file::project_tree_file; +use crate::project_tree::constants::{LINE_FEED, UTF8_INVALID_START}; +use crate::project_tree::errors::{ProjectTextError, Utf8Failure, Utf8Reason}; + +/// The file's UTF-8 text with universal newlines, or why it cannot be read. +pub fn read_project_text(root: &Path, path: &Path) -> Result { + let file: PathBuf = project_tree_file(root, path).map_err(ProjectTextError::File)?; + let bytes: Vec = match std::fs::read(&file) { + Ok(bytes) => bytes, + Err(error) => { + return Err(ProjectTextError::Read { + path: file, + errno: error.raw_os_error().unwrap_or_default(), + }); + } + }; + match String::from_utf8(bytes) { + Ok(text) => Ok(universal_newlines(&text)), + Err(error) => Err(ProjectTextError::Decode(utf8_failure( + error.as_bytes(), + error.utf8_error(), + ))), + } +} + +/// `\r\n` and lone `\r` as `\n`, as text mode reads them. +fn universal_newlines(text: &str) -> String { + if !text.contains('\r') { + return text.to_owned(); + } + text.replace("\r\n", LINE_FEED).replace('\r', LINE_FEED) +} + +/// The byte range and reason Python's strict UTF-8 decoder reports for `error`. +fn utf8_failure(bytes: &[u8], error: Utf8Error) -> Utf8Failure { + let start: usize = error.valid_up_to(); + let first: u8 = bytes[start]; + match error.error_len() { + None => Utf8Failure { + byte: first, + start, + end: bytes.len(), + reason: Utf8Reason::UnexpectedEnd, + }, + Some(length) => Utf8Failure { + byte: first, + start, + end: start + length, + reason: if UTF8_INVALID_START + .iter() + .any(|range| range.contains(&first)) + { + Utf8Reason::InvalidStart + } else { + Utf8Reason::InvalidContinuation + }, + }, + } +} diff --git a/crates/sqlbuild-rules/src/project_tree/main/resolve_project_path.rs b/crates/sqlbuild-rules/src/project_tree/main/resolve_project_path.rs new file mode 100644 index 0000000000..8d157d574d --- /dev/null +++ b/crates/sqlbuild-rules/src/project_tree/main/resolve_project_path.rs @@ -0,0 +1,45 @@ +//! `Path.resolve()`: symbolic links resolved through every existing prefix, the rest kept. + +use std::io::{Error, ErrorKind}; +use std::path::{Component, Path, PathBuf}; + +use crate::project_tree::_helpers::files::real_path; +use crate::project_tree::constants::EMBEDDED_NUL_BYTE; + +/// The absolute path with each existing prefix resolved and `..` applied to what it resolved to. +pub fn resolve_project_path(path: &Path) -> std::io::Result { + if path.as_os_str().as_encoded_bytes().contains(&0) { + return Err(Error::new(ErrorKind::InvalidInput, EMBEDDED_NUL_BYTE)); + } + let absolute: PathBuf = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir()?.join(path) + }; + let mut resolved: PathBuf = PathBuf::new(); + let mut missing: bool = false; + for component in absolute.components() { + match component { + Component::ParentDir => { + resolved.pop(); + } + Component::CurDir => {} + Component::Normal(part) => { + let next: PathBuf = resolved.join(part); + resolved = if missing { + next + } else { + match real_path(&next) { + Ok(real) => real, + Err(_unresolved) => { + missing = true; + next + } + } + }; + } + Component::Prefix(_) | Component::RootDir => resolved.push(component.as_os_str()), + } + } + Ok(resolved) +} diff --git a/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs b/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs index 7ec13537db..f0fa548828 100644 --- a/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs +++ b/crates/sqlbuild-rules/src/project_tree/tests/test_project_tree.rs @@ -1,8 +1,8 @@ use std::path::Path; +use crate::project_tree::_helpers::project_tree_file::project_tree_file; use crate::project_tree::errors::ProjectTreeFileError; use crate::project_tree::main::project_tree_entries::project_tree_entries; -use crate::project_tree::main::project_tree_file::project_tree_file; use crate::project_tree::tests::helpers::linked_project; use crate::project_tree::tests::test_types::{ProjectTreeEntriesTestCase, ProjectTreeFileTestCase}; diff --git a/fensu.toml b/fensu.toml index b4a5217090..cb01221719 100644 --- a/fensu.toml +++ b/fensu.toml @@ -240,6 +240,12 @@ symbols = [ ] reason = "These collector methods preserve dlt's external positional progress protocol." +[[targets.python.rule_exceptions]] +rule = "FFS120" +path = "src/sqlbuild/runtime/user_code/classes/user_code_finder.py" +symbols = ["UserCodeFinder.find_spec"] +reason = "importlib calls meta-path finders positionally as find_spec(fullname, path, target)." + [[targets.python.rule_exceptions]] rule = "FFS120" path = "src/sqlbuild/assets.py" diff --git a/python-boundary.toml b/python-boundary.toml index a2b95848b8..84f3282619 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -127,11 +127,6 @@ path = "rule_engine/_helpers/engine/rule_harness.py" rule = 2 note = "Public custom-rule test harness that writes a temporary project and runs user rules over it; rule-option TOML literals are native (rule_option_toml_value)." -[[stays]] -path = "rule_engine/_helpers/engine/fact_replay.py" -rule = 2 -note = "Rule 2: the fact digest scheme shared with the rule host (tracked_facts.py, custom_host.py); recomputes values through the user-facing fact views and encodes live Python objects with orjson. The readable-file selection is native (tracked_text_file_values)." - [[stays]] path = "rule_engine/main/define.py" rule = 3 @@ -185,7 +180,7 @@ note = "Fact view user rules read through RuleContext; the Python API over nativ [[stays]] path = "rule_engine/classes/project_tree.py" rule = 3 -note = "Project-tree fact view user rules read through RuleContext; the walk, path validation, file resolution and the children, descendants, glob and relative_parts queries are native (project_tree_*), and only the file read and the model lookup stay here." +note = "Project-tree fact view user rules read through RuleContext; the walk, path validation, file resolution and the children, descendants, glob, relative_parts and read_text queries are native (project_tree_*), and only the model lookup stays here." [[stays]] path = "rule_engine/classes/sql_document.py" @@ -197,11 +192,6 @@ path = "rule_engine/classes/fact_reads.py" rule = 2 note = "Records the fact-view reads of one user rule evaluation." -[[stays]] -path = "rule_engine/classes/shared_fact_encoder.py" -rule = 2 -note = "Encodes the Python fact-view values user rules read, for their digests." - [[stays]] path = "runtime/observability" rule = 1 @@ -227,6 +217,11 @@ path = "runtime/execution_history/exceptions.py" rule = 3 note = "Rule 3 shapes: public storage errors re-exported by sqlbuild.execution_history and raised by the native backends." +[[stays]] +path = "runtime/user_code" +rule = 2 +note = "Import machinery that loads project Python from current source, never from cached bytecode." + [[stays]] path = "runtime/execution_limits" rule = 1 @@ -755,7 +750,7 @@ note = "Shape rule: typing protocols, enums and aliases only (CompiledProjectVie [[stays]] path = "rule_engine/types.py" rule = 3 -note = "Shape rule: data protocol and aliases only (RuleContext, RuleCheck, RuleOptionValue, FactKey, CustomRulePlan), no logic; re-exported by rules/__init__.py and consumed by rule_engine/main/define.py, rule_engine/_helpers/host/custom_host.py, rule_engine/_helpers/engine/fact_replay.py and 4 more." +note = "Shape rule: data protocol and aliases only (RuleContext, RuleCheck, RuleOptionValue, FactKey, CustomRulePlan), no logic; re-exported by rules/__init__.py and consumed by rule_engine/main/define.py, rule_engine/_helpers/host/custom_host.py and 4 more." [[stays]] path = "rule_engine/constants.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 3d44ad436d..a5be1ba378 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1882,7 +1882,6 @@ class NativeProjectView: class NativeCompiledProject: def __init__(self) -> None: ... - def record_model(self, model: tuple[object, ...]) -> None: ... def record_model_analysis( self, name: str, @@ -5365,12 +5364,6 @@ def locate_model_binding_diagnostics( class NativeModelAnalysisSession: @property def fact_models(self) -> list[str]: ... - def run( - self, - ) -> tuple[list[tuple[str, list[tuple[str, str]]]], list[str]]: ... - def finish( - self, - ) -> tuple[list[tuple[str, list[tuple[str, str]]]], list[str], list[bool]]: ... def binding_rows(self) -> list[list[_AnalysisDiagnosticRow]]: ... def outcome_rows(self, index: int) -> tuple[Any, Any]: ... def prove_dynamic_contracts( @@ -5381,13 +5374,6 @@ class NativeModelAnalysisSession: @property def cache_stats(self) -> tuple[int, int, int, str | None] | None: ... -def start_model_analysis_session( - catalog: object, - request: tuple[object, ...], - cache: tuple[str, str] | None = None, - adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, -) -> NativeModelAnalysisSession: ... -def analyze_native_model_sql(*, request: Any) -> Any: ... def session_outcome_analysis( *, session: Any, position: int, placeholders: dict[str, str] | None ) -> Any: ... @@ -6716,7 +6702,9 @@ def finding_position( def rule_option_toml_value(value: object) -> str: ... def validate_project_relative(path: str) -> None: ... def project_tree_paths(root: Path) -> tuple[ProjectPath, ...]: ... -def project_tree_file_path(root: Path, path: str) -> str: ... +def project_tree_read_text(root: Path, path: str) -> str: ... +def fact_value_digest(value: object) -> str | None: ... +def fact_error_digest(error: BaseException) -> str | None: ... def project_tree_child_nodes( nodes: Iterable[ProjectPath], path: str | PurePath ) -> tuple[ProjectPath, ...]: ... @@ -6726,7 +6714,6 @@ def project_tree_descendant_nodes( def project_tree_glob_nodes( nodes: Iterable[ProjectPath], pattern: str ) -> tuple[ProjectPath, ...]: ... -def tracked_text_file_values(nodes: Iterable[ProjectPath]) -> tuple[str, ...]: ... def project_tree_relative_parts( *, root: Path, path: str | PurePath, under: str | PurePath ) -> tuple[str, ...]: ... @@ -6781,6 +6768,9 @@ def evaluate_custom_rules( implementation_fingerprints: dict[str, str] | None = None, ) -> CustomRulesOutcome: ... def custom_host_payload(project: CompiledProjectView) -> CompiledProject: ... +def custom_rule_fact_digest( + *, project: CompiledProjectView, project_dir: Path, key: tuple[str, ...] +) -> str | None: ... class CustomHostObserver: def __init__( @@ -9970,48 +9960,6 @@ class ConfigValueTypeError(ValueError): expected: Any actual_type: Any -class NativeModelAnalysisRequest(NativeRecord): - """The Python model analysis's arguments plus the facts dynamic pivot proofs need.""" - - known_functions: tuple[str, ...] - known_types: tuple[str, ...] - model_inputs: tuple[ModelInput, ...] - column_nullability_by_table: dict[str, dict[str, InferredNullability]] - column_types_by_table: dict[str, dict[str, str]] - inference_profile: ExpressionInferenceProfile - allow_compact_analysis: bool - rich_type_inference: bool - analysis_cache: AnalysisCacheContext | None - complete_binding_schemas: dict[str, dict[str, str]] - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] - def __init__( - self, - known_functions: tuple[str, ...], - known_types: tuple[str, ...], - model_inputs: tuple[ModelInput, ...], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - column_types_by_table: dict[str, dict[str, str]], - inference_profile: ExpressionInferenceProfile, - allow_compact_analysis: bool, - rich_type_inference: bool, - analysis_cache: AnalysisCacheContext | None, - complete_binding_schemas: dict[str, dict[str, str]], - dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], - ) -> None: ... - -class NativeModelAnalyses(NativeRecord): - """Each analysed model's position in the finished session that keeps its outcome.""" - - positions: dict[str, int] - succeeded: frozenset[str] - session: Any | None - def __init__( - self, - positions: dict[str, int], - succeeded: frozenset[str], - session: Any | None, - ) -> None: ... - class NativePivotTables(NativeRecord): """The relation facts Python's dynamic pivot proofs read, for proofs outside a session.""" @@ -16345,9 +16293,6 @@ class NonHermeticRuleError(RulesError): class RuleAssertionError(RulesError): """Raised when a custom-rule harness case produces an unexpected result.""" -class FactDigestError(RulesError): - """Raised when a recorded custom-rule fact cannot be recomputed or encoded deterministically.""" - class OpaqueModuleStateError(RulesError): """Raised when custom-rule module state holds a value whose mutations cannot be observed.""" diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index c5c4bc144c..1464e00c2b 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -259,17 +259,6 @@ "sqlbuild.rule_engine._helpers.engine.custom_payloads", "custom_rule_payloads", ), - "fact_outcome_digest": ( - "sqlbuild.rule_engine._helpers.engine.fact_replay", - "fact_outcome_digest", - ), - "full_fact_keys": ("sqlbuild.rule_engine._helpers.engine.fact_replay", "full_fact_keys"), - "is_tree_fact": ("sqlbuild.rule_engine._helpers.engine.fact_replay", "is_tree_fact"), - "shared_fact_encoder": ( - "sqlbuild.rule_engine._helpers.engine.fact_replay", - "shared_fact_encoder", - ), - "build_rule_fact_views": ("sqlbuild.rule_engine.classes.rule_context", "build_rule_fact_views"), "EventExporterInputError": ( "sqlbuild.runtime.event_exporting.exceptions", "EventExporterInputError", diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 2a97ba8825..d71a52e47d 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -450,15 +450,6 @@ from sqlbuild.rule_engine._helpers.engine._custom_catalogue import ( from sqlbuild.rule_engine._helpers.engine.custom_payloads import ( custom_rule_payloads as custom_rule_payloads, ) -from sqlbuild.rule_engine._helpers.engine.fact_replay import ( - fact_outcome_digest as fact_outcome_digest, -) -from sqlbuild.rule_engine._helpers.engine.fact_replay import full_fact_keys as full_fact_keys -from sqlbuild.rule_engine._helpers.engine.fact_replay import is_tree_fact as is_tree_fact -from sqlbuild.rule_engine._helpers.engine.fact_replay import ( - shared_fact_encoder as shared_fact_encoder, -) -from sqlbuild.rule_engine.classes.rule_context import build_rule_fact_views as build_rule_fact_views from sqlbuild.runtime.compute_logs.classes.text_tee import TextComputeLogTee as TextComputeLogTee from sqlbuild.runtime.compute_logs.exceptions import ( CaptureAlreadyExistsError as CaptureAlreadyExistsError, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 7d3f607161..a0d3a8c3f9 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -90,6 +90,7 @@ from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.diagnostics.main.log_sql import log_sql +from sqlbuild.runtime.user_code.main.user_code_spec import user_code_spec from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS @@ -1396,8 +1397,8 @@ def _load_python_udf_callable(self, *, source_file_path: Path, entry_point: str) module_name: str = "sqlbuild_python_udf_" + "_".join( source_file_path.with_suffix("").parts[-4:] ).replace("-", "_") - spec: ModuleSpec | None = importlib.util.spec_from_file_location( - module_name, source_file_path + spec: ModuleSpec | None = user_code_spec( + name=module_name, file_path=source_file_path, root=source_file_path.parent ) if spec is None or spec.loader is None: raise AdapterUserError(message=f"Could not load Python UDF from '{source_file_path}'") diff --git a/src/sqlbuild/adapter/discovery/_helpers/project_adapters.py b/src/sqlbuild/adapter/discovery/_helpers/project_adapters.py index 1c03d017c2..78f13d4760 100644 --- a/src/sqlbuild/adapter/discovery/_helpers/project_adapters.py +++ b/src/sqlbuild/adapter/discovery/_helpers/project_adapters.py @@ -15,6 +15,7 @@ from sqlbuild.adapter.contract.constants import PYTHON_IDENTIFIER_REPLACEMENT, PYTHON_INIT_FILE_NAME from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.discovery.models import DiscoveredAdapter +from sqlbuild.runtime.user_code.main.user_code_spec import user_code_spec def discover_project_adapters( @@ -69,9 +70,8 @@ def _is_public_adapter_file(*, adapters_dir: Path, file_path: Path) -> bool: def _load_adapter_module(*, project_dir: Path, file_path: Path) -> ModuleType: module_name: str = _module_name_for_path(project_dir=project_dir, file_path=file_path) - spec: ModuleSpec | None = importlib.util.spec_from_file_location( - module_name, - file_path, + spec: ModuleSpec | None = user_code_spec( + name=module_name, file_path=file_path, root=project_dir ) if spec is None or spec.loader is None: raise AdapterUserError( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py index 51216b3abe..f28340d71b 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/project_modules.py @@ -24,6 +24,8 @@ ) from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORTER_EVENT_PARAMETER_NAME from sqlbuild.runtime.observability.models import LifecycleEvent +from sqlbuild.runtime.user_code.main.register_user_code_root import register_user_code_root +from sqlbuild.runtime.user_code.main.user_code_spec import user_code_spec _PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" @@ -142,10 +144,14 @@ def install_python_root_package(*, node_root: Path, project_dir: Path) -> dict[s if not owned and module is not None: saved_modules[module_name] = module importlib.invalidate_caches() + _ = register_user_code_root(project_dir) init_file: Path = node_root / f"{PYTHON_INIT_MODULE_STEM}.py" spec: ModuleSpec | None = ( - importlib.util.spec_from_file_location( - PYTHON_NODE_ROOT, init_file, submodule_search_locations=[str(node_root)] + user_code_spec( + name=PYTHON_NODE_ROOT, + file_path=init_file, + root=project_dir, + submodule_search_locations=[str(node_root)], ) if init_file.is_file() else ModuleSpec(PYTHON_NODE_ROOT, None, is_package=True) @@ -188,6 +194,7 @@ def import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleTy relative_path: Path = project_relative_path(path=file_path, project_dir=project_dir) module_name: str = ".".join(relative_path.with_suffix("").parts) + _ = register_user_code_root(project_dir) old_path: list[str] = list(sys.path) sys.path.insert(0, str(project_dir)) try: @@ -319,7 +326,9 @@ def _exec_project_module( file_label: str, error_type: type[Exception], ) -> ModuleType: - spec: ModuleSpec | None = importlib.util.spec_from_file_location(module_name, file_path) + spec: ModuleSpec | None = user_code_spec( + name=module_name, file_path=file_path, root=project_dir + ) if spec is None or spec.loader is None: raise error_type(f"Could not load {file_label} file {file_path}") module: ModuleType = importlib.util.module_from_spec(spec) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py index c8225f220e..08de83b457 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py @@ -4,11 +4,13 @@ import importlib.util from collections.abc import Callable +from pathlib import Path from types import ModuleType from typing import Any from sqlbuild._native import DiscoveredMaterializationFile, PlannerInputError from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult +from sqlbuild.runtime.user_code.main.user_code_spec import user_code_spec def load_custom_materializations( @@ -19,7 +21,8 @@ def load_custom_materializations( registry: dict[str, Callable[[MaterializationContext], MaterializationResult]] = {} mat_file: DiscoveredMaterializationFile for mat_file in materialization_files: - spec: Any = importlib.util.spec_from_file_location(mat_file.name, mat_file.file_path) + file_path: Path = Path(mat_file.file_path) + spec: Any = user_code_spec(name=mat_file.name, file_path=file_path, root=file_path.parent) if spec is None or spec.loader is None: raise PlannerInputError( f"materialization '{mat_file.name}' at {mat_file.file_path} " diff --git a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py index 58cd57f4b7..718b815887 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/_custom_catalogue.py @@ -15,6 +15,7 @@ rule_from_value, ) from sqlbuild.rule_engine.constants import RULE_DECORATOR_TOKEN +from sqlbuild.runtime.user_code.main.user_code_spec import user_code_spec def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: @@ -29,7 +30,9 @@ def load_custom_rules(*, project_dir: Path) -> tuple[Rule, ...]: if RULE_DECORATOR_TOKEN not in file_path.read_text(encoding="utf-8"): continue name: str = _rule_module_name(file_path=file_path, project_dir=project_dir) - spec: ModuleSpec | None = importlib.util.spec_from_file_location(name, file_path) + spec: ModuleSpec | None = user_code_spec( + name=name, file_path=file_path, root=project_dir + ) if spec is None or spec.loader is None: raise RulesError(f"could not load rule file {file_path}") module: ModuleType = importlib.util.module_from_spec(spec) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py b/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py deleted file mode 100644 index dc149b703e..0000000000 --- a/src/sqlbuild/rule_engine/_helpers/engine/fact_replay.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Recompute and digest the compiler facts a custom-rule evaluation recorded.""" - -from __future__ import annotations - -import hashlib -from collections.abc import Callable -from decimal import Decimal -from pathlib import Path, PurePath - -import orjson - -from sqlbuild._native import FactDigestError, Model, RuleFactViews, tracked_text_file_values -from sqlbuild.rule_engine.classes.shared_fact_encoder import SharedFactEncoder -from sqlbuild.rule_engine.classes.sql_document import SqlDocument -from sqlbuild.rule_engine.constants import ( - FACT_AUDITS_ALL, - FACT_AUDITS_FOR_MODEL, - FACT_COLUMNS_DECLARED, - FACT_COLUMNS_INFERRED, - FACT_COLUMNS_NAMES, - FACT_CONTRACTS_ENFORCED, - FACT_CONTRACTS_GRAIN, - FACT_DECLARATIONS_CONSTANTS, - FACT_DECLARATIONS_ENUMS, - FACT_DECLARATIONS_PUBLIC_CONSTANTS, - FACT_DECLARATIONS_PUBLIC_ENUMS, - FACT_GRAPH_DEPENDENCIES, - FACT_GRAPH_DEPENDENTS, - FACT_PROJECT_AUDITS, - FACT_PROJECT_FUNCTIONS, - FACT_PROJECT_MODELS, - FACT_PROJECT_SEEDS, - FACT_PROJECT_SOURCES, - FACT_PROJECT_TESTS, - FACT_SQL_FOR_MODEL, - FACT_TESTS_ALL, - FACT_TESTS_FOR_MODEL, - FACT_TREE_CHILDREN, - FACT_TREE_DESCENDANTS, - FACT_TREE_GLOB, - FACT_TREE_PATHS, - FACT_TREE_READ_TEXT, - FACT_TREE_RELATIVE_PARTS, - FACT_TREE_RESOURCES_UNDER, -) -from sqlbuild.rule_engine.types import FactKey - -_ORJSON_OPTIONS: int = orjson.OPT_SORT_KEYS | orjson.OPT_NON_STR_KEYS - -_GLOBAL_FACTS: dict[str, Callable[[RuleFactViews], object]] = { - FACT_PROJECT_MODELS: lambda views: views.project.models, - FACT_PROJECT_SOURCES: lambda views: views.project.sources, - FACT_PROJECT_SEEDS: lambda views: views.project.seeds, - FACT_PROJECT_FUNCTIONS: lambda views: views.project.functions, - FACT_PROJECT_TESTS: lambda views: views.project.tests, - FACT_PROJECT_AUDITS: lambda views: views.project.audits, - FACT_TREE_PATHS: lambda views: views.project.tree.paths(), - FACT_TESTS_ALL: lambda views: views.tests.all(), - FACT_AUDITS_ALL: lambda views: views.audits.all(), - FACT_DECLARATIONS_PUBLIC_ENUMS: lambda views: views.declarations.public_enums, - FACT_DECLARATIONS_PUBLIC_CONSTANTS: lambda views: views.declarations.public_constants, - FACT_DECLARATIONS_ENUMS: lambda views: views.declarations.enums, - FACT_DECLARATIONS_CONSTANTS: lambda views: views.declarations.constants, -} -_MODEL_PATH_FACTS: dict[str, Callable[[RuleFactViews, Model], object]] = { - FACT_SQL_FOR_MODEL: lambda views, model: views.sql.for_model(model), - FACT_GRAPH_DEPENDENCIES: lambda views, model: views.graph.dependencies(model), - FACT_GRAPH_DEPENDENTS: lambda views, model: views.graph.dependents(model), - FACT_COLUMNS_DECLARED: lambda views, model: views.columns.declared(model), - FACT_COLUMNS_INFERRED: lambda views, model: views.columns.inferred(model), - FACT_COLUMNS_NAMES: lambda views, model: views.columns.names(model), - FACT_CONTRACTS_ENFORCED: lambda views, model: views.contracts.enforced(model), - FACT_CONTRACTS_GRAIN: lambda views, model: views.contracts.grain(model), -} -_MODEL_NAME_FACTS: dict[str, Callable[[RuleFactViews, Model], object]] = { - FACT_TESTS_FOR_MODEL: lambda views, model: views.tests.for_model(model), - FACT_AUDITS_FOR_MODEL: lambda views, model: views.audits.for_model(model), -} -_TEXT_FACTS: dict[str, Callable[[RuleFactViews, tuple[str, ...]], object]] = { - FACT_TREE_CHILDREN: lambda views, arguments: views.project.tree.children(*arguments), - FACT_TREE_DESCENDANTS: lambda views, arguments: views.project.tree.descendants(*arguments), - FACT_TREE_GLOB: lambda views, arguments: views.project.tree.glob(*arguments), - FACT_TREE_RELATIVE_PARTS: lambda views, arguments: views.project.tree.relative_parts( - path=arguments[0], under=arguments[1] - ), - FACT_TREE_RESOURCES_UNDER: lambda views, arguments: views.project.tree.resources_under( - *arguments - ), - FACT_TREE_READ_TEXT: lambda views, arguments: views.project.tree.read_text(*arguments), -} -_TEXT_FACT_ARITY: dict[str, int] = { - FACT_TREE_CHILDREN: 1, - FACT_TREE_DESCENDANTS: 1, - FACT_TREE_GLOB: 1, - FACT_TREE_RELATIVE_PARTS: 2, - FACT_TREE_RESOURCES_UNDER: 1, - FACT_TREE_READ_TEXT: 1, -} -_TREE_FACTS: frozenset[str] = frozenset({FACT_TREE_PATHS, *_TEXT_FACTS}) -_SHARED_FACTS: frozenset[str] = frozenset(_GLOBAL_FACTS) - _TREE_FACTS - - -def fact_key_payload(key: tuple[object, ...]) -> list[str]: - """Return the JSON form of one recorded fact key.""" - - return [value.as_posix() if isinstance(value, PurePath) else str(value) for value in key] - - -def full_fact_keys(views: RuleFactViews) -> tuple[FactKey, ...]: - """Return keys covering every fact a custom rule can observe through its context.""" - - keys: list[FactKey] = [(fact,) for fact in _GLOBAL_FACTS] - models: tuple[Model, ...] = views.project.models - for model in models: - path: str = model.path.as_posix() - keys.extend((fact, path) for fact in _MODEL_PATH_FACTS) - keys.extend((fact, model.name) for fact in _MODEL_NAME_FACTS) - keys.extend( - (FACT_TREE_READ_TEXT, value) - for value in tracked_text_file_values(views.project.tree.paths()) - ) - return tuple(keys) - - -def fact_outcome_digest( - *, views: RuleFactViews, key: FactKey, shared: SharedFactEncoder | None = None -) -> str: - """Recompute one recorded fact and digest its value, sharing whole-project encodings.""" - - try: - value: object = _fact_value(views=views, key=key) - except FactDigestError: - raise - except Exception as error: - return fact_error_digest(error) - if shared is not None and key[0] in _SHARED_FACTS: - try: - return hashlib.sha256(shared.encode(("value", value))).hexdigest() - except (TypeError, FactDigestError) as error: - raise FactDigestError(str(error)) from error - return fact_value_digest(value) - - -def shared_fact_encoder() -> SharedFactEncoder: - """Return an encoder for whole-project facts that share compiler objects.""" - - return SharedFactEncoder(options=_ORJSON_OPTIONS, encode_default=_encode_default) - - -def fact_value_digest(value: object) -> str: - """Digest one fact value exactly as a recomputed fact would digest it.""" - - return _encoded_digest(("value", value)) - - -def fact_error_digest(error: Exception) -> str: - """Digest one raised fact error exactly as a recomputed fact would digest it.""" - - return _encoded_digest(("error", type(error).__name__, str(error))) - - -def is_tree_fact(key: FactKey) -> bool: - """Return whether a fact is read from the live project filesystem.""" - - return bool(key) and key[0] in _TREE_FACTS - - -def _fact_value(*, views: RuleFactViews, key: FactKey) -> object: - if not key: - raise FactDigestError("empty fact key") - fact: str = key[0] - arguments: tuple[str, ...] = key[1:] - if fact in _GLOBAL_FACTS and not arguments: - return _GLOBAL_FACTS[fact](views) - if fact in _MODEL_PATH_FACTS and len(arguments) == 1: - return _MODEL_PATH_FACTS[fact]( - views, Model(name="", path=Path(arguments[0]), materialization=None) - ) - if fact in _MODEL_NAME_FACTS and len(arguments) == 1: - return _MODEL_NAME_FACTS[fact]( - views, Model(name=arguments[0], path=Path(), materialization=None) - ) - if fact in _TEXT_FACTS and len(arguments) == _TEXT_FACT_ARITY[fact]: - return _TEXT_FACTS[fact](views, arguments) - raise FactDigestError(f"unknown fact key: {fact}") - - -def _encoded_digest(value: object) -> str: - try: - encoded: bytes = orjson.dumps(value, default=_encode_default, option=_ORJSON_OPTIONS) - except (TypeError, FactDigestError) as error: - raise FactDigestError(str(error)) from error - return hashlib.sha256(encoded).hexdigest() - - -def _encode_default(value: object) -> object: - if isinstance(value, PurePath): - return value.as_posix() - if isinstance(value, SqlDocument): - return {"source": value.source} - if isinstance(value, Decimal): - return str(value) - if isinstance(value, (set, frozenset)): - return sorted( - orjson.dumps(item, default=_encode_default, option=_ORJSON_OPTIONS).decode() - for item in value - ) - if isinstance(value, tuple): - return list(value) - raise FactDigestError(f"fact value of type {type(value).__name__} is not reproducible") diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py index b00c6ecb87..7ac2bcc415 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host.py @@ -11,7 +11,7 @@ import sys import time from dataclasses import asdict -from pathlib import Path +from pathlib import Path, PurePath from typing import Any, BinaryIO from sqlbuild._native import ( @@ -29,7 +29,6 @@ build_rule_catalogue, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_key_payload from sqlbuild.rule_engine._helpers.host.custom_evaluation import evaluate_custom_rules from sqlbuild.rule_engine.classes.runtime_guard import RuntimeGuard from sqlbuild.rule_engine.constants import ( @@ -281,5 +280,11 @@ def _write_error(message: str) -> int: return 0 +def fact_key_payload(key: tuple[object, ...]) -> list[str]: + """Return the JSON form of one recorded fact key.""" + + return [value.as_posix() if isinstance(value, PurePath) else str(value) for value in key] + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py index 854fc1dc92..619fac553a 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py +++ b/src/sqlbuild/rule_engine/_helpers/host/tracked_facts.py @@ -10,15 +10,15 @@ CompiledSqlTest, ConstantDeclaration, EnumDeclaration, - FactDigestError, InferredColumn, Model, ModelSql, ProjectPath, RuleFactViews, + fact_error_digest, + fact_value_digest, ) from sqlbuild.compiler.compile.types import CompiledAuditView -from sqlbuild.rule_engine._helpers.engine.fact_replay import fact_error_digest, fact_value_digest from sqlbuild.rule_engine.classes.fact_reads import FactReads from sqlbuild.rule_engine.constants import ( FACT_AUDITS_ALL, @@ -137,10 +137,7 @@ def _observed[T](self, *, key: tuple[str, ...] | None, read: Callable[[], T]) -> def _digest_or_none(*, value: object = None, error: Exception | None = None) -> str | None: - try: - return fact_value_digest(value) if error is None else fact_error_digest(error) - except FactDigestError: - return None + return fact_value_digest(value) if error is None else fact_error_digest(error) class _TrackedProjectFacts(_TrackedView): diff --git a/src/sqlbuild/rule_engine/classes/project_tree.py b/src/sqlbuild/rule_engine/classes/project_tree.py index b009d556b0..c2b62eedf9 100644 --- a/src/sqlbuild/rule_engine/classes/project_tree.py +++ b/src/sqlbuild/rule_engine/classes/project_tree.py @@ -11,9 +11,9 @@ RuleUsageError, project_tree_child_nodes, project_tree_descendant_nodes, - project_tree_file_path, project_tree_glob_nodes, project_tree_paths, + project_tree_read_text, project_tree_relative_parts, ) from sqlbuild.compiler.compile.types import CompiledProjectView @@ -68,11 +68,7 @@ def resources_under(self, path: str) -> tuple[Model, ...]: ) def read_text(self, path: str) -> str: - file: str = project_tree_file_path(self._root, path) - try: - return Path(file).read_text(encoding="utf-8") - except (OSError, UnicodeError) as error: - raise RuleUsageError(f"could not read project file {path}: {error}") from error + return project_tree_read_text(self._root, path) def public_model(model: object) -> Model: diff --git a/src/sqlbuild/rule_engine/classes/shared_fact_encoder.py b/src/sqlbuild/rule_engine/classes/shared_fact_encoder.py deleted file mode 100644 index 6ce301a3bf..0000000000 --- a/src/sqlbuild/rule_engine/classes/shared_fact_encoder.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Digest whole-project fact values whose resources share large compiler objects.""" - -from __future__ import annotations - -import dataclasses -import hashlib -from collections.abc import Callable - -import orjson - -from sqlbuild.rule_engine.constants import SHARED_FACT_DATACLASS_TAG - - -class SharedFactEncoder: - """Encode each compiler dataclass once and refer to it by digest wherever it repeats.""" - - def __init__(self, *, options: int, encode_default: Callable[[object], object]) -> None: - self._options: int = options | orjson.OPT_PASSTHROUGH_DATACLASS - self._encode_default: Callable[[object], object] = encode_default - self._fields: dict[type, tuple[str, ...]] = {} - self._digests: dict[int, tuple[object, str]] = {} - - def encode(self, value: object) -> bytes: - return orjson.dumps(value, default=self._default, option=self._options) - - def _default(self, value: object) -> object: - kind: type = type(value) - names: tuple[str, ...] | None = self._fields.get(kind) - if names is None: - if not dataclasses.is_dataclass(kind): - return self._encode_default(value) - names = tuple(field.name for field in dataclasses.fields(kind)) - self._fields[kind] = names - cached: tuple[object, str] | None = self._digests.get(id(value)) - if cached is None: - encoded: bytes = self.encode( - [f"{kind.__module__}.{kind.__qualname__}", [getattr(value, name) for name in names]] - ) - cached = (value, hashlib.sha256(encoded).hexdigest()) - self._digests[id(value)] = cached - return [SHARED_FACT_DATACLASS_TAG, cached[1]] diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index 042f1b718e..02533e5f14 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -2,7 +2,6 @@ CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 -SHARED_FACT_DATACLASS_TAG: str = "\0dataclass" CUSTOM_HOST_INPUT_POLL_SECONDS: float = 0.002 CUSTOM_HOST_INPUT_WAIT_SECONDS: float = 120.0 CUSTOM_HOST_MODULE: str = "sqlbuild.rule_engine._helpers.host.custom_host" diff --git a/src/sqlbuild/runtime/user_code/__init__.py b/src/sqlbuild/runtime/user_code/__init__.py new file mode 100644 index 0000000000..1d533f8e84 --- /dev/null +++ b/src/sqlbuild/runtime/user_code/__init__.py @@ -0,0 +1 @@ +"""Loading project Python from source so edits are never hidden by cached bytecode.""" diff --git a/src/sqlbuild/runtime/user_code/_helpers/__init__.py b/src/sqlbuild/runtime/user_code/_helpers/__init__.py new file mode 100644 index 0000000000..8f6296a71e --- /dev/null +++ b/src/sqlbuild/runtime/user_code/_helpers/__init__.py @@ -0,0 +1 @@ +"""User-code import helpers.""" diff --git a/src/sqlbuild/runtime/user_code/_helpers/finder.py b/src/sqlbuild/runtime/user_code/_helpers/finder.py new file mode 100644 index 0000000000..c78688c994 --- /dev/null +++ b/src/sqlbuild/runtime/user_code/_helpers/finder.py @@ -0,0 +1,5 @@ +"""The process-wide user-code finder.""" + +from sqlbuild.runtime.user_code.classes.user_code_finder import UserCodeFinder + +user_code_finder: UserCodeFinder = UserCodeFinder() diff --git a/src/sqlbuild/runtime/user_code/classes/__init__.py b/src/sqlbuild/runtime/user_code/classes/__init__.py new file mode 100644 index 0000000000..7cc414164f --- /dev/null +++ b/src/sqlbuild/runtime/user_code/classes/__init__.py @@ -0,0 +1 @@ +"""User-code import machinery.""" diff --git a/src/sqlbuild/runtime/user_code/classes/source_only_loader.py b/src/sqlbuild/runtime/user_code/classes/source_only_loader.py new file mode 100644 index 0000000000..9e33baafcd --- /dev/null +++ b/src/sqlbuild/runtime/user_code/classes/source_only_loader.py @@ -0,0 +1,12 @@ +"""A source loader that neither reads nor writes cached bytecode.""" + +from importlib.machinery import SourceFileLoader +from types import CodeType + + +class SourceOnlyLoader(SourceFileLoader): + """Compile a module from current source; whole-second pyc checks miss same-size edits.""" + + def get_code(self, fullname: str) -> CodeType: + source_path: str = self.get_filename(fullname) + return self.source_to_code(self.get_data(source_path), source_path) diff --git a/src/sqlbuild/runtime/user_code/classes/user_code_finder.py b/src/sqlbuild/runtime/user_code/classes/user_code_finder.py new file mode 100644 index 0000000000..cecc1f56ed --- /dev/null +++ b/src/sqlbuild/runtime/user_code/classes/user_code_finder.py @@ -0,0 +1,47 @@ +"""A meta-path finder that loads modules under registered project roots from source.""" + +import sys +import sysconfig +from collections.abc import Sequence +from importlib.machinery import ModuleSpec, PathFinder, SourceFileLoader +from pathlib import Path +from types import ModuleType + +from sqlbuild.runtime.user_code.classes.source_only_loader import SourceOnlyLoader + +_INSTALLED_PACKAGE_DIRECTORIES: frozenset[str] = frozenset({"site-packages", "dist-packages"}) + + +class UserCodeFinder: + """Find modules normally, switching source files under a project root to source-only loading.""" + + def __init__(self) -> None: + self.roots: set[Path] = set() + self.installed: tuple[Path, ...] = tuple( + Path(location).resolve() + for location in { + sys.prefix, + sys.base_prefix, + sys.exec_prefix, + *sysconfig.get_paths().values(), + } + ) + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None = None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + spec: ModuleSpec | None = PathFinder.find_spec(fullname, path, target) + if spec is None or spec.origin is None or not isinstance(spec.loader, SourceFileLoader): + return None + origin: Path = Path(spec.origin) + if not any(origin.is_relative_to(root) for root in tuple(self.roots)): + return None + if _INSTALLED_PACKAGE_DIRECTORIES.intersection(origin.parts) or any( + origin.is_relative_to(location) for location in self.installed + ): + return None + spec.loader = SourceOnlyLoader(fullname, spec.origin) + return spec diff --git a/src/sqlbuild/runtime/user_code/main/__init__.py b/src/sqlbuild/runtime/user_code/main/__init__.py new file mode 100644 index 0000000000..d6f2f12326 --- /dev/null +++ b/src/sqlbuild/runtime/user_code/main/__init__.py @@ -0,0 +1 @@ +"""User-code import operations.""" diff --git a/src/sqlbuild/runtime/user_code/main/register_user_code_root.py b/src/sqlbuild/runtime/user_code/main/register_user_code_root.py new file mode 100644 index 0000000000..635ec27245 --- /dev/null +++ b/src/sqlbuild/runtime/user_code/main/register_user_code_root.py @@ -0,0 +1,19 @@ +"""Register a project root whose modules always load from current source.""" + +import sys +from importlib.machinery import PathFinder +from pathlib import Path + +from sqlbuild.runtime.user_code._helpers.finder import user_code_finder + + +def register_user_code_root(root: Path) -> None: + """Load every module later imported from under `root` from its current source.""" + + user_code_finder.roots.add(root.resolve()) + if user_code_finder in sys.meta_path: + return + position: int = ( + sys.meta_path.index(PathFinder) if PathFinder in sys.meta_path else len(sys.meta_path) + ) + sys.meta_path.insert(position, user_code_finder) diff --git a/src/sqlbuild/runtime/user_code/main/user_code_spec.py b/src/sqlbuild/runtime/user_code/main/user_code_spec.py new file mode 100644 index 0000000000..9f7162b19a --- /dev/null +++ b/src/sqlbuild/runtime/user_code/main/user_code_spec.py @@ -0,0 +1,26 @@ +"""Module specs for project Python files that always load from current source.""" + +from importlib.machinery import ModuleSpec +from importlib.util import spec_from_file_location +from pathlib import Path + +from sqlbuild.runtime.user_code.classes.source_only_loader import SourceOnlyLoader +from sqlbuild.runtime.user_code.main.register_user_code_root import register_user_code_root + + +def user_code_spec( + *, + name: str, + file_path: Path, + root: Path, + submodule_search_locations: list[str] | None = None, +) -> ModuleSpec | None: + """A spec loading `file_path` from source, with `root` registered for its own imports.""" + + _ = register_user_code_root(root) + return spec_from_file_location( + name, + file_path, + loader=SourceOnlyLoader(name, str(file_path)), + submodule_search_locations=submodule_search_locations, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 2fd96350e6..f02bbb8997 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4807,7 +4807,6 @@ def full_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( (native_module, "infer_native_expression_source_shapes"), - (native_module, "analyze_native_model_sql"), (native_module, "assemble_native_sql_tests"), (native_module, "complete_semantic_project"), (native_module, "evaluate_model_contracts"), @@ -4973,11 +4972,10 @@ def compile_with_binding_error(project_dir: Path) -> dict[str, Any]: _SLOW_LINT_MACRO: str = ( "import os\n" - "import threading\n" "import time\n\n\n" "def slow_total(amount: str) -> str:\n" - ' """Return the amount, pausing when the background SQL lint expands it."""\n' - " if threading.current_thread() is not threading.main_thread():\n" + ' """Return the amount, pausing for the macro tests that only SQL lint expands."""\n' + " if amount == 'slow_amount':\n" " with open(os.environ['SLOW_MACRO_MARKER'], 'a', encoding='utf-8') as marker:\n" " marker.write('expanding\\n')\n" " time.sleep(float(os.environ['SLOW_MACRO_SECONDS']))\n" @@ -4986,7 +4984,7 @@ def compile_with_binding_error(project_dir: Path) -> dict[str, Any]: def slow_lint_model_files(*, model_count: int) -> dict[str, str]: - """A macro used by a model, a function, and macro tests only background SQL lint expands.""" + """A macro used by a model, a function, and macro tests that pause when linted.""" return { "macros/slow.py": _SLOW_LINT_MACRO, @@ -5002,9 +5000,9 @@ def slow_lint_model_files(*, model_count: int) -> dict[str, str]: f"tests/slow_total_{index}.sql": ( f'TEST (mode macro, name "slow_total_{index}");\n\n' "WITH\n" - "input_values AS (SELECT 5 AS amount),\n" + "input_values AS (SELECT 5 AS slow_amount),\n" "__macro_actual__ AS (\n" - ' SELECT @slow_total("amount") AS amount FROM input_values\n' + ' SELECT @slow_total("slow_amount") AS amount FROM input_values\n' "),\n" "__macro_expected__ AS (SELECT 5 AS amount)\n" "SELECT 1\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index b9b48cb311..f8dc62f435 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -10,7 +10,6 @@ import sqlbuild._native as native_module from sqlbuild._native import ( ContractValidationResult, - NativeModelAnalyses, NativeSqlTestArtifact, NativeSqlTestAssembly, ProjectColumnLineage, @@ -31,7 +30,6 @@ description="models_contracts_lineage_and_sql_tests", expected_native_returns={ "infer_native_expression_source_shapes": [ANY, ANY, ANY, ANY], - "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [ANY], "complete_semantic_project": [ANY], "evaluate_model_contracts": [ANY], @@ -63,8 +61,6 @@ def test_given_project_when_compiling_then_native_seams_answer( planned_counts: list[int] = [] for seams in (native_seams,): assert isinstance(seams["build_project_column_lineage"][0], ProjectColumnLineage) - assert isinstance(seams["analyze_native_model_sql"][0], NativeModelAnalyses) - assert seams["analyze_native_model_sql"][0].session is not None assert isinstance(seams["infer_native_expression_source_shapes"][0], tuple) assembled_tests: object = seams["assemble_native_sql_tests"][0] assert isinstance(assembled_tests, tuple) @@ -75,7 +71,9 @@ def test_given_project_when_compiling_then_native_seams_answer( assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) planned_counts.append(len(planned_artifacts)) - assert isinstance(seams["complete_semantic_project"][0], native_module.NativeProjectView) + completed: object = seams["complete_semantic_project"][0] + assert isinstance(completed, native_module.NativeProjectView) + assert any(model.inferred_columns for model in completed.models) assert isinstance(seams["evaluate_model_contracts"][0], ContractValidationResult) assert [ len(call.args[0].tests) for call in native_sql_test_planning.call_args_list diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index cd14c2f000..5256b17d0d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -235,13 +235,3 @@ class OneModelCustomRuleEditCase: expected_warm_counts: tuple[int, int] expected_edit_counts: tuple[int, int] expected_edit_exit_code: int - - -@dataclass(frozen=True) -class BrokenInvalidationChainCase: - """An edit chain compiled with deliberately stale fact digests against a cache-free oracle.""" - - description: str - edits: tuple[RulesCacheEditCase, ...] - stale_digests_sitecustomize: str - expected_first_divergent_edit: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 0a7ea9587d..8c7a47a4be 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -494,23 +494,6 @@ def few_marts(*, project: Project, ctx: RuleContext) -> list[Finding]: return [ctx.finding(subject=marts[0], message=f"{len(marts)} marts")] """, } -STALE_SOURCE_FACTS_SITECUSTOMIZE: str = """from sqlbuild.rule_engine._helpers.engine import ( - fact_replay, -) - -_digest = fact_replay.fact_outcome_digest - - -def _stale_sources(*, views, key, shared=None): - if key == ("project.sources",): - return "stale" - return _digest(views=views, key=key, shared=shared) - - -fact_replay.fact_outcome_digest = _stale_sources -""" - - ONE_MODEL_EDIT_MODEL_COUNT: int = 6 # One cached result per model for the custom rule, plus the cached built-in rules request. ONE_MODEL_EDIT_CACHED_RESULTS: int = ONE_MODEL_EDIT_MODEL_COUNT + 1 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py index 72661eccf3..7e49d93a05 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py @@ -10,7 +10,6 @@ write_project_file, ) from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( - BrokenInvalidationChainCase, OneModelCustomRuleEditCase, RulesCacheEditCase, RulesEditChainCase, @@ -18,7 +17,6 @@ from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( ONE_MODEL_EDIT_CACHED_RESULTS, ONE_MODEL_EDIT_PROJECT, - STALE_SOURCE_FACTS_SITECUSTOMIZE, custom_rules_edit_chain, rule_cache_counts, rules_compile_outcome, @@ -109,35 +107,6 @@ def test_given_warm_custom_rules_when_applying_edit_chain_then_every_compile_equ assert rule_cache_counts(tmp_path / "orders")[1] == 0 -@pytest.mark.parametrize( - "test_case", - ( - BrokenInvalidationChainCase( - description="stale_shared_source_digests", - edits=CUSTOM_EDIT_SEQUENCE, - stale_digests_sitecustomize=STALE_SOURCE_FACTS_SITECUSTOMIZE, - expected_first_divergent_edit="source_in_shared_file_renamed", - ), - ), - ids=lambda case: case.description, -) -def test_given_broken_invalidation_when_applying_edit_chain_then_oracle_detects_divergence( - tmp_path: Path, test_case: BrokenInvalidationChainCase -) -> None: - write_project_file(tmp_path / "site", "sitecustomize.py", test_case.stale_digests_sitecustomize) - - _, edited, oracles = custom_rules_edit_chain( - root=tmp_path, edits=test_case.edits, environment=(("PYTHONPATH", str(tmp_path / "site")),) - ) - matches: list[bool] = [ - outcome == oracle for outcome, oracle in zip(edited, oracles, strict=True) - ] - - assert test_case.edits[matches.index(False)].description == ( - test_case.expected_first_divergent_edit - ) - - @pytest.mark.parametrize( "test_case", ( diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py index 4fa6e4789d..350086450d 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_types.py @@ -28,15 +28,6 @@ class GeneratedAnalysisTestCase: expected_minimum_native_enrichments: int -@dataclass(frozen=True) -class InternalFailureTestCase: - """A project whose native analysis fails internally, and the error the compile raises.""" - - description: str - files: dict[str, str] - expected_message: str - - @dataclass(frozen=True) class CyclicAnalysisTestCase: """Models that `ref` each other, which the native session analyses in one unordered wave.""" @@ -135,14 +126,6 @@ class TableFunctionColumnsTestCase: expected_error: str | None -@dataclass(frozen=True) -class MissingSessionAnalysisTestCase: - """A broken invariant: the session returns no analysis for a model it was asked about.""" - - description: str - expected_message: str - - @dataclass(frozen=True) class ProjectionFactsTestCase: """Fixture queries whose projection facts Python recorded in a golden.""" diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index f6f471fc2f..38befd7710 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -8,6 +8,7 @@ from contextlib import suppress from dataclasses import dataclass, field, replace from itertools import chain +from operator import attrgetter from pathlib import Path from typing import TYPE_CHECKING, Any, cast @@ -15,7 +16,6 @@ import sqlbuild._native as native_module from sqlbuild._native import ( - AnalysisCacheContext, CollectionRendering, ColumnLineageMode, CompactLineageFacts, @@ -24,12 +24,8 @@ InferredColumn, InferredNullability, ModelSqlAnalysis, - NativeModelAnalyses, - NativeModelAnalysisRequest, NativePivotTables, SchemaDynamicColumnFamily, - analyze_native_model_sql, - build_analysis_cache_context, build_compile_inputs, cursor_intrinsics_analysis_sql, discover_project_inputs, @@ -487,20 +483,6 @@ def shared_analysis_files( } -def started_sessions(*, monkeypatch: pytest.MonkeyPatch) -> list[Any]: - """Record every native session started; return the list they are appended to.""" - - started: list[Any] = [] - original: Callable[..., Any] = native_module.start_model_analysis_session - - def start(catalog: object, request: tuple[object, ...], *options: Any) -> Any: - started.append(original(catalog, request, *options)) - return started[-1] - - monkeypatch.setattr(native_module, "start_model_analysis_session", start) - return started - - def compile_inputs(*, project_dir: Path, files: dict[str, str]) -> CompileInputsView: """Write a project and attach its compile inputs with the DuckDB compile context.""" @@ -549,7 +531,7 @@ def compiled_project_view( @dataclass class NativeAnalysisRuns: - """Each native analysis seam call's uncached, cold-cached and warm-cached views.""" + """Each assembly's uncached, cold-cached and warm-cached views, and its uncached sessions.""" names: list[object] = field(default_factory=list) uncached: list[object] = field(default_factory=list) @@ -563,6 +545,7 @@ class NativeAnalysisRuns: native_enrichments: int = 0 native_column_values: int = 0 golden: list[GoldenEntry] = field(default_factory=list) + sessions: list[Any] = field(default_factory=list) def analyse_natively( @@ -574,50 +557,19 @@ def analyse_natively( cache_root: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - """Assemble `inputs`, analysing each model batch natively uncached, cold-cached and warm. + """Assemble `inputs` with a cold, then a warm analysis cache, then uncached, and compare them. Every analysis, catalog change, session and standalone dynamic pivot proof, and expression shape must not depend on the analysis store, on the session, or on Python's shape inference. - `runs.golden` gathers the entries the Python oracle recorded for the same seam calls. + `runs.golden` gathers the entries the uncached assembly records, in the order it records them. """ - original_analysis: Callable[..., NativeModelAnalyses] = analyze_native_model_sql - - def analysed_three_ways(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: - profile: ExpressionInferenceProfile = request.inference_profile - cache: AnalysisCacheContext | None = build_analysis_cache_context( - root=cache_root, inference_profile=profile, allow_compact_analysis=True - ) - cached_views: list[object] = [] - for _ in range(2): - catalog: Any = cast(Any, profile.binding_catalog).with_relations({}) - cached: NativeModelAnalyses = analyze_native_model_sql( - request=replace( - request, - analysis_cache=cache, - inference_profile=replace(profile, binding_catalog=catalog), - ) - ) - cached_views.append( - (_analysis_views(_projected(cached, request, runs)), _catalog_view(catalog)) - ) - native: NativeModelAnalyses = original_analysis( - request=replace(request, analysis_cache=None) - ) - session: Any = native.session - analyses: dict[str, ModelSqlAnalysis] = _projected(native, request, runs) - uncached_view: object = (_analysis_views(analyses), _catalog_view(profile.binding_catalog)) - columns: list[InferredColumn] = list( - chain.from_iterable(map(_native_columns, analyses.values())) - ) - runs.native_column_values += len(set(columns)) - runs.analysed_models += len(analyses) - runs.golden.append(golden_entry("models", _analysis_views(analyses))) - runs.golden.append(golden_entry("catalog", _catalog_view(profile.binding_catalog))) - for index, view in enumerate(cached_views): - _append(runs, f"model analyses, cached run {index}", uncached_view, view) - _compare_proofs(request=request, analyses=analyses, session=session, runs=runs) - return native + views: list[object] = [] + sinks: list[NativeAnalysisRuns] = [runs] + profiles: list[ExpressionInferenceProfile] = [] + proved: Callable[..., tuple[ProofRow | None, ...]] = ( + native_module.prove_native_dynamic_contracts + ) def recorded_shapes( *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile @@ -626,14 +578,27 @@ def recorded_shapes( native: tuple[dict[str, str] | None, ...] = infer_native_expression_source_shapes( expressions=expressions, profile=profile ) - runs.expression_shapes += sum(shape is not None for shape in native) - runs.golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) + profiles.append(profile) + sinks[-1].expression_shapes += sum(shape is not None for shape in native) + sinks[-1].golden.append(golden_entry("shapes", (native, catalog.expression_shapes))) return native + def recorded_analyses(**keywords: Any) -> tuple[ProofRow | None, ...]: + for profile in profiles[-1:]: + _record_analyses( + inputs=inputs, + catalog=profile.binding_catalog, + proof_inputs=(keywords["tables"], keywords["session"]), + views=views, + runs=sinks[-1], + ) + profiles.clear() + return proved(**keywords) + completed: Callable[..., CompiledProjectView] = native_module.complete_semantic_project def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> CompiledProjectView: - runs.golden.extend( + sinks[-1].golden.extend( golden_entry("proof", model.dynamic_column_contract) for model in project.models ) with monkeypatch.context() as completing: @@ -646,27 +611,57 @@ def recorded_proofs(*, project: CompiledProjectView, **keywords: Any) -> Compile with monkeypatch.context() as patch: patch.setattr(native_module, "complete_semantic_project", recorded_proofs) - patch.setattr(native_module, "analyze_native_model_sql", analysed_three_ways) patch.setattr(native_module, "infer_native_expression_source_shapes", recorded_shapes) - with suppress(CompileInputError): - _ = native_module.assemble_project( - inputs=inputs, - inference_profile=replace(inference_profile), - column_lineage_mode=lineage_mode, - ) - - -def duplicate_analysed_model_names(*, monkeypatch: pytest.MonkeyPatch) -> None: - """Name every analysed model after the first, an invariant discovery guarantees (D007).""" + patch.setattr(native_module, "prove_native_dynamic_contracts", recorded_analyses) + for cache_dir, sink in ( + (cache_root, NativeAnalysisRuns()), + (cache_root, NativeAnalysisRuns()), + (None, runs), + ): + sinks.append(sink) + with suppress(CompileInputError): + _ = native_module.assemble_project( + inputs=inputs, + inference_profile=replace(inference_profile), + column_lineage_mode=lineage_mode, + analysis_cache_dir=cache_dir, + ) + for index, view in enumerate(views[:-1]): + _append(runs, f"model analyses, cached run {index}", views[-1], view) - original: Callable[..., Any] = native_module.start_model_analysis_session - def renamed(catalog: object, request: tuple[object, ...], *options: Any) -> Any: - rows: list[tuple[object, ...]] = cast(list[tuple[object, ...]], request[-1]) - named: list[tuple[object, ...]] = [(rows[0][0], *row[1:]) for row in rows] - return original(catalog, (*request[:-1], named), *options) +def _record_analyses( + *, + inputs: CompileInputsView, + catalog: object, + proof_inputs: tuple[NativePivotTables, Any], + views: list[object], + runs: NativeAnalysisRuns, +) -> None: + """Record one assembly's analyses, catalog and session, and compare its pivot proofs.""" - monkeypatch.setattr(native_module, "start_model_analysis_session", renamed) + tables, session = proof_inputs + analysed: list[ModelInput] = list( + filter(attrgetter("sql_validation_enabled"), inputs.model_inputs) + ) + sources: list[str] = [session.outcome_rows(position)[0][2] for position in range(len(analysed))] + runs.native_enrichments += sum(source != LINEAGE_NATIVE for source in sources) + analyses: dict[str, ModelSqlAnalysis] = { + model_input.model_file.file_path.stem: session_outcome_analysis( + session=session, position=position, placeholders=model_placeholders(model_input) + ) + for position, model_input in enumerate(analysed) + } + views.append((_analysis_views(analyses), _catalog_view(catalog))) + columns: list[InferredColumn] = list( + chain.from_iterable(map(_native_columns, analyses.values())) + ) + runs.native_column_values += len(set(columns)) + runs.analysed_models += len(analyses) + runs.golden.append(golden_entry("models", _analysis_views(analyses))) + runs.golden.append(golden_entry("catalog", _catalog_view(catalog))) + _compare_proofs(inputs=inputs, tables=tables, analyses=analyses, session=session, runs=runs) + runs.sessions.append(session) def custom_nullability_rule(arguments: tuple[InferredNullability, ...]) -> InferredNullability: @@ -675,57 +670,25 @@ def custom_nullability_rule(arguments: tuple[InferredNullability, ...]) -> Infer return InferredNullability.NULLABLE -def analysis_request( - *, - inputs: CompileInputsView, - monkeypatch: pytest.MonkeyPatch, - inference_profile: ExpressionInferenceProfile = _DUCKDB_PROFILE, -) -> NativeModelAnalysisRequest: - """The native request the model analysis seam builds while assembling `inputs`.""" - - requests: list[NativeModelAnalysisRequest] = [] - - def captured(request: NativeModelAnalysisRequest) -> NativeModelAnalyses: - requests.append(request) - return NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None) - - with ( - monkeypatch.context() as patch, - suppress(CompileInputError, native_module.NativeCompilerError), - ): - patch.setattr(native_module, "analyze_native_model_sql", captured) - _ = native_module.assemble_project( - inputs=inputs, - inference_profile=inference_profile, - ) - return requests[0] - - def _native_columns(analysis: ModelSqlAnalysis) -> tuple[InferredColumn, ...]: return analysis.polyglot_analysis.columns or () def _compare_proofs( *, - request: NativeModelAnalysisRequest, + inputs: CompileInputsView, + tables: NativePivotTables, analyses: dict[str, ModelSqlAnalysis], session: Any, runs: NativeAnalysisRuns, ) -> None: """Each analysed pivot model's proof, proven again in the session and standalone.""" - tables: NativePivotTables = NativePivotTables( - dialect=request.inference_profile.sql_analysis_dialect, - column_types_by_table=request.column_types_by_table, - authoritative_column_types_by_table=request.complete_binding_schemas, - column_nullability_by_table=request.column_nullability_by_table, - dynamic_families_by_table=request.dynamic_families_by_table, - ) proved: list[bool] = [ bool(_dynamic_families(model_input)) and model_input.model_file.file_path.stem in analyses - for model_input in request.model_inputs + for model_input in inputs.model_inputs ] - pivot_inputs: list[ModelInput] = list(itertools.compress(request.model_inputs, proved)) + pivot_inputs: list[ModelInput] = list(itertools.compress(inputs.model_inputs, proved)) for model_input in pivot_inputs: families: tuple[SchemaDynamicColumnFamily, ...] = _dynamic_families(model_input) analysis: ModelSqlAnalysis = analyses[model_input.model_file.file_path.stem] @@ -762,28 +725,6 @@ def _append(runs: NativeAnalysisRuns, name: str, uncached: object, cached: objec runs.cached.append(cached) -def _projected( - native: NativeModelAnalyses, request: NativeModelAnalysisRequest, runs: NativeAnalysisRuns -) -> dict[str, ModelSqlAnalysis]: - """The test oracle's projection of each finished outcome as the analysis it replaced.""" - - placeholders: dict[str, dict[str, str] | None] = { - model_input.model_file.file_path.stem: model_placeholders(model_input) - for model_input in request.model_inputs - } - sources: list[str] = [ - cast(Any, native.session).outcome_rows(position)[0][2] - for position in native.positions.values() - ] - runs.native_enrichments += sum(source != LINEAGE_NATIVE for source in sources) - return { - name: session_outcome_analysis( - session=native.session, position=position, placeholders=placeholders.get(name) - ) - for name, position in native.positions.items() - } - - def _analysis_views(analyses: dict[str, ModelSqlAnalysis]) -> object: return { name: ( @@ -821,17 +762,13 @@ def __call__(self, arguments: tuple[InferredNullability, ...]) -> InferredNullab return self._outcomes[-1]() -def adapter_rule_request( - *, - test_case: AdapterRuleCallbackTestCase, - rule: RecordingNullabilityRule, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> NativeModelAnalysisRequest: - """A generated project plus the adapter-rule models, analysed with `rule` for UPPER and IFF.""" +def adapter_rule_project( + *, test_case: AdapterRuleCallbackTestCase, rule: RecordingNullabilityRule, tmp_path: Path +) -> tuple[CompileInputsView, ExpressionInferenceProfile]: + """A generated project with the adapter-rule models, and a profile calling `rule`.""" - request: NativeModelAnalysisRequest = analysis_request( - inputs=compile_inputs( + return ( + compile_inputs( project_dir=tmp_path / "project", files={ **generated_analysis_files( @@ -840,14 +777,11 @@ def adapter_rule_request( **ADAPTER_RULE_MODELS, }, ), - monkeypatch=monkeypatch, - inference_profile=ExpressionInferenceProfile( + ExpressionInferenceProfile( sql_analysis_dialect="duckdb", function_nullability_rules={"UPPER": rule, "IFF": rule}, ), ) - rule.calls.clear() - return request def _nullable() -> InferredNullability: diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 26028f3a36..ca5f413ed6 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -10,12 +10,9 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( ColumnLineageMode, InferredNullability, - NativeModelAnalysisRequest, - analyze_native_model_sql, assemble_project, ) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -27,7 +24,6 @@ CyclicAnalysisTestCase, CyclicCompileTestCase, GeneratedAnalysisTestCase, - InternalFailureTestCase, SharedAnalysisTestCase, StandalonePivotProofTestCase, ) @@ -36,17 +32,15 @@ NativeAnalysisRuns, NativePivotProofs, RecordingNullabilityRule, - adapter_rule_request, + adapter_rule_project, analyse_natively, compile_inputs, compiled_project_view, custom_nullability_rule, - duplicate_analysed_model_names, generated_analysis_files, native_pivot_proofs, pivot_project_files, shared_analysis_files, - started_sessions, ) from tests.integration.src.sqlbuild.compiler.helpers import ( golden_differences, @@ -251,7 +245,6 @@ def test_given_equal_model_queries_when_analysing_natively_then_shares_and_cache regions=test_case.regions, inexact_regions=test_case.inexact_regions ), ) - sessions: list[Any] = started_sessions(monkeypatch=monkeypatch) runs: NativeAnalysisRuns = NativeAnalysisRuns() _ = [ @@ -266,7 +259,7 @@ def test_given_equal_model_queries_when_analysing_natively_then_shares_and_cache for dialect in test_case.dialects ] - uncached: list[Any] = sessions[2::3] + uncached: list[Any] = runs.sessions shared: int = sum(session.sharing[0] for session in uncached) assert ( @@ -310,7 +303,6 @@ def test_given_cyclic_models_when_analysing_natively_then_session_answers_and_ca inputs: CompileInputsView = compile_inputs( project_dir=tmp_path / "project", files=test_case.files ) - sessions: list[Any] = started_sessions(monkeypatch=monkeypatch) runs: NativeAnalysisRuns = NativeAnalysisRuns() _ = [ @@ -334,8 +326,8 @@ def test_given_cyclic_models_when_analysing_natively_then_session_answers_and_ca assert mismatches(inputs=runs.names, expected=runs.uncached, actual=runs.cached) == [] assert ( runs.analysed_models, - len(sessions) // 3, - None in sessions, + len(runs.sessions), + None in runs.sessions, ) == ( test_case.expected_analysed, len(test_case.dialects) * len(test_case.lineage_modes), @@ -381,36 +373,6 @@ def test_given_cyclic_project_when_compiling_then_native_output_is_complete_and_ ) -@pytest.mark.parametrize( - "test_case", - [ - InternalFailureTestCase( - description="two analysed models sharing a name break the session's invariant", - files=_ORDERS_PROJECT, - expected_message=( - "NativeCompilerError: native model analysis: two analysed models share a name" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_internal_native_failure_when_assembling_then_raises_without_python_analysis( - test_case: InternalFailureTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - inputs: CompileInputsView = compile_inputs( - project_dir=tmp_path / "project", files=test_case.files - ) - duplicate_analysed_model_names(monkeypatch=monkeypatch) - - with pytest.raises(native_module.NativeCompilerError) as raised: - _ = assemble_project( - inputs=inputs, - inference_profile=ExpressionInferenceProfile(sql_analysis_dialect="duckdb"), - ) - - assert str(raised.value) == test_case.expected_message - - @pytest.mark.parametrize( "test_case", [ @@ -483,14 +445,14 @@ def test_given_unanalysed_pivot_model_when_assembling_then_native_proves_it_with ids=lambda case: case.description, ) def test_given_adapter_nullability_rule_when_analysing_natively_then_rule_is_called_back( - test_case: AdapterRuleCallbackTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: AdapterRuleCallbackTestCase, tmp_path: Path ) -> None: rule: RecordingNullabilityRule = RecordingNullabilityRule(raised=test_case.raised) - request: NativeModelAnalysisRequest = adapter_rule_request( - test_case=test_case, rule=rule, tmp_path=tmp_path, monkeypatch=monkeypatch - ) + inputs, profile = adapter_rule_project(test_case=test_case, rule=rule, tmp_path=tmp_path) - assert analyze_native_model_sql(request=request).positions + project: CompiledProjectView = assemble_project(inputs=inputs, inference_profile=profile) + + assert any(model.inferred_columns for model in project.models) assert len(rule.calls) >= test_case.expected_minimum_calls, test_case.description assert all( isinstance(value, InferredNullability) @@ -512,15 +474,13 @@ def test_given_adapter_nullability_rule_when_analysing_natively_then_rule_is_cal ids=lambda case: case.description, ) def test_given_failing_adapter_nullability_rule_when_analysing_natively_then_its_error_raises( - test_case: AdapterRuleCallbackTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: AdapterRuleCallbackTestCase, tmp_path: Path ) -> None: rule: RecordingNullabilityRule = RecordingNullabilityRule(raised=test_case.raised) - request: NativeModelAnalysisRequest = adapter_rule_request( - test_case=test_case, rule=rule, tmp_path=tmp_path, monkeypatch=monkeypatch - ) + inputs, profile = adapter_rule_project(test_case=test_case, rule=rule, tmp_path=tmp_path) with pytest.raises(LookupError, match="adapter rule failed"): - _ = analyze_native_model_sql(request=request) + _ = assemble_project(inputs=inputs, inference_profile=profile) assert len(rule.calls) >= test_case.expected_minimum_calls, test_case.description diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 78bfa1f58a..8553dbd30f 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -11,14 +11,12 @@ import pytest -import sqlbuild._native as native_module -from sqlbuild._native import NativeColumnQuery, NativeModelAnalyses, infer_native_query_columns +from sqlbuild._native import NativeColumnQuery, infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.analysis_session._test_types import ( FallbackColumnsTestCase, - MissingSessionAnalysisTestCase, NormalizationRejectionTestCase, TableFunctionColumnsTestCase, ) @@ -111,40 +109,6 @@ def test_given_table_function_when_compiling_then_body_columns_match_declaration assert error == test_case.expected_error -@pytest.mark.parametrize( - "test_case", - [ - MissingSessionAnalysisTestCase( - description="a requested model the session left unanalysed", - expected_message=( - "NativeCompilerError: native model analysis: the session returned no analysis " - "for model 'orders'" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_session_without_model_analysis_when_assembling_then_raises_native_compiler_error( - test_case: MissingSessionAnalysisTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr( - native_module, - "analyze_native_model_sql", - lambda request: NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None), - ) - - with pytest.raises(native_module.NativeCompilerError) as raised: - _ = compiled_project( - project_dir=tmp_path, - files={ - "sqlbuild_project.toml": _PROJECT, - "models/orders.sql": _HEADER + "SELECT 1 AS a", - }, - ) - - assert str(raised.value) == test_case.expected_message - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/rule_engine/fact_digests/_test_types.py b/tests/integration/src/sqlbuild/rule_engine/fact_digests/_test_types.py new file mode 100644 index 0000000000..cd1d6746d4 --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/fact_digests/_test_types.py @@ -0,0 +1,10 @@ +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class FactDigestEditTestCase: + description: str + key: tuple[str, ...] + edited_files: dict[str, str] + expected_digest_changes: bool = True + base_files: dict[str, str] = field(default_factory=dict) diff --git a/tests/integration/src/sqlbuild/rule_engine/fact_digests/helpers.py b/tests/integration/src/sqlbuild/rule_engine/fact_digests/helpers.py new file mode 100644 index 0000000000..80f6d565e0 --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/fact_digests/helpers.py @@ -0,0 +1,64 @@ +from pathlib import Path + +from sqlbuild._native import ( + build_compiled_project, + custom_rule_fact_digest, + discover_project_inputs, +) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.types import CompiledProjectView + +PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Raw orders.\n" + ' expression: "(SELECT 1 AS id, 2 AS customer_id)"\n' + ), + "seeds/regions.csv": "region_id,name\n1,north\n", + "seeds/regions.yml": ( + "seeds:\n - name: regions\n description: Regions.\n columns:\n" + " - name: region_id\n type: INTEGER\n" + " - name: name\n type: VARCHAR\n" + ), + "constants/policy.sql": "CONSTANT (name minimum_amount, value 7);\n", + "enums/order_status.sql": "ENUM (name order_status, members [PAID, OPEN]);\n", + "functions/sql/is_paid.sql": ( + "FUNCTION (\n arguments (status STRING),\n returns BOOLEAN,\n);\n\nstatus = 'PAID'\n" + ), + "models/staging/stg_orders.sql": ( + "MODEL (\n columns (\n order_id (type INTEGER, audits [not_null]),\n ),\n);\n\n" + 'SELECT CAST(id AS INTEGER) AS order_id FROM __source("raw_orders")\n' + ), + "models/marts/order_totals.sql": ( + 'MODEL (description "Order totals");\n\nSELECT o.order_id FROM __ref("stg_orders") AS o\n' + ), + "tests/unit/test_order_totals.sql": ( + "TEST ();\n\nWITH\n__ref__stg_orders AS (SELECT 1 AS order_id),\n" + "__expected__order_totals AS (SELECT 1 AS order_id)\nSELECT 1\n" + ), + "config/owners.yaml": "owners: [orders]\n", +} + + +def write_files(*, root: Path, files: dict[str, str]) -> None: + """Write each project file below `root`.""" + + for relative_path, contents in files.items(): + path: Path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + +def compiled(root: Path) -> CompiledProjectView: + """Compile the DuckDB project at `root`.""" + + return build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=root), adapter=DuckDbAdapter() + ) + + +def fact_digest(*, root: Path, key: tuple[str, ...]) -> str | None: + """Compile the project at `root` and digest one fact of it.""" + + return custom_rule_fact_digest(project=compiled(root), project_dir=root, key=key) diff --git a/tests/integration/src/sqlbuild/rule_engine/fact_digests/test_fact_digests.py b/tests/integration/src/sqlbuild/rule_engine/fact_digests/test_fact_digests.py new file mode 100644 index 0000000000..057b2efafd --- /dev/null +++ b/tests/integration/src/sqlbuild/rule_engine/fact_digests/test_fact_digests.py @@ -0,0 +1,160 @@ +"""Custom-rule fact digests are stable and change whenever the fact's inputs change.""" + +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.rule_engine.fact_digests._test_types import ( + FactDigestEditTestCase, +) +from tests.integration.src.sqlbuild.rule_engine.fact_digests.helpers import ( + PROJECT_FILES, + fact_digest, + write_files, +) + +_STG: str = "models/staging/stg_orders.sql" +_TOTALS: str = "models/marts/order_totals.sql" +_NEW_MODEL: dict[str, str] = { + "models/marts/order_counts.sql": "MODEL ();\n\nSELECT 1 AS order_count\n" +} +_TOTALS_WITH_COLUMN: dict[str, str] = { + _TOTALS: 'MODEL ();\n\nSELECT o.order_id, 1 AS amount FROM __ref("stg_orders") AS o\n' +} +_STG_TYPED: dict[str, str] = { + _STG: ( + "MODEL (\n columns (\n order_id (type BIGINT, audits [not_null]),\n ),\n);\n\n" + 'SELECT CAST(id AS INTEGER) AS order_id FROM __source("raw_orders")\n' + ) +} +_STG_AUDITED: dict[str, str] = { + _STG: ( + "MODEL (\n columns (\n order_id (type INTEGER, audits [not_null, unique]),\n ),\n);\n\n" + 'SELECT CAST(id AS INTEGER) AS order_id FROM __source("raw_orders")\n' + ) +} +_TOTALS_CONTRACT: dict[str, str] = { + _TOTALS: ( + 'MODEL (description "Order totals", contract enforced,\n' + " columns (order_id (type INTEGER)));\n\n" + 'SELECT o.order_id FROM __ref("stg_orders") AS o\n' + ) +} +_STG_ACCEPTED_INTEGERS: dict[str, str] = { + _STG: ( + "MODEL (\n columns (\n order_id (type INTEGER, audits [accepted_values(values [1, 2])]),\n" + " ),\n);\n\n" + 'SELECT CAST(id AS INTEGER) AS order_id FROM __source("raw_orders")\n' + ) +} +_STG_ACCEPTED_STRINGS: dict[str, str] = { + _STG: _STG_ACCEPTED_INTEGERS[_STG].replace("values [1, 2]", "values ['1', '2']") +} +_TOTALS_WITHOUT_REF: dict[str, str] = {_TOTALS: "MODEL ();\n\nSELECT 1 AS order_id\n"} +_TEST_EDITED: dict[str, str] = { + "tests/unit/test_order_totals.sql": ( + "TEST ();\n\nWITH\n__ref__stg_orders AS (SELECT 2 AS order_id),\n" + "__expected__order_totals AS (SELECT 2 AS order_id)\nSELECT 1\n" + ) +} + + +@pytest.mark.parametrize( + "test_case", + ( + FactDigestEditTestCase("project models", ("project.models",), _NEW_MODEL), + FactDigestEditTestCase( + "project sources", + ("project.sources",), + {"sources/raw.yml": PROJECT_FILES["sources/raw.yml"].replace("Raw", "All")}, + ), + FactDigestEditTestCase( + "project seeds", + ("project.seeds",), + {"seeds/regions.yml": PROJECT_FILES["seeds/regions.yml"].replace("Regions", "Areas")}, + ), + FactDigestEditTestCase( + "project functions", + ("project.functions",), + { + "functions/sql/is_paid.sql": PROJECT_FILES["functions/sql/is_paid.sql"].replace( + "'PAID'", "'OPEN'" + ) + }, + ), + FactDigestEditTestCase("project tests", ("project.tests",), _TEST_EDITED), + FactDigestEditTestCase("all tests", ("tests.all",), _TEST_EDITED), + FactDigestEditTestCase("project audits", ("project.audits",), _STG_AUDITED), + FactDigestEditTestCase("all audits", ("audits.all",), _STG_AUDITED), + FactDigestEditTestCase("tree paths", ("tree.paths",), {"docs/notes.yaml": "a: 1\n"}), + FactDigestEditTestCase( + "public enums", + ("declarations.public_enums",), + {"enums/order_status.sql": "ENUM (name order_status, members [PAID, OPEN, VOID]);\n"}, + ), + FactDigestEditTestCase( + "all enums", + ("declarations.enums",), + {"enums/order_status.sql": "ENUM (name order_status, members [PAID]);\n"}, + ), + FactDigestEditTestCase( + "public constants", + ("declarations.public_constants",), + {"constants/policy.sql": "CONSTANT (name minimum_amount, value 8);\n"}, + ), + FactDigestEditTestCase( + "all constants", + ("declarations.constants",), + {"constants/policy.sql": "CONSTANT (name minimum_amount, value 9);\n"}, + ), + FactDigestEditTestCase("model SQL", ("sql.for_model", _TOTALS), _TOTALS_WITH_COLUMN), + FactDigestEditTestCase( + "model dependencies", ("graph.dependencies", _TOTALS), _TOTALS_WITHOUT_REF + ), + FactDigestEditTestCase("model dependents", ("graph.dependents", _STG), _TOTALS_WITHOUT_REF), + FactDigestEditTestCase("declared columns", ("columns.declared", _STG), _STG_TYPED), + FactDigestEditTestCase( + "inferred columns", ("columns.inferred", _TOTALS), _TOTALS_WITH_COLUMN + ), + FactDigestEditTestCase("column names", ("columns.names", _TOTALS), _TOTALS_WITH_COLUMN), + FactDigestEditTestCase("contract", ("contracts.enforced", _TOTALS), _TOTALS_CONTRACT), + FactDigestEditTestCase("model tests", ("tests.for_model", "order_totals"), _TEST_EDITED), + FactDigestEditTestCase("model audits", ("audits.for_model", "stg_orders"), _STG_AUDITED), + FactDigestEditTestCase( + "audit argument numbers become strings", + ("columns.declared", _STG), + _STG_ACCEPTED_STRINGS, + base_files=_STG_ACCEPTED_INTEGERS, + ), + FactDigestEditTestCase("tree children", ("tree.children", "models/marts"), _NEW_MODEL), + FactDigestEditTestCase("tree descendants", ("tree.descendants", "models"), _NEW_MODEL), + FactDigestEditTestCase("tree glob", ("tree.glob", "models/**/*.sql"), _NEW_MODEL), + FactDigestEditTestCase( + "resources under", ("tree.resources_under", "models/marts"), _NEW_MODEL + ), + FactDigestEditTestCase( + "read text", ("tree.read_text", "config/owners.yaml"), {"config/owners.yaml": "a: 2\n"} + ), + FactDigestEditTestCase( + "missing model appears", ("sql.for_model", "models/marts/order_counts.sql"), _NEW_MODEL + ), + ), + ids=lambda case: case.description, +) +def test_given_fact_inputs_when_digesting_then_digest_is_stable_and_follows_edits( + tmp_path: Path, test_case: FactDigestEditTestCase +) -> None: + write_files(root=tmp_path, files={**PROJECT_FILES, **test_case.base_files}) + first: str | None = fact_digest(root=tmp_path, key=test_case.key) + again: str | None = fact_digest(root=tmp_path, key=test_case.key) + write_files(root=tmp_path, files=test_case.edited_files) + edited: str | None = fact_digest(root=tmp_path, key=test_case.key) + + assert first is not None + assert again == first + assert edited is not None + assert (edited != first) is test_case.expected_digest_changes + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py index ddaa1811cd..67ba931bbd 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/helpers.py @@ -12,20 +12,80 @@ CompiledProject, Model, build_compiled_project, + custom_rule_fact_digest, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine import fact_replay -from sqlbuild.rule_engine._helpers.engine.fact_replay import ( - fact_outcome_digest, - shared_fact_encoder, -) from sqlbuild.rule_engine.classes.project_tree import public_model -from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views +from sqlbuild.rule_engine.constants import ( + FACT_AUDITS_ALL, + FACT_AUDITS_FOR_MODEL, + FACT_COLUMNS_DECLARED, + FACT_COLUMNS_INFERRED, + FACT_COLUMNS_NAMES, + FACT_CONTRACTS_ENFORCED, + FACT_CONTRACTS_GRAIN, + FACT_DECLARATIONS_CONSTANTS, + FACT_DECLARATIONS_ENUMS, + FACT_DECLARATIONS_PUBLIC_CONSTANTS, + FACT_DECLARATIONS_PUBLIC_ENUMS, + FACT_GRAPH_DEPENDENCIES, + FACT_GRAPH_DEPENDENTS, + FACT_PROJECT_AUDITS, + FACT_PROJECT_FUNCTIONS, + FACT_PROJECT_MODELS, + FACT_PROJECT_SEEDS, + FACT_PROJECT_SOURCES, + FACT_PROJECT_TESTS, + FACT_SQL_FOR_MODEL, + FACT_TESTS_ALL, + FACT_TESTS_FOR_MODEL, + FACT_TREE_CHILDREN, + FACT_TREE_DESCENDANTS, + FACT_TREE_GLOB, + FACT_TREE_PATHS, + FACT_TREE_READ_TEXT, + FACT_TREE_RELATIVE_PARTS, + FACT_TREE_RESOURCES_UNDER, +) from sqlbuild.rule_engine.types import FactKey _TREE_ARGUMENTS: tuple[str, ...] = ("models/marts/order_totals.sql", "models") +_GLOBAL_FACTS: tuple[str, ...] = ( + FACT_PROJECT_MODELS, + FACT_PROJECT_SOURCES, + FACT_PROJECT_SEEDS, + FACT_PROJECT_FUNCTIONS, + FACT_PROJECT_TESTS, + FACT_PROJECT_AUDITS, + FACT_TREE_PATHS, + FACT_TESTS_ALL, + FACT_AUDITS_ALL, + FACT_DECLARATIONS_PUBLIC_ENUMS, + FACT_DECLARATIONS_PUBLIC_CONSTANTS, + FACT_DECLARATIONS_ENUMS, + FACT_DECLARATIONS_CONSTANTS, +) +_MODEL_PATH_FACTS: tuple[str, ...] = ( + FACT_SQL_FOR_MODEL, + FACT_GRAPH_DEPENDENCIES, + FACT_GRAPH_DEPENDENTS, + FACT_COLUMNS_DECLARED, + FACT_COLUMNS_INFERRED, + FACT_COLUMNS_NAMES, + FACT_CONTRACTS_ENFORCED, + FACT_CONTRACTS_GRAIN, +) +_MODEL_NAME_FACTS: tuple[str, ...] = (FACT_TESTS_FOR_MODEL, FACT_AUDITS_FOR_MODEL) +_TEXT_FACTS: tuple[tuple[str, int], ...] = ( + (FACT_TREE_CHILDREN, 1), + (FACT_TREE_DESCENDANTS, 1), + (FACT_TREE_GLOB, 1), + (FACT_TREE_RELATIVE_PARTS, 2), + (FACT_TREE_RESOURCES_UNDER, 1), + (FACT_TREE_READ_TEXT, 1), +) class OrdersReferenceResolver: @@ -86,30 +146,18 @@ def every_fact_key(project: CompiledProjectView) -> tuple[FactKey, ...]: models: tuple[Model, ...] = tuple(public_model(model) for model in project.models) return ( - *((fact,) for fact in fact_replay._GLOBAL_FACTS), - *( - (fact, model.path.as_posix()) - for fact, model in product(fact_replay._MODEL_PATH_FACTS, models) - ), - *((fact, model.name) for fact, model in product(fact_replay._MODEL_NAME_FACTS, models)), - *( - (fact, *_TREE_ARGUMENTS[: fact_replay._TEXT_FACT_ARITY[fact]]) - for fact in fact_replay._TEXT_FACTS - ), + *((fact,) for fact in _GLOBAL_FACTS), + *((fact, model.path.as_posix()) for fact, model in product(_MODEL_PATH_FACTS, models)), + *((fact, model.name) for fact, model in product(_MODEL_NAME_FACTS, models)), + *((fact, *_TREE_ARGUMENTS[:arity]) for fact, arity in _TEXT_FACTS), ) -def fact_digests(*, project: CompiledProjectView, root: Path) -> dict[FactKey, tuple[str, str]]: - """Digest every fact as hosts observe it and as whole-project digests encode it.""" +def fact_digests(*, project: CompiledProjectView, root: Path) -> dict[FactKey, str | None]: + """Digest every fact the custom-rule cache can key a result on.""" - views: RuleFactViews = build_rule_fact_views( - project=project, project_dir=root, dialect="duckdb" - ) return { - key: ( - fact_outcome_digest(views=views, key=key), - fact_outcome_digest(views=views, key=key, shared=shared_fact_encoder()), - ) + key: custom_rule_fact_digest(project=project, project_dir=root, key=key) for key in every_fact_key(project) } diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py index de14299b49..69a2e08d13 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py @@ -117,8 +117,8 @@ def test_given_compiled_project_when_trimming_host_payload_then_every_fact_is_un trimmed: CompiledProjectView = custom_host_payload(project) monkeypatch.undo() - full_digests: dict[FactKey, tuple[str, str]] = fact_digests(project=project, root=tmp_path) - trimmed_digests: dict[FactKey, tuple[str, str]] = fact_digests(project=trimmed, root=tmp_path) + full_digests: dict[FactKey, str | None] = fact_digests(project=project, root=tmp_path) + trimmed_digests: dict[FactKey, str | None] = fact_digests(project=trimmed, root=tmp_path) assert trimmed_digests == full_digests assert tuple(map(public_model, trimmed.models)) == tuple(map(public_model, project.models)) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py index a7ecb72a6d..a79e4f3baf 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rule_reads.py @@ -1,15 +1,13 @@ """Custom-rule results are reused only when every fact the invocation read is unchanged.""" -from collections.abc import Callable +import os from pathlib import Path -from typing import Any import pytest import sqlbuild._native as native_module from sqlbuild._native import CustomRulesOutcome from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.rule_engine._helpers.engine import fact_replay from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( CustomRuleProjectFileTestCase, CustomRuleReadTrackingTestCase, @@ -26,6 +24,18 @@ _CUSTOMERS_SQL: str = "SELECT 1 AS customer_id" _OWNERS_BEFORE: str = "owners: [customers]\n" _OWNERS_AFTER: str = "owners: [orders]\n" +# Rule hosts report every filesystem read as changed since the rule observed it. +_STALE_OBSERVATION_SITECUSTOMIZE: str = """from sqlbuild.rule_engine._helpers.host import tracked_facts + +_digest = tracked_facts._digest_or_none + + +def _stale(*, value=None, error=None): + return _digest(value=("stale", value)) if error is None else _digest(error=error) + + +tracked_facts._digest_or_none = _stale +""" _FLAG_ORDERS: str = ( ' return [ctx.finding(subject=model)] if model.name == "orders" and {condition} else []' ) @@ -253,17 +263,15 @@ def test_given_file_edited_while_rules_run_when_rerunning_then_new_contents_are_ project: CompiledProjectView = two_model_project( customers_sql=_CUSTOMERS_SQL, customers_config={} ) - digest: Callable[..., str] = fact_replay.fact_outcome_digest - - def edit_then_digest(**kwargs: Any) -> str: - _ = owners.write_text(_OWNERS_AFTER, encoding="utf-8") - return digest(**kwargs) - - monkeypatch.setattr(fact_replay, "fact_outcome_digest", edit_then_digest) + site: Path = tmp_path.parent / f"{tmp_path.name}-site" + site.mkdir() + _ = (site / "sitecustomize.py").write_text(_STALE_OBSERVATION_SITECUSTOMIZE, encoding="utf-8") + monkeypatch.setenv("PYTHONPATH", os.pathsep.join((str(site), os.environ.get("PYTHONPATH", "")))) during_edit: CustomRulesOutcome = evaluate_cached_custom_rules( project=project, project_dir=tmp_path, cache_enabled=True ) monkeypatch.undo() + _ = owners.write_text(_OWNERS_AFTER, encoding="utf-8") rerun: CustomRulesOutcome = evaluate_cached_custom_rules( project=project, project_dir=tmp_path, cache_enabled=True ) diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/_test_types.py deleted file mode 100644 index 8e009b8338..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/_test_types.py +++ /dev/null @@ -1,16 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class SourceDigestTestCase: - description: str - edited_source_names: tuple[str, ...] - edited_contents: str - expected_equal: bool - - -@dataclass(frozen=True) -class ModelFactDigestTestCase: - description: str - fact: str - expected_host_digest_match: bool diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py deleted file mode 100644 index ab577d63c1..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ /dev/null @@ -1,67 +0,0 @@ -from pathlib import Path - -from sqlbuild._native import ( - CompiledObjectKey, - CompiledSource, - DiscoveredSourceFile, - SourceEntry, - project_with, -) -from sqlbuild.compiler.compile.types import ( - CompiledProjectView, - CompiledResourceType, - CompiledSourceView, -) -from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views -from tests.unit.src.sqlbuild.rule_engine.main.evaluate.helpers import build_project - -SOURCES_CONTENTS: str = "sources:\n - name: raw_orders\n - name: raw_customers\n" -SOURCE_NAMES: tuple[str, ...] = ("raw_orders", "raw_customers") -_SOURCES_PATH: Path = Path("sources/raw.yml") - - -def shared_file_views(*, tmp_path: Path, names: tuple[str, ...], contents: str) -> RuleFactViews: - """Return Rule fact views over sources that share one declaration-file object.""" - - declared: DiscoveredSourceFile = _source_file(tmp_path=tmp_path, names=names, contents=contents) - return _views(tmp_path=tmp_path, files=tuple(declared for _ in names), names=names) - - -def copied_file_views(*, tmp_path: Path, names: tuple[str, ...], contents: str) -> RuleFactViews: - """Return Rule fact views over sources that each hold an equal copy of their file.""" - - files: tuple[DiscoveredSourceFile, ...] = tuple( - _source_file(tmp_path=tmp_path, names=names, contents=contents) for _ in names - ) - return _views(tmp_path=tmp_path, files=files, names=names) - - -def _source_file(*, tmp_path: Path, names: tuple[str, ...], contents: str) -> DiscoveredSourceFile: - return DiscoveredSourceFile( - file_path=tmp_path / _SOURCES_PATH, - relative_path=_SOURCES_PATH, - contents=contents, - source_entries=tuple(SourceEntry(name=name) for name in names), - ) - - -def _views( - *, tmp_path: Path, files: tuple[DiscoveredSourceFile, ...], names: tuple[str, ...] -) -> RuleFactViews: - sources: tuple[CompiledSourceView, ...] = tuple( - CompiledSource( - key=CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=name), - deps=(), - name=name, - source_entry=SourceEntry(name=name), - source_file=declared, - ) - for name, declared in zip(names, files, strict=True) - ) - project: CompiledProjectView = project_with( - project=build_project( - name="orders", relative_path="models/orders.sql", sql="SELECT 1", config_values={} - ), - sources=sources, - ) - return build_rule_fact_views(project=project, project_dir=tmp_path, dialect="duckdb") diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/test_shared_fact_encoder.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/test_shared_fact_encoder.py deleted file mode 100644 index 44aac62117..0000000000 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/test_shared_fact_encoder.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Whole-project facts digest by content, however often resources share compiler objects.""" - -from pathlib import Path - -import pytest - -from sqlbuild.rule_engine._helpers.engine.fact_replay import _fact_value as fact_value -from sqlbuild.rule_engine._helpers.engine.fact_replay import ( - fact_outcome_digest, - fact_value_digest, - shared_fact_encoder, -) -from sqlbuild.rule_engine.classes.rule_context import RuleFactViews -from sqlbuild.rule_engine.constants import ( - FACT_COLUMNS_DECLARED, - FACT_PROJECT_SOURCES, - FACT_SQL_FOR_MODEL, -) -from sqlbuild.rule_engine.types import FactKey -from tests.unit.src.sqlbuild.rule_engine.classes.shared_fact_encoder._test_types import ( - ModelFactDigestTestCase, - SourceDigestTestCase, -) -from tests.unit.src.sqlbuild.rule_engine.classes.shared_fact_encoder.helpers import ( - SOURCE_NAMES, - SOURCES_CONTENTS, - copied_file_views, - shared_file_views, -) - - -@pytest.mark.parametrize( - "test_case", - ( - SourceDigestTestCase( - description="unchanged declarations", - edited_source_names=SOURCE_NAMES, - edited_contents=SOURCES_CONTENTS, - expected_equal=True, - ), - SourceDigestTestCase( - description="shared file contents edited", - edited_source_names=SOURCE_NAMES, - edited_contents=SOURCES_CONTENTS + "# reviewed\n", - expected_equal=False, - ), - SourceDigestTestCase( - description="source renamed", - edited_source_names=("raw_orders", "raw_clients"), - edited_contents=SOURCES_CONTENTS, - expected_equal=False, - ), - SourceDigestTestCase( - description="source added", - edited_source_names=(*SOURCE_NAMES, "raw_products"), - edited_contents=SOURCES_CONTENTS, - expected_equal=False, - ), - ), - ids=lambda case: case.description, -) -def test_given_sources_sharing_a_file_when_digesting_then_digest_follows_content_only( - tmp_path: Path, test_case: SourceDigestTestCase -) -> None: - shared: RuleFactViews = shared_file_views( - tmp_path=tmp_path, names=SOURCE_NAMES, contents=SOURCES_CONTENTS - ) - copied: RuleFactViews = copied_file_views( - tmp_path=tmp_path, - names=test_case.edited_source_names, - contents=test_case.edited_contents, - ) - - before: str = fact_outcome_digest( - views=shared, key=(FACT_PROJECT_SOURCES,), shared=shared_fact_encoder() - ) - after: str = fact_outcome_digest( - views=copied, key=(FACT_PROJECT_SOURCES,), shared=shared_fact_encoder() - ) - - assert (before == after) is test_case.expected_equal - - -@pytest.mark.parametrize( - "test_case", - ( - ModelFactDigestTestCase("expanded and authored SQL", FACT_SQL_FOR_MODEL, True), - ModelFactDigestTestCase("declared columns", FACT_COLUMNS_DECLARED, True), - ), - ids=lambda case: case.description, -) -def test_given_model_fact_when_digesting_with_shared_encoder_then_matches_host_observed_digest( - tmp_path: Path, test_case: ModelFactDigestTestCase -) -> None: - views: RuleFactViews = shared_file_views( - tmp_path=tmp_path, names=SOURCE_NAMES, contents=SOURCES_CONTENTS - ) - key: FactKey = (test_case.fact, views.project.models[0].path.as_posix()) - - digest: str = fact_outcome_digest(views=views, key=key, shared=shared_fact_encoder()) - - assert (digest == fact_value_digest(fact_value(views=views, key=key))) is ( - test_case.expected_host_digest_match - ) - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-vv"])) diff --git a/tests/unit/src/sqlbuild/runtime/user_code/__init__.py b/tests/unit/src/sqlbuild/runtime/user_code/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/runtime/user_code/main/__init__.py b/tests/unit/src/sqlbuild/runtime/user_code/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/runtime/user_code/main/_test_types.py b/tests/unit/src/sqlbuild/runtime/user_code/main/_test_types.py new file mode 100644 index 0000000000..07926e0966 --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/user_code/main/_test_types.py @@ -0,0 +1,12 @@ +"""Parameterized user-code loading test cases.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class StaleBytecodeTestCase: + """One project file edited without changing its size or mtime after bytecode was cached.""" + + description: str + edited_file: str + expected_value: str diff --git a/tests/unit/src/sqlbuild/runtime/user_code/main/helpers.py b/tests/unit/src/sqlbuild/runtime/user_code/main/helpers.py new file mode 100644 index 0000000000..dfc9cc70df --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/user_code/main/helpers.py @@ -0,0 +1,59 @@ +"""Helpers for user-code loading tests.""" + +import os +import py_compile +import subprocess +import sys +from pathlib import Path + +_PROJECT_FILES: dict[str, str] = { + "rules/__init__.py": "", + "rules/limits.py": "MAX_NAME_LENGTH: int = 40\n", + "rules/naming.py": ( + "from rules.limits import MAX_NAME_LENGTH\n\nRULE_LIMIT: int = MAX_NAME_LENGTH * 1\n" + ), +} +_LOAD_SCRIPT: str = ( + "import sys\n" + "from importlib.util import module_from_spec\n" + "from pathlib import Path\n" + "from sqlbuild.runtime.user_code.main.user_code_spec import user_code_spec\n" + "root = Path(sys.argv[1])\n" + "sys.path.insert(0, str(root))\n" + "spec = user_code_spec(name='rule_naming', file_path=root / 'rules' / 'naming.py', root=root)\n" + "module = module_from_spec(spec)\n" + "spec.loader.exec_module(module)\n" + "print(module.RULE_LIMIT)\n" +) + + +def write_project_with_cached_bytecode(*, root: Path) -> None: + """Write the project files and cache standard bytecode for each, as a previous run would.""" + + for relative_path, contents in _PROJECT_FILES.items(): + path: Path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + _ = py_compile.compile(str(path), doraise=True) + + +def edit_keeping_size_and_mtime(*, root: Path, relative_path: str, old: str, new: str) -> None: + """Apply a same-size edit and restore the file's previous timestamps.""" + + path: Path = root / relative_path + before: os.stat_result = path.stat() + contents: str = path.read_text(encoding="utf-8") + _ = path.write_text(contents.replace(old, new, 1), encoding="utf-8") + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + + +def load_rule_limit(*, root: Path) -> str: + """Load the rule module through the user-code loader in a fresh interpreter.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", _LOAD_SCRIPT, str(root)], + capture_output=True, + text=True, + check=True, + ) + return completed.stdout.strip() diff --git a/tests/unit/src/sqlbuild/runtime/user_code/main/test_user_code_spec.py b/tests/unit/src/sqlbuild/runtime/user_code/main/test_user_code_spec.py new file mode 100644 index 0000000000..4fd798537f --- /dev/null +++ b/tests/unit/src/sqlbuild/runtime/user_code/main/test_user_code_spec.py @@ -0,0 +1,46 @@ +"""Project Python always loads from current source, never from stale cached bytecode.""" + +from pathlib import Path + +import pytest + +from tests.unit.src.sqlbuild.runtime.user_code.main._test_types import StaleBytecodeTestCase +from tests.unit.src.sqlbuild.runtime.user_code.main.helpers import ( + edit_keeping_size_and_mtime, + load_rule_limit, + write_project_with_cached_bytecode, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StaleBytecodeTestCase( + description="file loaded directly", edited_file="rules/naming.py", expected_value="80" + ), + StaleBytecodeTestCase( + description="module imported by name", + edited_file="rules/limits.py", + expected_value="10", + ), + ], + ids=lambda case: case.description, +) +def test_given_same_size_edit_with_cached_bytecode_when_loading_then_current_source_runs( + test_case: StaleBytecodeTestCase, tmp_path: Path +) -> None: + write_project_with_cached_bytecode(root=tmp_path) + replacements: dict[str, tuple[str, str]] = { + "rules/naming.py": ("MAX_NAME_LENGTH * 1", "MAX_NAME_LENGTH * 2"), + "rules/limits.py": ("= 40", "= 10"), + } + old, new = replacements[test_case.edited_file] + edit_keeping_size_and_mtime( + root=tmp_path, relative_path=test_case.edited_file, old=old, new=new + ) + + assert load_rule_limit(root=tmp_path) == test_case.expected_value + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"]))