From 10019cf23622ab31fd2e3b196bea5770b8c7aaa0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:58:46 +0100 Subject: [PATCH 1/7] perf: discover SQL model files as native records the model loop reads directly --- .../_helpers/compiled_project/model_loop.rs | 47 ++- .../compiled_project/model_loop_header.rs | 11 +- .../compiled_project/model_loop_host.rs | 91 ++++-- .../_helpers/discovery/header_declarations.rs | 4 +- .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/model_file_record.rs | 292 ++++++++++++++++++ .../_helpers/discovery/project_files.rs | 92 +++++- src/sqlbuild/_native.pyi | 42 ++- .../compile/_helpers/attachment/core.py | 16 +- .../discovery/_helpers/native/model_files.py | 73 ++--- src/sqlbuild/compiler/discovery/models.py | 18 +- src/sqlbuild/compiler/discovery/types.py | 1 + src/sqlbuild/compiler/frontier/constants.py | 15 + .../sqlbuild/compiler/model_config/helpers.py | 11 +- 14 files changed, 576 insertions(+), 138 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index c19639c182..49237456cb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -19,8 +19,8 @@ use crate::bindings::_helpers::compiled_project::model_loop_hooks::{ ExpandedHooks, HookModel, SqlHookClasses, expanded_hooks, }; use crate::bindings::_helpers::compiled_project::model_loop_host::{ - FinishedModel, InterpolationRow, ModelLoopRow, RecordClasses, RecordInputs, ReferenceRow, - finished_record, + FinishedModel, InterpolationRow, ModelLoopInput, ModelLoopRow, ModelPaths, RecordClasses, + RecordInputs, ReferenceRow, finished_record, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ analysis_override, hook_presence, reference_row, @@ -54,6 +54,7 @@ use crate::bindings::_helpers::compiled_project::source_views::{ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ AttachedSqlTests, SqlTestSettings, TestFileRow, TestScope, }; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use crate::bindings::_helpers::render::declaration_expansion::{ @@ -241,9 +242,13 @@ impl NativeModelLoop { slf: &Bound<'py, Self>, project: Bound<'py, NativeCompiledProject>, host: Bound<'py, PyAny>, - models: Vec, + models: Vec, ) -> PyResult>> { let py: Python<'py> = slf.py(); + let models: Vec = models + .into_iter() + .map(|input| ModelLoopRow::new(py, input)) + .collect::>()?; project .borrow_mut() .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); @@ -274,18 +279,27 @@ impl NativeModelLoop { let visibility: ModelVisibility<'py> = ModelVisibility::default(); let signatures: HookSignatures = HookSignatures::default(); for row in models { - let ModelLoopRow((name, _, relative_path, file_path), header, _, _, _, _, _) = &row; - let (_, header_values, enums, constants) = header; + let ModelPaths { + name, + relative_path, + file_path, + .. + } = &row.paths; + let record: &NativeModelFile = row.model_file.get(); + let (enums, constants) = record.declaration_tuples(py); let visible: VisibleDeclarations<'py> = visibility.for_model( &context, &VisibilityModel { name, file_path, - local: (enums.bind(py), constants.bind(py)), + local: (enums.as_any(), constants.as_any()), }, )?; - let config: Bound<'py, PyAny> = - configs.effective(&context, header_values.bind(py), (name, relative_path))?; + let config: Bound<'py, PyAny> = configs.effective( + &context, + record.header_value_dict(py), + (name, relative_path), + )?; let model: LoopModel<'_, 'py> = LoopModel { row: &row, config, @@ -454,15 +468,20 @@ impl NativeModelLoop { signatures, visibility, } = model; - let ModelLoopRow( - (name, posix_path, relative_path, file_path), - (locations, _, _, _), + let ModelLoopRow { + paths: + ModelPaths { + name, + posix_path, + relative_path, + file_path, + }, authored_query, contents, scan, interpolation, - _, - ) = row; + model_file, + } = row; let values: Bound<'py, PyDict> = config.getattr(VALUES_ATTRIBUTE)?.downcast_into()?; let schema: Option> = model_schema(context, &values, name)?; let hooked: bool = hook_presence(&values)?; @@ -620,7 +639,7 @@ impl NativeModelLoop { let source: HeaderSource<'_, 'py> = HeaderSource { name, values: &values, - locations: locations.bind(py), + model_file: model_file.get(), relative_path, visible: (&visible.declarations, &visible.consumer), schema: schema.as_ref(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index b21ececf28..dc7a179449 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -21,6 +21,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_schema::{ ModelSchema, merged_columns, schema_enums, }; use crate::bindings::_helpers::compiled_project::project::schema_fact; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; use crate::bindings::_helpers::model_config::config_errors::native_config_error; @@ -41,7 +42,7 @@ const ENUMS_FIELD: &str = "enums"; pub(crate) struct HeaderSource<'a, 'py> { pub(crate) name: &'a str, pub(crate) values: &'a Bound<'py, PyDict>, - pub(crate) locations: &'a Bound<'py, PyDict>, + pub(crate) model_file: &'a NativeModelFile, pub(crate) relative_path: &'a str, /// The model's visible declarations and the identity its usages are recorded for. pub(crate) visible: (&'a Bound<'py, PyAny>, &'a Bound<'py, PyAny>), @@ -99,8 +100,12 @@ pub(crate) fn header_schema<'py>( let parsed_columns = metadata .columns .map_err(|error| raised_config_error(py, context, error))?; - let local_columns: Bound<'py, PyTuple> = - column_tuple(py, context.contracts, &parsed_columns, source.locations)?; + let local_columns: Bound<'py, PyTuple> = column_tuple( + py, + context.contracts, + &parsed_columns, + &source.model_file.header_locations(py)?, + )?; let columns: Bound<'py, PyTuple> = match source.schema { Some(schema) => merged_columns(context, schema, &local_columns, model)?, None => local_columns, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 5d4cc109a3..55a54a0bcb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -21,6 +21,7 @@ use crate::bindings::_helpers::compiled_project::project::{ use crate::bindings::_helpers::compiled_project::project::{ DeclarationPy, config_values, declarations, }; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::render::declaration_expansion::{ ExpandedDeclarations, ScannedReference, span_rows, }; @@ -35,21 +36,59 @@ const CONSTANTS_ARGUMENT: &str = "constants"; const LOCAL_ENUMS_ATTRIBUTE: &str = "local_enums"; const LOCAL_CONSTANTS_ATTRIBUTE: &str = "local_constants"; const NAME_ATTRIBUTE: &str = "name"; +const STEM_ATTRIBUTE: &str = "stem"; +const AS_POSIX_METHOD: &str = "as_posix"; -/// `((name, POSIX path, relative path, file path), column locations, query, contents, scan)`. -#[derive(FromPyObject)] -pub(crate) struct ModelLoopRow( - pub(crate) (String, String, String, String), - /// Header column locations, header values, and the model's own enums and constants. - pub(crate) (Py, Py, Py, Py), - pub(crate) String, - pub(crate) String, - pub(crate) Option<(Vec, Option)>, +/// What the host hands the loop for one model: its file, declaration scan and interpolation. +pub(crate) type ModelLoopInput = ( + Py, + Option, + InterpolationRow, +); +/// A model's scanned declaration references and the scan's stop marker. +pub(crate) type DeclarationScan = (Vec, Option); + +/// The names a model's file is known by. +pub(crate) struct ModelPaths { + pub(crate) name: String, + pub(crate) posix_path: String, + pub(crate) relative_path: String, + pub(crate) file_path: String, +} + +/// One model the loop builds: its discovered file and what the host prepared for it. +pub(crate) struct ModelLoopRow { + pub(crate) paths: ModelPaths, + pub(crate) authored_query: String, + pub(crate) contents: String, + pub(crate) scan: Option, /// The `@@`-interpolated query, its template reads, its error, and its substitution spans. - pub(crate) InterpolationRow, + pub(crate) interpolation: InterpolationRow, /// The discovered model file, owned by the discovery inputs. - pub(crate) Py, -); + pub(crate) model_file: Py, +} + +impl ModelLoopRow { + pub(crate) fn new(py: Python<'_>, input: ModelLoopInput) -> PyResult { + let (model_file, scan, interpolation) = input; + let record: &NativeModelFile = model_file.get(); + let (file_path, relative_path) = record.paths(py); + let (query_sql, contents) = record.texts(py); + Ok(Self { + paths: ModelPaths { + name: file_path.getattr(STEM_ATTRIBUTE)?.extract()?, + posix_path: relative_path.call_method0(AS_POSIX_METHOD)?.extract()?, + relative_path: relative_path.str()?.to_str()?.to_owned(), + file_path: file_path.str()?.to_str()?.to_owned(), + }, + authored_query: query_sql.to_str()?.to_owned(), + contents: contents.to_str()?.to_owned(), + scan, + interpolation, + model_file, + }) + } +} /// `(interpolated query, template reads, error, substitution spans)`. pub(crate) type InterpolationRow = (String, Vec<(String, String)>, Option, Vec); /// `(kind, name, package, call argument count)`. @@ -96,15 +135,19 @@ pub(crate) fn finished_record<'py>( finished: FinishedModel<'py>, ) -> PyResult<(ModelInputRecord, CompiledModelFacts)> { let py: Python<'py> = inputs.visible.py(); - let ModelLoopRow( - (name, relative_path, _, _), - _, - authored_query_sql, - authored_sql, - _, - (interpolated_sql, _, _, interpolation_spans), + let ModelLoopRow { + paths: + ModelPaths { + name, + posix_path: relative_path, + .. + }, + authored_query: authored_query_sql, + contents: authored_sql, + interpolation: (interpolated_sql, _, _, interpolation_spans), model_file, - ) = row; + .. + } = row; let stripped: Bound<'py, PyDict> = stripped_values(py, &finished.values)?; let mut usages: Vec> = finished.declarations.usages; if let Some(schema) = &finished.header.schema { @@ -180,11 +223,7 @@ pub(crate) fn finished_record<'py>( type_enforcement: schema.type_enforcement, model_schema: schema.model_schema.clone(), }); - let discovered_query: Py = model_file - .bind(py) - .getattr("query_sql")? - .downcast_into::()? - .unbind(); + let discovered_query: Py = model_file.get().texts(py).0.clone().unbind(); let authored: Py = shared_sql(py, &authored_query_sql, &[&discovered_query])?; let macro_source: Py = shared_sql(py, &finished.declarations.spliced.sql, &[&authored])?; @@ -195,7 +234,7 @@ pub(crate) fn finished_record<'py>( &[&authored, ¯o_source, &expanded], )?; let record: ModelInputRecord = ModelInputRecord { - model_file: model_file.clone_ref(py), + model_file: model_file.clone_ref(py).into_any(), config: config.unbind(), query_sql: query, macro_source_sql: macro_source, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs index 06d34a6a1d..f177fcbbee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs @@ -114,11 +114,11 @@ fn optional_raw( .transpose() } -type HeaderDeclarations<'py> = (Bound<'py, PyTuple>, Bound<'py, PyTuple>); +pub(crate) type HeaderDeclarations<'py> = (Bound<'py, PyTuple>, Bound<'py, PyTuple>); /// `parse_model_enum_declarations` then `parse_model_constant_declarations` for one model. #[pyfunction] -fn model_header_declarations<'py>( +pub(crate) fn model_header_declarations<'py>( py: Python<'py>, header: (Option>, Option>), model_name: String, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 40244e1477..e965bf94db 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod globbing; pub(crate) mod header_declarations; pub(crate) mod key_checks; pub(crate) mod locations; +pub(crate) mod model_file_record; pub(crate) mod project_files; pub(crate) mod registration; pub(crate) mod sql_test_blocks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs new file mode 100644 index 0000000000..bb98e62dbe --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -0,0 +1,292 @@ +//! A discovered SQL model file as a native record whose Python attributes are built on demand. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_discovery::models::LineColumnSpan; + +const SOURCE_LOCATION_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SOURCE_LOCATION_CLASS: &str = "SourceLocation"; +const FIELD_NAMES: [&str; 10] = [ + "file_path", + "relative_path", + "contents", + "header_values", + "header_column_locations", + "output_column_locations", + "query_sql", + "enum_declarations", + "constant_declarations", + "extract_implicit_alias_columns", +]; + +/// Authored locations: native rows until Python reads them, then a cached dict. +struct Locations { + rows: Vec<(String, LineColumnSpan)>, + projected: PyOnceLock>, +} + +impl Locations { + fn from_rows(rows: Vec<(String, LineColumnSpan)>) -> Self { + Self { + rows, + projected: PyOnceLock::new(), + } + } + + fn from_dict(py: Python<'_>, dict: Py) -> Self { + let projected: PyOnceLock> = PyOnceLock::new(); + let _ = projected.set(py, dict); + Self { + rows: Vec::new(), + projected, + } + } + + /// Python's `{name: SourceLocation(...)}`: a repeated name keeps its first position and last value. + fn dict<'py>(&self, py: Python<'py>, path: &Py) -> PyResult> { + Ok(self + .projected + .get_or_try_init(py, || { + let class: Bound<'py, PyAny> = py + .import(SOURCE_LOCATION_MODULE)? + .getattr(SOURCE_LOCATION_CLASS)?; + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (name, span) in &self.rows { + let location: Bound<'py, PyAny> = class.call1(( + path.bind(py), + span.line, + span.column, + span.end_line, + span.end_column, + ))?; + dict.set_item(name, location)?; + } + Ok::<_, pyo3::PyErr>(dict.unbind()) + })? + .bind(py) + .clone()) + } +} + +/// A discovered SQL model file and its raw contents. +#[pyclass(module = "sqlbuild._native", name = "DiscoveredSqlModelFile", frozen)] +pub(crate) struct NativeModelFile { + file_path: Py, + relative_path: Py, + contents: Py, + header_values: Py, + header_column_locations: Locations, + output_column_locations: Locations, + query_sql: Py, + enum_declarations: Py, + constant_declarations: Py, + extract_implicit_alias_columns: bool, +} + +/// What native discovery read for one model file, before its locations are projected. +pub(crate) struct ModelFileParts { + pub(crate) file_path: Py, + pub(crate) relative_path: Py, + pub(crate) contents: Py, + pub(crate) header_values: Py, + pub(crate) header_column_locations: Vec<(String, LineColumnSpan)>, + pub(crate) output_column_locations: Vec<(String, LineColumnSpan)>, + pub(crate) query_sql: Py, + pub(crate) declarations: (Py, Py), + pub(crate) extract_implicit_alias_columns: bool, +} + +impl NativeModelFile { + pub(crate) fn discovered(parts: ModelFileParts) -> Self { + let (enum_declarations, constant_declarations) = parts.declarations; + Self { + file_path: parts.file_path, + relative_path: parts.relative_path, + contents: parts.contents, + header_values: parts.header_values, + header_column_locations: Locations::from_rows(parts.header_column_locations), + output_column_locations: Locations::from_rows(parts.output_column_locations), + query_sql: parts.query_sql, + enum_declarations, + constant_declarations, + extract_implicit_alias_columns: parts.extract_implicit_alias_columns, + } + } + + pub(crate) fn header_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_column_locations.dict(py, &self.relative_path) + } + + pub(crate) fn header_value_dict<'py>(&self, py: Python<'py>) -> &Bound<'py, PyDict> { + self.header_values.bind(py) + } + + pub(crate) fn declaration_tuples<'py>( + &self, + py: Python<'py>, + ) -> (&Bound<'py, PyTuple>, &Bound<'py, PyTuple>) { + ( + self.enum_declarations.bind(py), + self.constant_declarations.bind(py), + ) + } + + pub(crate) fn paths<'py>(&self, py: Python<'py>) -> (&Bound<'py, PyAny>, &Bound<'py, PyAny>) { + (self.file_path.bind(py), self.relative_path.bind(py)) + } + + pub(crate) fn texts<'py>( + &self, + py: Python<'py>, + ) -> (&Bound<'py, PyString>, &Bound<'py, PyString>) { + (self.query_sql.bind(py), self.contents.bind(py)) + } + + fn fields<'py>(&self, py: Python<'py>) -> PyResult>> { + Ok(vec![ + self.file_path.bind(py).clone(), + self.relative_path.bind(py).clone(), + self.contents.bind(py).clone().into_any(), + self.header_values.bind(py).clone().into_any(), + self.header_locations(py)?.into_any(), + self.output_column_locations + .dict(py, &self.relative_path)? + .into_any(), + self.query_sql.bind(py).clone().into_any(), + self.enum_declarations.bind(py).clone().into_any(), + self.constant_declarations.bind(py).clone().into_any(), + pyo3::types::PyBool::new(py, self.extract_implicit_alias_columns) + .to_owned() + .into_any(), + ]) + } +} + +#[pymethods] +impl NativeModelFile { + #[new] + #[pyo3(signature = ( + file_path, + relative_path, + contents, + header_values, + header_column_locations, + output_column_locations, + query_sql, + enum_declarations = None, + constant_declarations = None, + extract_implicit_alias_columns = true, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + py: Python<'_>, + file_path: Py, + relative_path: Py, + contents: Py, + header_values: Py, + header_column_locations: Py, + output_column_locations: Py, + query_sql: Py, + enum_declarations: Option>, + constant_declarations: Option>, + extract_implicit_alias_columns: bool, + ) -> Self { + Self { + file_path, + relative_path, + contents, + header_values, + header_column_locations: Locations::from_dict(py, header_column_locations), + output_column_locations: Locations::from_dict(py, output_column_locations), + query_sql, + enum_declarations: enum_declarations.unwrap_or_else(|| PyTuple::empty(py).unbind()), + constant_declarations: constant_declarations + .unwrap_or_else(|| PyTuple::empty(py).unbind()), + extract_implicit_alias_columns, + } + } + + #[getter] + fn file_path(&self, py: Python<'_>) -> Py { + self.file_path.clone_ref(py) + } + + #[getter] + fn relative_path(&self, py: Python<'_>) -> Py { + self.relative_path.clone_ref(py) + } + + #[getter] + fn contents(&self, py: Python<'_>) -> Py { + self.contents.clone_ref(py) + } + + #[getter] + fn header_values(&self, py: Python<'_>) -> Py { + self.header_values.clone_ref(py) + } + + #[getter] + fn header_column_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_locations(py) + } + + #[getter] + fn output_column_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.output_column_locations.dict(py, &self.relative_path) + } + + #[getter] + fn query_sql(&self, py: Python<'_>) -> Py { + self.query_sql.clone_ref(py) + } + + #[getter] + fn enum_declarations(&self, py: Python<'_>) -> Py { + self.enum_declarations.clone_ref(py) + } + + #[getter] + fn constant_declarations(&self, py: Python<'_>) -> Py { + self.constant_declarations.clone_ref(py) + } + + #[getter] + fn extract_implicit_alias_columns(&self) -> bool { + self.extract_implicit_alias_columns + } + + fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult { + let Ok(other) = other.downcast::() else { + return Ok(false); + }; + let other: &NativeModelFile = other.get(); + for (left, right) in self.fields(py)?.iter().zip(other.fields(py)?) { + if !left.eq(right)? { + return Ok(false); + } + } + Ok(true) + } + + fn __repr__(&self, py: Python<'_>) -> PyResult { + let fields: Vec = FIELD_NAMES + .iter() + .zip(self.fields(py)?) + .map(|(name, value)| Ok(format!("{name}={}", value.repr()?))) + .collect::>()?; + Ok(format!("DiscoveredSqlModelFile({})", fields.join(", "))) + } + + fn __reduce__<'py>( + slf: &Bound<'py, Self>, + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { + let py: Python<'py> = slf.py(); + Ok(( + slf.get_type().into_any(), + PyTuple::new(py, slf.get().fields(py)?)?, + )) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index 8b45ff5c80..d2e899aebd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -1,8 +1,10 @@ //! Native discovery results as the plain data the Python discovery facade materialises. use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBytes, PyTuple}; +use pyo3::prelude::{ + Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBytes, PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_discovery::declarations::main::declaration_layout::declaration_layout; use sqlbuild_discovery::declarations::models::{ @@ -32,6 +34,10 @@ use std::path::{Path, PathBuf}; use std::sync::Mutex; use crate::bindings::_helpers::boundary::config_values::config_value_to_python; +use crate::bindings::_helpers::discovery::header_declarations::{ + HeaderDeclarations, model_header_declarations, +}; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileParts, NativeModelFile}; use crate::bindings::_helpers::sqltext::authored_values::map_to_python; use crate::bindings::models::{ ModelDiscoveryRequest, ModelTextRequest, NativeProjectTree, SqlTestDiscoveryRequest, @@ -40,6 +46,13 @@ use crate::bindings::models::{ use crate::bindings::types::CompilerDetach; pub(crate) type PyObject = Py; + +const PATHLIB_MODULE: &str = "pathlib"; +const PATH_CLASS: &str = "Path"; +const STEM_ATTRIBUTE: &str = "stem"; +const ENUMS_KEY: &str = "enums"; +const CONSTANTS_KEY: &str = "constants"; +const RAISED_TAG: &str = "raised"; type Located = (String, usize, usize, usize, usize); type Listing = (PyObject, Vec<(PyObject, bool, bool)>); @@ -298,12 +311,14 @@ impl NativeProjectTree { } } -/// Discover the model files: one payload per file, or the collection's failure payload. +/// Discover the model files: `(relative path, record or failure payload)` per file, or the +/// collection's failure payload. #[pyfunction] fn discover_model_files( py: Python<'_>, request: ModelDiscoveryRequest, tree: &NativeProjectTree, + project_dir: Bound<'_, PyAny>, ) -> PyResult { let root = ProjectRoot { directory: PathBuf::from(request.project_dir), @@ -315,10 +330,78 @@ fn discover_model_files( extract_implicit_alias_columns: request.extract_implicit_alias_columns, extract_output_column_locations: request.extract_output_column_locations, }; + let extract_implicit_alias_columns: bool = options.extract_implicit_alias_columns; let discovered = py .compiler_detach(|| Ok(discover_models(&root, &tree.inner, &options))) .map_err(crate::bindings::_helpers::boundary::panics::compiler_error)?; - files_object(py, &tree.inner, discovered, model_object) + let files: Vec> = match discovered { + Ok(files) => files, + Err(failure) => return stage_failure_object(py, &tree.inner, failure), + }; + let path_class: Bound<'_, PyAny> = py.import(PATHLIB_MODULE)?.getattr(PATH_CLASS)?; + files + .into_iter() + .map(|file| { + let relative_path: Bound<'_, PyAny> = + path_class.call1((path_object(py, &tree.inner, file.relative_path)?,))?; + let item: PyObject = match file.outcome { + FileOutcome::Parsed(model) => model_record( + py, + model, + (project_dir.div(&relative_path)?, relative_path.clone()), + extract_implicit_alias_columns, + )?, + FileOutcome::Unreadable(failure) => read_object(py, failure)?, + FileOutcome::Failed(failure) => failure_object(py, failure)?, + }; + Ok((relative_path.unbind(), item)) + }) + .collect::>>()? + .into_py_any(py) +} + +/// One parsed model file as its native record, or `("raised", error)` for invalid declarations. +fn model_record<'py>( + py: Python<'py>, + model: DiscoveredModelFile, + paths: (Bound<'py, PyAny>, Bound<'py, PyAny>), + extract_implicit_alias_columns: bool, +) -> PyResult { + let (file_path, relative_path) = paths; + let header_values: Bound<'py, PyDict> = map_to_python(py, model.header_values)? + .into_bound(py) + .downcast_into::()?; + let model_name: String = file_path.getattr(STEM_ATTRIBUTE)?.extract()?; + let declarations: HeaderDeclarations<'py> = match model_header_declarations( + py, + ( + header_values.get_item(ENUMS_KEY)?, + header_values.get_item(CONSTANTS_KEY)?, + ), + model_name, + relative_path.clone(), + ) { + Ok(declarations) => declarations, + Err(error) => { + return tuple_object( + py, + vec![object(py, RAISED_TAG)?, error.into_value(py).into_any()], + ); + } + }; + let (enum_declarations, constant_declarations) = declarations; + let record: NativeModelFile = NativeModelFile::discovered(ModelFileParts { + file_path: file_path.unbind(), + relative_path: relative_path.unbind(), + contents: PyString::new(py, &model.contents).unbind(), + header_values: header_values.unbind(), + header_column_locations: model.header_column_locations, + output_column_locations: model.output_column_locations, + query_sql: PyString::new(py, &model.query_sql).unbind(), + declarations: (enum_declarations.unbind(), constant_declarations.unbind()), + extract_implicit_alias_columns, + }); + record.into_py_any(py) } /// Parse in-memory SQL model contents into the payload model discovery returns for a file. @@ -576,6 +659,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(parse_sql_test_contents, module)?)?; module.add_function(wrap_pyfunction!(parse_scenario_contents, module)?)?; module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!(discover_model_files, module)?)?; module.add_function(wrap_pyfunction!(parse_model_contents, module)?)?; module.add_function(wrap_pyfunction!(discover_declaration_layout, module)?)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 22dfb2eb6a..8cf1a6b0e9 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -53,7 +53,6 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredSourceFile, DiscoveredSqlFunctionFile, DiscoveredSqlHookFile, - DiscoveredSqlModelFile, EnumDeclaration, ) from sqlbuild.compiler.references.models import LiteralSqlRelation @@ -836,9 +835,46 @@ class NativeProjectTree: def __init__(self, project_dir: str) -> None: ... def listings(self) -> list[tuple[str, list[tuple[str, bool, bool]]]]: ... +class DiscoveredSqlModelFile: + """A discovered SQL model file and its raw contents.""" + + def __init__( + self, + file_path: Path, + relative_path: Path, + contents: str, + header_values: dict[str, object], + header_column_locations: dict[str, SourceLocation], + output_column_locations: dict[str, SourceLocation], + query_sql: str, + enum_declarations: tuple[EnumDeclaration, ...] = (), + constant_declarations: tuple[ConstantDeclaration, ...] = (), + extract_implicit_alias_columns: bool = True, + ) -> None: ... + @property + def file_path(self) -> Path: ... + @property + def relative_path(self) -> Path: ... + @property + def contents(self) -> str: ... + @property + def header_values(self) -> dict[str, object]: ... + @property + def header_column_locations(self) -> dict[str, SourceLocation]: ... + @property + def output_column_locations(self) -> dict[str, SourceLocation]: ... + @property + def query_sql(self) -> str: ... + @property + def enum_declarations(self) -> tuple[EnumDeclaration, ...]: ... + @property + def constant_declarations(self) -> tuple[ConstantDeclaration, ...]: ... + @property + def extract_implicit_alias_columns(self) -> bool: ... + def discover_model_files( - request: dict[str, object], tree: NativeProjectTree -) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... + request: dict[str, object], tree: NativeProjectTree, project_dir: Path +) -> list[tuple[Path, object]] | tuple[object, ...]: ... def discover_sql_test_files( request: dict[str, object], tree: NativeProjectTree ) -> list[tuple[str, tuple[object, ...]]] | tuple[object, ...]: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index e4c67b106b..e262ecad97 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -357,23 +357,9 @@ def _run_native_model_loop( host, [ ( - ( - model_file.file_path.stem, - model_file.relative_path.as_posix(), - str(model_file.relative_path), - str(model_file.file_path), - ), - ( - model_file.header_column_locations, - model_file.header_values, - model_file.enum_declarations, - model_file.constant_declarations, - ), - model_file.query_sql, - model_file.contents, + model_file, loop.native_declaration_references.get(model_file.file_path), _interpolation_row(interpolations[model_file.file_path]), - model_file, ) for model_file in files ], diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py index be9389a8f3..2cf3ff2943 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -1,4 +1,4 @@ -"""SQL model files discovered by the native engine, materialised as Python discovery records.""" +"""SQL model files discovered by the native engine as native records.""" from __future__ import annotations @@ -10,7 +10,6 @@ from sqlbuild.compiler.discovery._helpers.native.payloads import ( materialise_native_files, native_collection, - native_locations, native_payload_error, native_project_tree, native_request, @@ -21,13 +20,10 @@ REMOVED_SQL_MODEL_HEADER_KEYS, SQL_MODEL_HEADER_KEYS, ) -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredSqlModelFile, - DiscoveryFileFault, - EnumDeclaration, -) -from sqlbuild.compiler.discovery.types import NativeFiles, NativeLocation +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile, DiscoveryFileFault +from sqlbuild.compiler.discovery.types import NativeModelFiles + +_RAISED_TAG: str = "raised" def discover_native_model_files( @@ -42,7 +38,7 @@ def discover_native_model_files( with DirectorySnapshot.scope(project_dir=project_dir): tree: _native.NativeProjectTree = native_project_tree(project_dir) - files: NativeFiles = native_collection( + files: NativeModelFiles = native_collection( result=_native.discover_model_files( native_request( project_dir=project_dir, @@ -54,6 +50,7 @@ def discover_native_model_files( }, ), tree, + project_dir, ), project_dir=project_dir, ) @@ -61,54 +58,26 @@ def discover_native_model_files( return materialise_native_files( project_dir=project_dir, files=( - (Path(relative_path), payload) - for relative_path, payload in files - if selected_model_names is None or Path(relative_path).stem in selected_model_names + (relative_path, item) + for relative_path, item in files + if selected_model_names is None or relative_path.stem in selected_model_names ), - build=lambda relative_path, payload: _model_file( - project_dir=project_dir, - relative_path=relative_path, - payload=payload, - extract_implicit_alias_columns=extract_implicit_alias_columns, + build=lambda relative_path, item: _model_file( + project_dir=project_dir, relative_path=relative_path, item=item ), on_fault=on_fault, ) def _model_file( - *, - project_dir: Path, - relative_path: Path, - payload: tuple[object, ...], - extract_implicit_alias_columns: bool, + *, project_dir: Path, relative_path: Path, item: object ) -> DiscoveredSqlModelFile: - file_path: Path = project_dir / relative_path - error: Exception | None = native_payload_error(payload=payload, file_path=file_path) - if error is not None: - raise error - _tag, contents, values, header_locations, output_locations, query_sql = payload - header_values: dict[str, object] = cast(dict[str, object], values) - enum_declarations, constant_declarations = cast( - tuple[tuple[EnumDeclaration, ...], tuple[ConstantDeclaration, ...]], - _native.model_header_declarations( - (header_values.get("enums"), header_values.get("constants")), - file_path.stem, - relative_path, - ), - ) - return DiscoveredSqlModelFile( - file_path=file_path, - relative_path=relative_path, - contents=str(contents), - header_values=header_values, - header_column_locations=native_locations( - locations=cast(list[NativeLocation], header_locations), relative_path=relative_path - ), - output_column_locations=native_locations( - locations=cast(list[NativeLocation], output_locations), relative_path=relative_path - ), - query_sql=str(query_sql), - enum_declarations=enum_declarations, - constant_declarations=constant_declarations, - extract_implicit_alias_columns=extract_implicit_alias_columns, + if isinstance(item, DiscoveredSqlModelFile): + return item + payload: tuple[object, ...] = cast(tuple[object, ...], item) + if payload[0] == _RAISED_TAG: + raise cast(Exception, payload[1]) + error: Exception | None = native_payload_error( + payload=payload, file_path=project_dir / relative_path ) + raise error if error is not None else ValueError(payload) diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index fdf22d8355..012a87f6cf 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -6,7 +6,7 @@ from dataclasses import dataclass, field, replace from pathlib import Path -from sqlbuild._native import NativeDiscoverySession +from sqlbuild._native import DiscoveredSqlModelFile, NativeDiscoverySession from sqlbuild.compiler.auditing.models import MeasurementContract from sqlbuild.compiler.auditing.types import AuditEvaluationMode from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE @@ -69,22 +69,6 @@ class PythonHookEntry: kwargs: dict[str, object] -@dataclass(frozen=True) -class DiscoveredSqlModelFile: - """A discovered SQL model file and its raw contents.""" - - file_path: Path - relative_path: Path - contents: str - header_values: dict[str, object] - header_column_locations: dict[str, SourceLocation] - output_column_locations: dict[str, SourceLocation] - query_sql: str - enum_declarations: tuple[EnumDeclaration, ...] = field(default_factory=tuple) - constant_declarations: tuple[ConstantDeclaration, ...] = field(default_factory=tuple) - extract_implicit_alias_columns: bool = True - - @dataclass(frozen=True) class DiscoveredSqlFunctionFile: """A discovered SQL function file and its raw contents.""" diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index e06a14f4de..a054a52dca 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -16,6 +16,7 @@ type NativeLocation = tuple[str, int, int, int, int] type NativeListing = tuple[str, list[tuple[str, bool, bool]]] type NativeFiles = list[tuple[str, tuple[object, ...]]] +type NativeModelFiles = list[tuple[Path, object]] type NativeFileScope = tuple[str, str, str | None, str | None, str | None] diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 69e2dfea59..9f28dee1d6 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -33,6 +33,18 @@ ) } STAGE_CAPTURE_PROJECTED_ATTRIBUTES: dict[str, tuple[str, ...]] = { + "sqlbuild._native:DiscoveredSqlModelFile": ( + "file_path", + "relative_path", + "contents", + "header_values", + "header_column_locations", + "output_column_locations", + "query_sql", + "enum_declarations", + "constant_declarations", + "extract_implicit_alias_columns", + ), "sqlbuild._native:NativeAuditInput": ( "audit_file", "audit_block", @@ -271,6 +283,9 @@ ), } STAGE_CAPTURE_CANONICAL_TYPES: dict[str, str] = { + "sqlbuild._native:DiscoveredSqlModelFile": ( + "sqlbuild.compiler.discovery.models:DiscoveredSqlModelFile" + ), "sqlbuild._native:NativeInputsView": "sqlbuild.compiler.compile.models:CompileProjectInputs", "sqlbuild._native:NativeProjectView": "sqlbuild.compiler.compile.models:CompiledProject", "sqlbuild._native:NativeModelInput": "sqlbuild.compiler.compile.models:CompileModelInput", diff --git a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py index 477e9f8342..8800958555 100644 --- a/tests/integration/src/sqlbuild/compiler/model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/model_config/helpers.py @@ -393,10 +393,17 @@ def _model_file( name: SourceLocation(path=base.relative_path, line=line + 2, column=3) for line, name in enumerate(names[: len(names) * (index % 2)]) } - return replace( - base, + return DiscoveredSqlModelFile( + file_path=base.file_path, + relative_path=base.relative_path, + contents=base.contents, header_values={**base.header_values, "columns": columns, "audits": audits}, header_column_locations=locations, + output_column_locations=base.output_column_locations, + query_sql=base.query_sql, + enum_declarations=base.enum_declarations, + constant_declarations=base.constant_declarations, + extract_implicit_alias_columns=base.extract_implicit_alias_columns, ) From 064bd27ebd03790e04d3785c8fc93af34bbb2f6f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:58:58 +0100 Subject: [PATCH 2/7] chore: lower the Python boundary baseline --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 205db07533..d2c88aac9b 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 146314 +remaining_lines = 146269 [[stays]] path = "executor" From b50fb22b2b5f1a2e09238b449d8d32165d1c375e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:05:48 +0100 Subject: [PATCH 3/7] fix: keep model-file fault, equality and test-hook behaviour --- .../bindings/_helpers/discovery/model_file_record.rs | 10 +++++----- .../src/bindings/_helpers/discovery/project_files.rs | 5 +++-- .../compiler/discovery/_helpers/native/payloads.py | 6 +++--- .../src/sqlbuild/cli/commands/main/helpers.py | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index bb98e62dbe..79951291d9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{pyclass, pymethods}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; use sqlbuild_discovery::models::LineColumnSpan; const SOURCE_LOCATION_MODULE: &str = "sqlbuild.spec.contracts.models"; @@ -258,17 +258,17 @@ impl NativeModelFile { self.extract_implicit_alias_columns } - fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult { + fn __eq__(&self, py: Python<'_>, other: &Bound<'_, PyAny>) -> PyResult> { let Ok(other) = other.downcast::() else { - return Ok(false); + return Ok(py.NotImplemented()); }; let other: &NativeModelFile = other.get(); for (left, right) in self.fields(py)?.iter().zip(other.fields(py)?) { if !left.eq(right)? { - return Ok(false); + return Ok(false.into_py_any(py)?); } } - Ok(true) + true.into_py_any(py) } fn __repr__(&self, py: Python<'_>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index d2e899aebd..ee1431697d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -1,6 +1,6 @@ //! Native discovery results as the plain data the Python discovery facade materialises. -use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::exceptions::{PyException, PyRuntimeError, PyValueError}; use pyo3::prelude::{ Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, }; @@ -382,12 +382,13 @@ fn model_record<'py>( relative_path.clone(), ) { Ok(declarations) => declarations, - Err(error) => { + Err(error) if error.is_instance_of::(py) => { return tuple_object( py, vec![object(py, RAISED_TAG)?, error.into_value(py).into_any()], ); } + Err(error) => return Err(error), }; let (enum_declarations, constant_declarations) = declarations; let record: NativeModelFile = NativeModelFile::discovered(ModelFileParts { diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index eb2106d006..2606887e43 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -119,11 +119,11 @@ def native_collection[ResultT]( raise native_failure(payload) -def materialise_native_files[RecordT]( +def materialise_native_files[ItemT, RecordT]( *, project_dir: Path, - files: Iterable[tuple[Path, tuple[object, ...]]], - build: Callable[[Path, tuple[object, ...]], RecordT], + files: Iterable[tuple[Path, ItemT]], + build: Callable[[Path, ItemT], RecordT], on_fault: Callable[[DiscoveryFileFault], None] | None, ) -> tuple[RecordT, ...]: """Build each native file payload, reporting and skipping a failing file when asked.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 20947e2be4..9b8f37cf44 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -655,9 +655,9 @@ def record_eager_output_column_scans(monkeypatch: pytest.MonkeyPatch) -> list[di eager: list[dict[str, object]] = [] original: Callable[..., object] = native_model_files_module._native.discover_model_files - def recording(request: dict[str, object], tree: NativeProjectTree) -> object: + def recording(request: dict[str, object], tree: NativeProjectTree, project_dir: Path) -> object: eager.extend(filter(lambda item: item["extract_output_column_locations"], (request,))) - return original(request, tree) + return original(request, tree, project_dir) monkeypatch.setattr(native_model_files_module._native, "discover_model_files", recording) return eager From 7d8964266bf7a9ecf7eb8e7701ac1f66a26ff69d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:40:25 +0100 Subject: [PATCH 4/7] refactor: validate discovered project inputs natively --- Cargo.lock | 1 + crates/sqlbuild-discovery/Cargo.toml | 1 + .../src/_helpers/reading.rs | 2 +- crates/sqlbuild-discovery/src/lib.rs | 1 + crates/sqlbuild-discovery/src/models.rs | 6 + .../project_validation/_helpers/csv_header.rs | 365 +++++++ .../project_validation/_helpers/identities.rs | 143 +++ .../src/project_validation/_helpers/mod.rs | 5 + .../_helpers/python_nodes.rs | 206 ++++ .../_helpers/resource_names.rs | 72 ++ .../src/project_validation/_helpers/seeds.rs | 122 +++ .../src/project_validation/main/mod.rs | 1 + .../main/validate_project_inputs.rs | 354 +++++++ .../src/project_validation/mod.rs | 6 + .../src/project_validation/models.rs | 194 ++++ .../src/project_validation/tests.rs | 4 + .../tests/test_csv_header.rs | 167 +++ .../tests/test_validate_project_inputs.rs | 294 ++++++ .../main/matched_path_default_keys.rs | 32 + .../src/path_defaults/main/mod.rs | 1 + .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/model_file_record.rs | 2 +- .../_helpers/discovery/project_validation.rs | 255 +++++ .../_helpers/discovery/registration.rs | 3 +- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 1 + .../discovery/_helpers/native/model_files.py | 4 +- .../discovery/_helpers/native/payloads.py | 4 + .../_helpers/validation/discovery.py | 980 ++++-------------- .../_helpers/validation/identities.py | 123 --- src/sqlbuild/compiler/discovery/models.py | 1 - .../compiler/discovery/_test_types.py | 10 + .../test_native_project_validation.py | 109 ++ .../discovery/_helpers/_test_types.py | 16 - .../discovery/_helpers/test_path_defaults.py | 115 -- 35 files changed, 2535 insertions(+), 1068 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/_helpers/seeds.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/main/validate_project_inputs.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/mod.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/models.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/tests.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/tests/test_csv_header.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs create mode 100644 crates/sqlbuild-model-config/src/path_defaults/main/matched_path_default_keys.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/validation/identities.py create mode 100644 tests/integration/src/sqlbuild/compiler/discovery/test_native_project_validation.py delete mode 100644 tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_path_defaults.py diff --git a/Cargo.lock b/Cargo.lock index d25a9e9d2d..41007b15cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -834,6 +834,7 @@ dependencies = [ "rayon", "sqlbuild-config", "sqlbuild-core", + "sqlbuild-model-config", "sqlbuild-sqltext", "tempfile", ] diff --git a/crates/sqlbuild-discovery/Cargo.toml b/crates/sqlbuild-discovery/Cargo.toml index e5b3130fff..3b1132a79f 100644 --- a/crates/sqlbuild-discovery/Cargo.toml +++ b/crates/sqlbuild-discovery/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true rayon.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-model-config.workspace = true sqlbuild-sqltext.workspace = true [target.'cfg(windows)'.dependencies] diff --git a/crates/sqlbuild-discovery/src/_helpers/reading.rs b/crates/sqlbuild-discovery/src/_helpers/reading.rs index bbc3dea814..36f9bc3f2f 100644 --- a/crates/sqlbuild-discovery/src/_helpers/reading.rs +++ b/crates/sqlbuild-discovery/src/_helpers/reading.rs @@ -114,7 +114,7 @@ pub(crate) fn windows_errno(code: i32) -> i32 { } /// CPython's UTF-8 decoder error: the range and reason it reports for the first bad sequence. -fn decode_failure(bytes: Vec, error: TextDecodeError) -> ReadFailure { +pub(crate) fn decode_failure(bytes: Vec, error: TextDecodeError) -> ReadFailure { let start: usize = error.valid_up_to; let (end, reason): (usize, &'static str) = match error.error_len { None => (bytes.len(), "unexpected end of data"), diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index c15f5b2b32..940181a53c 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -9,6 +9,7 @@ pub mod declarations; pub mod key_checks; pub mod model_files; pub mod models; +pub mod project_validation; pub mod sql_tests; pub mod tree; pub mod yaml_files; diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index b38c79864f..f00fcbd4df 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -26,6 +26,10 @@ pub enum FailureKind { SqlAudit, /// `ResourceIdentityError` (D016). ResourceIdentity, + /// `DiscoveryConflictError` (D007). + Conflict, + /// `SeedDiscoveryError` (D008). + Seed, } impl FailureKind { @@ -42,6 +46,8 @@ impl FailureKind { Self::SqlHook => "sql_hook", Self::SqlAudit => "sql_audit", Self::ResourceIdentity => "resource_identity", + Self::Conflict => "conflict", + Self::Seed => "seed", } } } diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs new file mode 100644 index 0000000000..611a042977 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs @@ -0,0 +1,365 @@ +//! The first record Python's `csv.reader` returns for a seed file opened with `newline=""`. + +use crate::_helpers::reading::decode_failure; +use crate::models::ReadFailure; +use crate::project_validation::models::CsvSettings; +use sqlbuild_core::text::main::decode_text::decode_text; + +/// The size of the chunks `TextIOWrapper` decodes. +const DECODE_CHUNK: usize = 8192; +const UNQUOTED_NEWLINE: &str = + "new-line character seen in unquoted field - do you need to open the file with newline=''?"; + +/// The header row, `None` for an empty file, or the error Python raises reading it. +pub(crate) fn csv_header(bytes: &[u8], settings: &CsvSettings) -> Result, HeaderError> { + let dialect: Dialect = Dialect::new(settings)?; + let settings: &Dialect = &dialect; + let mut end: usize = bytes.len().min(DECODE_CHUNK); + loop { + let text: String = decoded_prefix(bytes, end)?; + let complete: bool = end == bytes.len(); + match first_record(&text, settings, complete)? { + Some(record) => return Ok(record), + None if complete => return Ok(Vec::new()), + None => end = bytes.len().min(end + DECODE_CHUNK), + } + } +} + +/// Why the header could not be read. +#[derive(Debug)] +pub(crate) enum HeaderError { + Read(ReadFailure), + /// `_csv.Error` with its message. + Csv(String), + /// The `TypeError` `csv.reader` raises for a setting that is not one character. + Setting(String), +} + +/// The reader's settings as `csv.reader` receives them from the facade. +pub(crate) struct Dialect { + delimiter: char, + quotechar: Option, + escapechar: Option, + doublequote: bool, + skipinitialspace: bool, +} + +impl Dialect { + fn new(settings: &CsvSettings) -> Result { + let delimiter: char = match settings.delimiter.as_deref() { + None | Some("") => ',', + Some(text) => one_character("delimiter", text)?, + }; + let optional = |name: &str, value: &Option| -> Result, HeaderError> { + value + .as_deref() + .map(|text| one_character(name, text)) + .transpose() + }; + Ok(Self { + delimiter, + quotechar: optional("quotechar", &settings.quotechar)?, + escapechar: optional("escapechar", &settings.escapechar)?, + doublequote: settings.doublequote.unwrap_or(true), + skipinitialspace: settings.skipinitialspace.unwrap_or(false), + }) + } +} + +fn one_character(name: &str, text: &str) -> Result { + let mut characters = text.chars(); + match (characters.next(), characters.next()) { + (Some(character), None) => Ok(character), + _ => Err(HeaderError::Setting(format!( + "\"{name}\" must be a 1-character string" + ))), + } +} + +/// The text of `bytes[..end]`; a multi-byte character cut by `end` waits for the next chunk. +fn decoded_prefix(bytes: &[u8], end: usize) -> Result { + let prefix: &[u8] = &bytes[..end]; + match std::str::from_utf8(prefix) { + Ok(text) => Ok(text.to_owned()), + Err(error) if error.error_len().is_none() && end < bytes.len() => { + Ok(String::from_utf8_lossy(&prefix[..error.valid_up_to()]).into_owned()) + } + Err(_) => decode_text(prefix) + .map_err(|error| HeaderError::Read(decode_failure(prefix.to_vec(), error))), + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum State { + StartRecord, + StartField, + EscapedChar, + InField, + InQuotedField, + EscapeInQuotedField, + QuoteInQuotedField, + EatCrnl, + AfterEscapedCrnl, +} + +/// One character, or the end of a line. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Input { + Char(char), + Eol, +} + +struct Parser<'a> { + settings: &'a Dialect, + state: State, + field: String, + field_len: usize, + fields: Vec, +} + +impl Parser<'_> { + fn save_field(&mut self) { + self.fields.push(std::mem::take(&mut self.field)); + self.field_len = 0; + } + + fn add(&mut self, character: char) { + self.field.push(character); + self.field_len += 1; + } + + fn is_quote(&self, character: char) -> bool { + self.settings.quotechar == Some(character) + } + + fn is_escape(&self, character: char) -> bool { + self.settings.escapechar == Some(character) + } + + fn delimiter(&self) -> char { + self.settings.delimiter + } + + /// CPython's `parse_process_char`. + fn process(&mut self, input: Input) -> Result<(), HeaderError> { + let newline = |input: Input| matches!(input, Input::Char('\n' | '\r')); + loop { + match self.state { + State::StartRecord => { + if input == Input::Eol { + return Ok(()); + } + if newline(input) { + self.state = State::EatCrnl; + return Ok(()); + } + self.state = State::StartField; + } + State::StartField => { + match input { + Input::Eol => { + self.save_field(); + self.state = State::StartRecord; + } + Input::Char('\n' | '\r') => { + self.save_field(); + self.state = State::EatCrnl; + } + Input::Char(character) if self.is_quote(character) => { + self.state = State::InQuotedField; + } + Input::Char(character) if self.is_escape(character) => { + self.state = State::EscapedChar; + } + Input::Char(' ') if self.settings.skipinitialspace => {} + Input::Char(character) if character == self.delimiter() => { + self.save_field(); + } + Input::Char(character) => { + self.add(character); + self.state = State::InField; + } + } + return Ok(()); + } + State::EscapedChar => { + match input { + Input::Char(character @ ('\n' | '\r')) => { + self.add(character); + self.state = State::AfterEscapedCrnl; + } + Input::Eol => { + self.add('\n'); + self.state = State::InField; + } + Input::Char(character) => { + self.add(character); + self.state = State::InField; + } + } + return Ok(()); + } + State::AfterEscapedCrnl => { + if input == Input::Eol { + return Ok(()); + } + self.state = State::InField; + } + State::InField => { + match input { + Input::Eol => { + self.save_field(); + self.state = State::StartRecord; + } + Input::Char('\n' | '\r') => { + self.save_field(); + self.state = State::EatCrnl; + } + Input::Char(character) if self.is_escape(character) => { + self.state = State::EscapedChar; + } + Input::Char(character) if character == self.delimiter() => { + self.save_field(); + self.state = State::StartField; + } + Input::Char(character) => self.add(character), + } + return Ok(()); + } + State::InQuotedField => { + match input { + Input::Eol => {} + Input::Char(character) if self.is_escape(character) => { + self.state = State::EscapeInQuotedField; + } + Input::Char(character) if self.is_quote(character) => { + self.state = if self.settings.doublequote { + State::QuoteInQuotedField + } else { + State::InField + }; + } + Input::Char(character) => self.add(character), + } + return Ok(()); + } + State::EscapeInQuotedField => { + self.add(match input { + Input::Eol => '\n', + Input::Char(character) => character, + }); + self.state = State::InQuotedField; + return Ok(()); + } + State::QuoteInQuotedField => { + match input { + Input::Char(character) if self.is_quote(character) => { + self.add(character); + self.state = State::InQuotedField; + } + Input::Char(character) if character == self.delimiter() => { + self.save_field(); + self.state = State::StartField; + } + Input::Eol => { + self.save_field(); + self.state = State::StartRecord; + } + Input::Char('\n' | '\r') => { + self.save_field(); + self.state = State::EatCrnl; + } + Input::Char(character) => { + self.add(character); + self.state = State::InField; + } + } + return Ok(()); + } + State::EatCrnl => { + return match input { + Input::Char('\n' | '\r') => Ok(()), + Input::Eol => { + self.state = State::StartRecord; + Ok(()) + } + Input::Char(_) => Err(HeaderError::Csv(UNQUOTED_NEWLINE.to_owned())), + }; + } + } + } + } +} + +/// The first record of `text`, or `None` when `text` ends before it (`complete` is false) or is empty. +fn first_record( + text: &str, + settings: &Dialect, + complete: bool, +) -> Result>, HeaderError> { + let mut parser: Parser<'_> = Parser { + settings, + state: State::StartRecord, + field: String::new(), + field_len: 0, + fields: Vec::new(), + }; + let mut lines = lines(text).peekable(); + while let Some(line) = lines.next() { + let terminated: bool = line.ends_with(['\n', '\r']); + let last: bool = lines.peek().is_none(); + if last && !complete && !terminated_line(line, terminated, text) { + return Ok(None); + } + for character in line.chars() { + parser.process(Input::Char(character))?; + } + parser.process(Input::Eol)?; + if parser.state == State::StartRecord { + return Ok(Some(parser.fields)); + } + } + if !complete { + return Ok(None); + } + if parser.field_len != 0 || parser.state == State::InQuotedField { + parser.save_field(); + return Ok(Some(parser.fields)); + } + Ok(None) +} + +/// Whether a final line is known to be whole: it ends with `\n`, or `\r` not followed later. +fn terminated_line(line: &str, terminated: bool, text: &str) -> bool { + terminated && (line.ends_with('\n') || !text.ends_with('\r')) +} + +/// The lines of `text` as `newline=""` iteration yields them, terminators included. +fn lines(text: &str) -> impl Iterator { + let mut rest: &str = text; + std::iter::from_fn(move || { + if rest.is_empty() { + return None; + } + let bytes: &[u8] = rest.as_bytes(); + let mut end: usize = bytes.len(); + for (index, byte) in bytes.iter().enumerate() { + if *byte == b'\n' { + end = index + 1; + break; + } + if *byte == b'\r' { + end = if bytes.get(index + 1) == Some(&b'\n') { + index + 2 + } else { + index + 1 + }; + break; + } + } + let (line, tail) = rest.split_at(end); + rest = tail; + Some(line) + }) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs new file mode 100644 index 0000000000..6310139019 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/identities.rs @@ -0,0 +1,143 @@ +//! Python's `validate_discovered_resource_identities`: canonical identities, in discovery order. + +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::project_validation::models::{NamedPath, ProjectFacts, ValidationFailure}; +use sqlbuild_model_config::resource_identities::main::resource_identity_failure::resource_identity_failure; + +/// One identity to check: its name, the kind the error names, its path and whether it is private. +struct Candidate<'a> { + name: &'a str, + kind: &'a str, + path: &'a str, + private: bool, +} + +fn public<'a>(name: &'a str, kind: &'a str, path: &'a str) -> Candidate<'a> { + Candidate { + name, + kind, + path, + private: false, + } +} + +fn named<'a>(entries: &'a [NamedPath], kind: &'a str) -> impl Iterator> { + entries + .iter() + .map(move |entry| public(&entry.name, kind, &entry.path)) +} + +/// Reject the first identity that is not canonical lowercase snake_case. +pub(crate) fn validate_identities(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + for candidate in candidates(facts) { + if let Some((corrected, help)) = + resource_identity_failure(candidate.name, candidate.private) + { + return Err(ValidationFailure::Discovery(DiscoveryFailure { + kind: FailureKind::ResourceIdentity, + message: format!( + "Invalid {} identity '{}' in {}; use snake_case '{corrected}'", + candidate.kind, candidate.name, candidate.path + ), + help: Some(help), + })); + } + } + Ok(()) +} + +fn candidates(facts: &ProjectFacts) -> impl Iterator> { + let models = facts.models.iter().flat_map(|model| { + std::iter::once(public(&model.file.name, "model", &model.file.path)).chain( + model + .local_declarations + .iter() + .map(|declaration| Candidate { + name: &declaration.name, + kind: "model-local declaration", + path: &declaration.path, + private: true, + }), + ) + }); + let functions = + named(&facts.sql_functions, "function").chain(named(&facts.python_functions, "function")); + let seeds = facts + .seeds + .iter() + .map(|seed| public(&seed.file.name, "seed", &seed.file.path)); + let schemas = facts.schemas.iter().flat_map(|schema| { + let models = schema + .model_names + .iter() + .map(move |name| public(name, "schema model", &schema.path)); + let seeds = schema + .seeds + .iter() + .map(move |seed| public(&seed.name, "schema seed", &schema.path)); + models.chain(seeds) + }); + let sources = facts.sources.iter().flat_map(|file| { + file.entries + .iter() + .map(move |entry| public(&entry.name, "source", &file.path)) + }); + let tests = facts.tests.iter().flat_map(|test| { + test.blocks.iter().flat_map(move |block| { + let name: &str = block.name.as_deref().unwrap_or(&test.file.name); + std::iter::once(public(name, "SQL test", &test.file.path)).chain( + block + .cases + .iter() + .map(move |case| public(case, "SQL test case", &test.file.path)), + ) + }) + }); + let audits = facts.audits.iter().flat_map(|audit| { + audit.block_names.iter().flat_map(move |block_name| { + let name: &str = if audit.generic { + &audit.file.name + } else { + block_name.as_deref().unwrap_or(&audit.file.name) + }; + let instance = block_name + .as_deref() + .filter(|_| audit.generic) + .map(|instance| public(instance, "generic audit instance", &audit.file.path)); + std::iter::once(public(name, "audit", &audit.file.path)).chain(instance) + }) + }); + let declarations = facts.declaration_files.iter().flat_map(|file| { + file.declarations.iter().map(move |declaration| Candidate { + name: &declaration.name, + kind: "declaration", + path: &declaration.path, + private: file.private, + }) + }); + let nodes = ["loader", "task", "asset", "check"] + .into_iter() + .flat_map(move |label| { + facts + .nodes + .iter() + .filter(move |node| node.kind.label() == label) + .map(move |node| public(&node.name, label, &node.path)) + }); + models + .chain(functions) + .chain(named(&facts.sql_hooks, "SQL hook")) + .chain(seeds) + .chain(schemas) + .chain(sources) + .chain(tests) + .chain(named(&facts.scenarios, "scenario")) + .chain(audits) + .chain(declarations) + .chain(named(&facts.model_schemas, "model schema")) + .chain(named(&facts.materializations, "materialization")) + .chain(nodes) + .chain(named(&facts.hook_functions, "Python hook")) + .chain(named(&facts.event_exporters, "event exporter")) + .chain(named(&facts.providers, "provider")) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/mod.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/mod.rs new file mode 100644 index 0000000000..10673cb9e2 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod csv_header; +pub(crate) mod identities; +pub(crate) mod python_nodes; +pub(crate) mod resource_names; +pub(crate) mod seeds; diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs new file mode 100644 index 0000000000..070cdb6c89 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/python_nodes.rs @@ -0,0 +1,206 @@ +//! Python node dependency checks: known targets, no SQL inputs on loaders, no cycles. + +use std::collections::{HashMap, HashSet}; + +use crate::project_validation::main::validate_project_inputs::conflict; +use crate::project_validation::models::{DependencyFact, NodeFact, NodeKind, ValidationFailure}; + +/// `_python_node_dependency_key`: a decorated object's name, else the object itself. +#[derive(Clone, Copy, PartialEq, Eq)] +enum DependencyKey<'a> { + Name(&'a str), + Object(Option), +} + +fn dependency_key(dependency: &DependencyFact) -> Option> { + match dependency { + DependencyFact::SqlResource { .. } => None, + DependencyFact::Object { + decorated_name: Some(name), + .. + } => Some(DependencyKey::Name(name)), + DependencyFact::Object { owner, .. } => Some(DependencyKey::Object(*owner)), + } +} + +/// The loaders, tasks and assets a dependency key names, as `node_by_dependency_key` maps them. +struct ExecutableNodes<'a> { + nodes: &'a [NodeFact], + by_name: HashMap<&'a str, usize>, +} + +impl<'a> ExecutableNodes<'a> { + fn new(nodes: &'a [NodeFact]) -> Self { + let mut by_name: HashMap<&'a str, usize> = HashMap::new(); + for (index, node) in nodes.iter().enumerate() { + if node.kind != NodeKind::Check { + by_name.insert(&node.name, index); + } + } + Self { nodes, by_name } + } + + fn resolve(&self, key: DependencyKey<'_>) -> Option { + match key { + DependencyKey::Name(name) => self.by_name.get(name).copied(), + DependencyKey::Object(Some(owner)) if self.nodes[owner].kind != NodeKind::Check => { + Some(owner) + } + DependencyKey::Object(_) => None, + } + } +} + +/// `_validate_loader_dependencies`: no cycles through loader function dependencies. +pub(crate) fn validate_loader_dependencies(nodes: &[NodeFact]) -> Result<(), ValidationFailure> { + let next = |index: usize| -> Vec { + nodes[index] + .depends_on + .iter() + .filter_map(|dependency| match dependency { + DependencyFact::Object { + owner: Some(owner), .. + } if nodes[*owner].kind == NodeKind::Loader => Some(*owner), + _ => None, + }) + .collect() + }; + let roots: Vec = (0..nodes.len()) + .filter(|index| nodes[*index].kind == NodeKind::Loader) + .collect(); + find_cycle(nodes, &roots, next, "Loader dependency cycle detected") +} + +/// `_validate_python_node_dependencies` over loaders, tasks and assets. +pub(crate) fn validate_python_node_dependencies( + nodes: &[NodeFact], +) -> Result<(), ValidationFailure> { + let executable: ExecutableNodes<'_> = ExecutableNodes::new(nodes); + let roots: Vec = (0..nodes.len()) + .filter(|index| nodes[*index].kind != NodeKind::Check) + .collect(); + for index in &roots { + let node: &NodeFact = &nodes[*index]; + for dependency in &node.depends_on { + match dependency { + DependencyFact::SqlResource { name } => { + if node.kind == NodeKind::Loader { + return Err(conflict(format!( + "Loader '{}' depends on SQL resource '{name}'; loaders may depend on \ + Python loader/task/asset functions only", + node.name + ))); + } + } + DependencyFact::Object { .. } => { + let known: bool = dependency_key(dependency) + .and_then(|key| executable.resolve(key)) + .is_some(); + if known { + continue; + } + if node.kind == NodeKind::Loader { + return Err(conflict(format!( + "Loader '{}' depends on an unknown loader", + node.name + ))); + } + return Err(conflict(format!( + "Python node '{}' depends on an unknown Python node; use function \ + references to decorated loaders, tasks, or assets", + node.name + ))); + } + } + } + } + let next = |index: usize| -> Vec { + nodes[index] + .depends_on + .iter() + .filter_map(dependency_key) + .filter_map(|key| executable.resolve(key)) + .collect() + }; + find_cycle(nodes, &roots, next, "Python node dependency cycle detected") +} + +/// `_validate_check_dependencies`: checks depend on loaders, tasks and assets, never checks. +pub(crate) fn validate_check_dependencies(nodes: &[NodeFact]) -> Result<(), ValidationFailure> { + let allowed: HashSet<&str> = nodes + .iter() + .filter(|node| node.kind != NodeKind::Check) + .map(|node| node.name.as_str()) + .collect(); + let checks: HashSet<&str> = nodes + .iter() + .filter(|node| node.kind == NodeKind::Check) + .map(|node| node.name.as_str()) + .collect(); + for check in nodes.iter().filter(|node| node.kind == NodeKind::Check) { + for dependency in &check.depends_on { + let Some(key) = dependency_key(dependency) else { + continue; + }; + let name: Option<&str> = match key { + DependencyKey::Name(name) => Some(name), + DependencyKey::Object(_) => None, + }; + if name.is_some_and(|name| checks.contains(name)) { + return Err(conflict(format!( + "Check '{}' depends on another check; checks may depend on loaders, tasks, \ + assets, models, sources, and seeds only", + check.name + ))); + } + if !name.is_some_and(|name| allowed.contains(name)) { + return Err(conflict(format!( + "Check '{}' depends on an unknown Python node; use function references to \ + decorated loaders, tasks, or assets", + check.name + ))); + } + } + } + Ok(()) +} + +/// Python's recursive `visit`: depth-first from each root, sharing visited names across roots. +fn find_cycle( + nodes: &[NodeFact], + roots: &[usize], + next: impl Fn(usize) -> Vec, + label: &str, +) -> Result<(), ValidationFailure> { + let mut visited: HashSet<&str> = HashSet::new(); + for root in roots { + let mut visiting: Vec<&str> = Vec::new(); + visit(nodes, *root, &next, &mut visiting, &mut visited, label)?; + } + Ok(()) +} + +fn visit<'a>( + nodes: &'a [NodeFact], + index: usize, + next: &impl Fn(usize) -> Vec, + path: &mut Vec<&'a str>, + visited: &mut HashSet<&'a str>, + label: &str, +) -> Result<(), ValidationFailure> { + let name: &str = &nodes[index].name; + if visited.contains(name) { + return Ok(()); + } + if path.contains(&name) { + let cycle: Vec<&str> = path.iter().copied().chain(std::iter::once(name)).collect(); + return Err(conflict(format!("{label}: {}", cycle.join(" -> ")))); + } + path.push(name); + for dependency in next(index) { + visit(nodes, dependency, next, path, visited, label)?; + } + path.pop(); + visited.insert(name); + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs new file mode 100644 index 0000000000..f1d20ddd7b --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/resource_names.rs @@ -0,0 +1,72 @@ +//! Python's `_validate_unique_project_resource_names`: one namespace across resource kinds. + +use std::collections::{HashMap, HashSet}; + +use crate::project_validation::main::validate_project_inputs::{conflict, schema_seeds}; +use crate::project_validation::models::{NodeKind, ProjectFacts, ValidationFailure}; + +/// Reject the first resource name declared twice across models, sources, seeds and the rest. +pub(crate) fn validate_unique_project_resource_names( + facts: &ProjectFacts, +) -> Result<(), ValidationFailure> { + let mut seen: HashMap = HashMap::new(); + let mut declare = |name: &str, kind: &str, path: &str| -> Result<(), ValidationFailure> { + if let Some((existing_kind, existing_path)) = seen.get(name) { + return Err(conflict(format!( + "Project resource name '{name}' is declared as both {existing_kind} in \ + {existing_path} and {kind} in {path}; model, source, seed, function, loader, \ + task, asset, check, provider, hook, unit test, and scenario names must be \ + globally unique" + ))); + } + seen.insert(name.to_owned(), (kind.to_owned(), path.to_owned())); + Ok(()) + }; + for model in &facts.models { + declare(&model.file.name, "model", &model.file.path)?; + } + let mut managed_sources: HashSet<&str> = HashSet::new(); + for file in &facts.sources { + for entry in &file.entries { + if entry.managed { + managed_sources.insert(&entry.name); + } + declare(&entry.name, "source", &file.path)?; + } + } + for seed in schema_seeds(facts) { + declare(&seed.name, "seed", &seed.path)?; + } + for function in facts.sql_functions.iter().chain(&facts.python_functions) { + declare(&function.name, "function", &function.path)?; + } + for node in &facts.nodes { + if node.kind == NodeKind::Loader && managed_sources.contains(node.name.as_str()) { + continue; + } + declare(&node.name, node.kind.label(), &node.path)?; + } + for provider in &facts.providers { + declare(&provider.name, "provider", &provider.path)?; + } + for hook in &facts.sql_hooks { + declare(&hook.name, "sql hook", &hook.path)?; + } + for hook in &facts.hook_functions { + declare(&hook.name, "python hook", &hook.path)?; + } + for scenario in &facts.scenarios { + declare(&scenario.name, "scenario", &scenario.path)?; + } + let mut test_names: HashSet<&str> = HashSet::new(); + for test in &facts.tests { + for block in &test.blocks { + let name: &str = block.name.as_deref().unwrap_or(&test.file.name); + if !test_names.insert(name) { + continue; + } + declare(name, "unit test", &test.file.path)?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/seeds.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/seeds.rs new file mode 100644 index 0000000000..eceef3335e --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/seeds.rs @@ -0,0 +1,122 @@ +//! Python's `_validate_declared_seed_files`: every declared seed has one CSV whose header matches. + +use std::collections::{HashMap, HashSet}; + +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::_helpers::reading::read_failure; +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::project_validation::_helpers::csv_header::{HeaderError, csv_header}; +use crate::project_validation::models::{ + ProjectFacts, SeedEntryFact, SeedFileFact, ValidationFailure, +}; + +fn seed_error(message: String) -> ValidationFailure { + DiscoveryFailure::new(FailureKind::Seed, message).into() +} + +/// Reject undeclared, duplicate or mismatched seed CSVs in Python's order. +pub(crate) fn validate_declared_seed_files(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + unique_seed_csv_names(&facts.seeds)?; + let mut by_name: HashMap<&str, Vec<&SeedFileFact>> = HashMap::new(); + for seed in &facts.seeds { + by_name.entry(&seed.file.name).or_default().push(seed); + } + let mut declared: HashSet<&str> = HashSet::new(); + for schema in &facts.schemas { + for entry in &schema.seeds { + declared.insert(&entry.name); + let matching: &[&SeedFileFact] = + by_name.get(entry.name.as_str()).map_or(&[], Vec::as_slice); + match matching { + [] => { + return Err(seed_error(format!( + "Seed declaration '{}' in {} has no matching CSV file under seeds/", + entry.name, schema.path + ))); + } + [seed] => seed_csv_header(entry, seed)?, + _ => { + let paths: Vec<&str> = matching + .iter() + .map(|seed| seed.file.path.as_str()) + .collect(); + return Err(seed_error(format!( + "Seed declaration '{}' matches multiple CSV files: {}", + entry.name, + paths.join(", ") + ))); + } + } + } + } + for seed in &facts.seeds { + if !declared.contains(seed.file.name.as_str()) { + return Err(seed_error(format!( + "Seed CSV {} has no matching declaration for seed '{}' under seeds/**/*.yml", + seed.file.path, seed.file.name + ))); + } + } + Ok(()) +} + +fn unique_seed_csv_names(seeds: &[SeedFileFact]) -> Result<(), ValidationFailure> { + let mut seen: HashMap<&str, &str> = HashMap::new(); + for seed in seeds { + if let Some(existing) = seen.get(seed.file.name.as_str()) { + return Err(seed_error(format!( + "Duplicate seed CSV name '{}' found: {existing}, {}. Seed CSV filenames must be \ + unique under seeds/.", + seed.file.name, seed.file.path + ))); + } + seen.insert(&seed.file.name, &seed.file.path); + } + Ok(()) +} + +fn seed_csv_header(entry: &SeedEntryFact, seed: &SeedFileFact) -> Result<(), ValidationFailure> { + let unreadable = |failure| ValidationFailure::Unreadable { + absolute_path: seed.absolute_path.clone(), + failure, + }; + let bytes: Vec = + std::fs::read(&seed.absolute_path).map_err(|error| unreadable(read_failure(&error)))?; + let header: Vec = match csv_header(&bytes, &entry.csv) { + Ok(header) => header, + Err(HeaderError::Read(failure)) => return Err(unreadable(failure)), + Err(HeaderError::Csv(message)) => return Err(ValidationFailure::Csv(message)), + Err(HeaderError::Setting(message)) => return Err(ValidationFailure::CsvSetting(message)), + }; + let path: &str = &seed.file.path; + if header.is_empty() { + return Err(seed_error(format!("{path} must contain a CSV header row"))); + } + let mut seen: HashSet<&str> = HashSet::new(); + for column in &header { + if !seen.insert(column) { + return Err(seed_error(format!( + "{path} contains duplicate CSV header column '{column}'" + ))); + } + } + if header != entry.columns { + return Err(seed_error(format!( + "{path} header {} does not match declared seed columns {} for '{}'", + python_tuple_repr(&header), + python_tuple_repr(&entry.columns), + entry.name + ))); + } + Ok(()) +} + +/// Python's `repr` of a tuple of strings. +fn python_tuple_repr(items: &[String]) -> String { + let parts: Vec = items.iter().map(|item| python_str_repr(item)).collect(); + match parts.as_slice() { + [single] => format!("({single},)"), + _ => format!("({})", parts.join(", ")), + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/main/mod.rs b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs new file mode 100644 index 0000000000..1fc67badd8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs @@ -0,0 +1 @@ +pub mod validate_project_inputs; diff --git a/crates/sqlbuild-discovery/src/project_validation/main/validate_project_inputs.rs b/crates/sqlbuild-discovery/src/project_validation/main/validate_project_inputs.rs new file mode 100644 index 0000000000..7ce88a8949 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/main/validate_project_inputs.rs @@ -0,0 +1,354 @@ +//! Python's `validate_discovered_inputs`: cross-file conflicts across discovered project inputs. + +use std::collections::{HashMap, HashSet}; + +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::project_validation::_helpers::identities::validate_identities; +use crate::project_validation::_helpers::python_nodes::{ + validate_check_dependencies, validate_loader_dependencies, validate_python_node_dependencies, +}; +use crate::project_validation::_helpers::resource_names::validate_unique_project_resource_names; +use crate::project_validation::_helpers::seeds::validate_declared_seed_files; +use crate::project_validation::models::{ + ModelFact, NamedPath, NodeFact, NodeKind, ProjectFacts, SourceFileFact, ValidationFailure, +}; +use sqlbuild_model_config::path_defaults::main::matched_path_default_keys::matched_path_default_keys; + +const DLT_LOADER_KIND: &str = "dlt"; +const MODELS_PREFIX: &str = "models/"; +const CURRENT_DIRECTORY: &str = "."; +const KNOWN_FOLDER_LIMIT: usize = 5; +const RESERVED_MODEL_NAMES: [&str; 1] = ["_chain_"]; + +/// Validate cross-file conflicts in Python's order; the first conflict is reported. +pub fn validate_project_inputs(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + validate_identities(facts)?; + unique_model_file_names(&facts.models)?; + unique_named(&facts.scenarios, |name, existing, path| { + format!("Duplicate scenario file name found for '{name}' in {existing} and {path}") + })?; + unique_source_names(&facts.sources)?; + let loaders: Vec<&NodeFact> = nodes_of(facts, NodeKind::Loader); + unique_named( + &loaders + .iter() + .map(|node| node_path(node)) + .collect::>(), + |name, existing, path| { + format!("Duplicate source loader found for '{name}' in {existing} and {path}") + }, + )?; + unique_named( + &facts.nodes.iter().map(node_path).collect::>(), + |name, existing, path| { + format!( + "Duplicate Python node found for '{name}' in {existing} and {path}; loader, \ + task, asset, and check names must be globally unique" + ) + }, + )?; + source_loader_name_collisions(&facts.sources, &loaders)?; + validate_loader_dependencies(&facts.nodes)?; + validate_python_node_dependencies(&facts.nodes)?; + validate_check_dependencies(&facts.nodes)?; + source_loader_references(&facts.sources, &loaders)?; + terminal_loader_config_ownership(&facts.sources, &loaders)?; + let schema_models: Vec = facts + .schemas + .iter() + .flat_map(|schema| { + schema.model_names.iter().map(|name| NamedPath { + name: name.clone(), + path: schema.path.clone(), + }) + }) + .collect(); + unique_named(&schema_models, |name, existing, path| { + format!( + "Duplicate schema.yml model declaration found for '{name}' in {existing} and {path}" + ) + })?; + unique_named(&schema_seeds(facts), |name, existing, path| { + format!("Duplicate seed declaration found for '{name}' in {existing} and {path}") + })?; + unique_logical_relation_names(facts)?; + validate_declared_seed_files(facts)?; + validate_unique_project_resource_names(facts)?; + path_defaults_match_models(facts)?; + Ok(()) +} + +pub(crate) fn conflict(message: String) -> ValidationFailure { + DiscoveryFailure::new(FailureKind::Conflict, message).into() +} + +pub(crate) fn schema_seeds(facts: &ProjectFacts) -> Vec { + facts + .schemas + .iter() + .flat_map(|schema| { + schema.seeds.iter().map(|seed| NamedPath { + name: seed.name.clone(), + path: schema.path.clone(), + }) + }) + .collect() +} + +fn nodes_of(facts: &ProjectFacts, kind: NodeKind) -> Vec<&NodeFact> { + facts + .nodes + .iter() + .filter(|node| node.kind == kind) + .collect() +} + +fn node_path(node: &NodeFact) -> NamedPath { + NamedPath { + name: node.name.clone(), + path: node.path.clone(), + } +} + +/// Reject the second of two entries with the same name. +fn unique_named( + entries: &[impl std::borrow::Borrow], + message: impl Fn(&str, &str, &str) -> String, +) -> Result<(), ValidationFailure> { + let mut seen: HashMap<&str, &str> = HashMap::new(); + for entry in entries { + let entry: &NamedPath = entry.borrow(); + if let Some(existing) = seen.get(entry.name.as_str()) { + return Err(conflict(message(&entry.name, existing, &entry.path))); + } + seen.insert(&entry.name, &entry.path); + } + Ok(()) +} + +fn unique_model_file_names(models: &[ModelFact]) -> Result<(), ValidationFailure> { + let mut seen: HashMap<&str, &str> = HashMap::new(); + for model in models { + let NamedPath { name, path } = &model.file; + if RESERVED_MODEL_NAMES.contains(&name.as_str()) { + return Err(conflict(format!( + "Model name '{name}' in {path} is reserved by SQLBuild for compiled output structure" + ))); + } + if let Some(existing) = seen.get(name.as_str()) { + return Err(conflict(format!( + "Duplicate model file name found for '{name}' in {existing} and {path}" + ))); + } + seen.insert(name, path); + } + Ok(()) +} + +fn unique_source_names(sources: &[SourceFileFact]) -> Result<(), ValidationFailure> { + let entries: Vec = sources + .iter() + .flat_map(|file| { + file.entries.iter().map(|entry| NamedPath { + name: entry.name.clone(), + path: file.path.clone(), + }) + }) + .collect(); + unique_named(&entries, |name, existing, path| { + format!("Duplicate source declaration found for '{name}' in {existing} and {path}") + }) +} + +fn source_loader_name_collisions( + sources: &[SourceFileFact], + loaders: &[&NodeFact], +) -> Result<(), ValidationFailure> { + let loader_paths: HashMap<&str, &str> = loaders + .iter() + .map(|loader| (loader.name.as_str(), loader.path.as_str())) + .collect(); + for file in sources { + for entry in &file.entries { + let Some(loader_path) = loader_paths.get(entry.name.as_str()) else { + continue; + }; + if entry.managed && entry.loader.as_deref() == Some(entry.name.as_str()) { + continue; + } + return Err(conflict(format!( + "Source '{name}' in {path} conflicts with loader '{name}' in {loader_path}; \ + source and loader names must be globally unique unless the source is managed", + name = entry.name, + path = file.path, + ))); + } + } + Ok(()) +} + +/// Python's `integration_loader_name`. +fn integration_loader_name(kind: &str, source_name: &str) -> String { + if kind == DLT_LOADER_KIND { + source_name.to_owned() + } else { + format!("{kind}__{source_name}") + } +} + +fn source_loader_references( + sources: &[SourceFileFact], + loaders: &[&NodeFact], +) -> Result<(), ValidationFailure> { + let loader_names: HashSet<&str> = loaders.iter().map(|loader| loader.name.as_str()).collect(); + for file in sources { + for entry in &file.entries { + let expected: String = match &entry.integration_loader { + Some(integration) => integration_loader_name(&integration.kind, &entry.name), + None => entry.name.clone(), + }; + if entry.managed && entry.loader.as_deref() != Some(expected.as_str()) { + return Err(conflict(format!( + "Managed source '{}' in {} must use terminal loader '{expected}'", + entry.name, file.path + ))); + } + if entry.managed && !loader_names.contains(expected.as_str()) { + return Err(conflict(format!( + "Managed source '{}' in {} requires loader '{expected}'", + entry.name, file.path + ))); + } + match &entry.loader { + Some(loader) if !loader_names.contains(loader.as_str()) => { + return Err(conflict(format!( + "Source '{}' in {} references unknown loader '{loader}'", + entry.name, file.path + ))); + } + _ => {} + } + } + } + Ok(()) +} + +fn terminal_loader_config_ownership( + sources: &[SourceFileFact], + loaders: &[&NodeFact], +) -> Result<(), ValidationFailure> { + let by_name: HashMap<&str, &NodeFact> = loaders + .iter() + .map(|loader| (loader.name.as_str(), *loader)) + .collect(); + for file in sources { + for entry in &file.entries { + let Some(loader) = &entry.loader else { + continue; + }; + let configured: bool = by_name + .get(loader.as_str()) + .is_some_and(|loader| loader.decorator_config); + if configured { + return Err(conflict(format!( + "Source '{}' references loader '{loader}', but terminal source loader write \ + and schema config must be declared in source YAML, not the @loader decorator", + entry.name + ))); + } + } + } + Ok(()) +} + +fn unique_logical_relation_names(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + let mut seen: HashMap = HashMap::new(); + for model in &facts.models { + seen.insert(model.file.name.clone(), ("model", model.file.path.clone())); + } + let sources = facts.sources.iter().flat_map(|file| { + file.entries + .iter() + .map(move |entry| ("source", entry.name.clone(), file.path.clone())) + }); + let seeds = schema_seeds(facts) + .into_iter() + .map(|seed| ("seed", seed.name, seed.path)); + for (kind, name, path) in sources.chain(seeds) { + if let Some((existing_kind, existing_path)) = seen.get(&name) { + return Err(conflict(format!( + "Logical relation name '{name}' is declared as both {existing_kind} in \ + {existing_path} and {kind} in {path}" + ))); + } + seen.insert(name, (kind, path)); + } + Ok(()) +} + +fn path_defaults_match_models(facts: &ProjectFacts) -> Result<(), ValidationFailure> { + if facts.path_default_keys.is_empty() { + return Ok(()); + } + let model_paths: Vec = facts + .models + .iter() + .map(|model| { + let path: String = model.file.path.replace('\\', "/"); + path.strip_prefix(MODELS_PREFIX) + .map_or(path.clone(), str::to_owned) + }) + .collect(); + let mut matched: HashSet = HashSet::new(); + for model_path in &model_paths { + let keys: Vec = matched_path_default_keys(model_path, &facts.path_default_keys) + .map_err(|error| { + ValidationFailure::Discovery(DiscoveryFailure { + kind: FailureKind::Conflict, + message: error.message, + help: error.help, + }) + })?; + matched.extend(keys); + } + for key in &facts.path_default_keys { + if matched.contains(key) { + continue; + } + let mut folders: Vec = model_paths + .iter() + .map(|path| parent_folder(path)) + .filter(|folder| folder != CURRENT_DIRECTORY) + .collect::>() + .into_iter() + .collect(); + folders.sort(); + let known: String = if folders.is_empty() { + "".to_owned() + } else { + folders + .iter() + .take(KNOWN_FOLDER_LIMIT) + .cloned() + .collect::>() + .join(", ") + }; + return Err(conflict(format!( + "path_defaults['{key}'] does not match any model paths. Known model folders include: \ + {known}" + ))); + } + Ok(()) +} + +/// `str(Path(path).parent)` for a discovered POSIX relative path. +fn parent_folder(path: &str) -> String { + let parts: Vec<&str> = path + .split('/') + .filter(|part| !part.is_empty() && *part != CURRENT_DIRECTORY) + .collect(); + match parts.split_last() { + Some((_, parents)) if !parents.is_empty() => parents.join("/"), + _ => CURRENT_DIRECTORY.to_owned(), + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/mod.rs b/crates/sqlbuild-discovery/src/project_validation/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/project_validation/models.rs b/crates/sqlbuild-discovery/src/project_validation/models.rs new file mode 100644 index 0000000000..c861220554 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/models.rs @@ -0,0 +1,194 @@ +//! The facts cross-file discovery validation reads, in each collection's discovery order. + +use crate::models::{DiscoveryFailure, ReadFailure}; + +/// A resource name and the project-relative path that declares it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NamedPath { + pub name: String, + pub path: String, +} + +/// A discovered SQL model file: its stem, path and the declarations its header owns. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelFact { + pub file: NamedPath, + /// `(name, declaring path)` of the header's enums, then its constants. + pub local_declarations: Vec, +} + +/// One declarative integration loader a source names. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct IntegrationLoaderFact { + pub kind: String, +} + +/// One source entry of a source YAML file. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SourceFact { + pub name: String, + pub managed: bool, + pub loader: Option, + pub integration_loader: Option, +} + +/// A source YAML file and its entries. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SourceFileFact { + pub path: String, + pub entries: Vec, +} + +/// The decorated Python node kinds, in the order discovery lists them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NodeKind { + Loader, + Task, + Asset, + Check, +} + +impl NodeKind { + /// The kind as resource-name conflicts word it. + pub fn label(self) -> &'static str { + match self { + Self::Loader => "loader", + Self::Task => "task", + Self::Asset => "asset", + Self::Check => "check", + } + } +} + +/// One dependency a Python node declares. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DependencyFact { + /// A `SqlResourceRef` to a model, source or seed. + SqlResource { name: String }, + /// Any other dependency object. + Object { + /// The index of the discovered node whose function this object is. + owner: Option, + /// The name a loader, task, asset or check decorator records on the object. + decorated_name: Option, + }, +} + +/// A decorated Python loader, task, asset or check function. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NodeFact { + pub kind: NodeKind, + pub name: String, + pub path: String, + /// Whether a loader's decorator sets destination, write or schema config. + pub decorator_config: bool, + pub depends_on: Vec, +} + +/// The CSV reader settings a seed declaration authors. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CsvSettings { + pub delimiter: Option, + pub quotechar: Option, + pub escapechar: Option, + pub doublequote: Option, + pub skipinitialspace: Option, +} + +/// One seed entry of a schema YAML file. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SeedEntryFact { + pub name: String, + pub columns: Vec, + pub csv: CsvSettings, +} + +/// A schema YAML file's model and seed entries. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SchemaFileFact { + pub path: String, + pub model_names: Vec, + pub seeds: Vec, +} + +/// A seed CSV file: its stem and project-relative path, and where to read it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SeedFileFact { + pub file: NamedPath, + pub absolute_path: String, +} + +/// One SQL test block and its case names. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestBlockFact { + pub name: Option, + pub cases: Vec, +} + +/// A SQL test file: its path, stem and blocks. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestFileFact { + pub file: NamedPath, + pub blocks: Vec, +} + +/// An audit file: its path and stem, whether it declares a generic audit, and its block names. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AuditFileFact { + pub file: NamedPath, + pub generic: bool, + pub block_names: Vec>, +} + +/// An enum or constant declaration file's declarations and whether its scope is private. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ScopedDeclarationsFact { + pub private: bool, + pub declarations: Vec, +} + +/// Everything cross-file validation reads about one discovery. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ProjectFacts { + pub models: Vec, + pub sql_functions: Vec, + pub python_functions: Vec, + pub sql_hooks: Vec, + pub seeds: Vec, + pub schemas: Vec, + pub sources: Vec, + pub tests: Vec, + pub scenarios: Vec, + pub audits: Vec, + /// Enum declaration files, then constant declaration files. + pub declaration_files: Vec, + /// Every model schema declaration across the model schema files. + pub model_schemas: Vec, + pub materializations: Vec, + /// Loaders, tasks, assets, then checks. + pub nodes: Vec, + pub hook_functions: Vec, + pub event_exporters: Vec, + pub providers: Vec, + pub path_default_keys: Vec, +} + +/// Why discovered inputs are rejected: a discovery error, or a seed CSV Python cannot read. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ValidationFailure { + Discovery(DiscoveryFailure), + Unreadable { + absolute_path: String, + failure: ReadFailure, + }, + /// Python's `csv.Error` reading a seed header, with its message. + Csv(String), + /// Python's `TypeError` for a CSV setting that is not one character, with its message. + CsvSetting(String), +} + +impl From for ValidationFailure { + fn from(failure: DiscoveryFailure) -> Self { + Self::Discovery(failure) + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests.rs b/crates/sqlbuild-discovery/src/project_validation/tests.rs new file mode 100644 index 0000000000..d27fe09e71 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_csv_header.rs"] +mod csv_header; +#[path = "tests/test_validate_project_inputs.rs"] +mod validate_project_inputs; diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_csv_header.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_csv_header.rs new file mode 100644 index 0000000000..2a2de30ad7 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_csv_header.rs @@ -0,0 +1,167 @@ +use crate::project_validation::_helpers::csv_header::{HeaderError, csv_header}; +use crate::project_validation::models::CsvSettings; + +struct CsvHeaderTestCase { + description: &'static str, + text: &'static str, + settings: CsvSettings, + expected: &'static [&'static str], +} + +fn quoted() -> CsvSettings { + CsvSettings { + quotechar: Some("\"".to_owned()), + ..CsvSettings::default() + } +} + +/// Expectations recorded from Python's `next(csv.reader(...))` with the facade's arguments. +#[test] +fn given_seed_csv_text_when_reading_the_header_then_matches_python_csv_reader() { + let test_cases = [ + CsvHeaderTestCase { + description: "plain", + text: "id,name\n1,a\n", + settings: CsvSettings::default(), + expected: &["id", "name"], + }, + CsvHeaderTestCase { + description: "crlf", + text: "id,name\r\n1,a\r\n", + settings: CsvSettings::default(), + expected: &["id", "name"], + }, + CsvHeaderTestCase { + description: "bare cr", + text: "id,name\r1,a\r", + settings: CsvSettings::default(), + expected: &["id", "name"], + }, + CsvHeaderTestCase { + description: "no terminator", + text: "id,name", + settings: CsvSettings::default(), + expected: &["id", "name"], + }, + CsvHeaderTestCase { + description: "empty", + text: "", + settings: CsvSettings::default(), + expected: &[], + }, + CsvHeaderTestCase { + description: "blank first line", + text: "\nid\n", + settings: CsvSettings::default(), + expected: &[], + }, + CsvHeaderTestCase { + description: "unset quotechar keeps quotes", + text: "\"a,x\",b\n", + settings: CsvSettings::default(), + expected: &["\"a", "x\"", "b"], + }, + CsvHeaderTestCase { + description: "quoted with quotechar", + text: "\"a,x\",b\n", + settings: quoted(), + expected: &["a,x", "b"], + }, + CsvHeaderTestCase { + description: "doubled quote", + text: "\"a\"\"b\",c\n", + settings: quoted(), + expected: &["a\"b", "c"], + }, + CsvHeaderTestCase { + description: "no doublequote", + text: "\"a\"b,c\n", + settings: CsvSettings { + doublequote: Some(false), + ..quoted() + }, + expected: &["ab", "c"], + }, + CsvHeaderTestCase { + description: "quoted newline", + text: "\"a\nb\",c\n", + settings: quoted(), + expected: &["a\nb", "c"], + }, + CsvHeaderTestCase { + description: "semicolon", + text: "a;b;c\n", + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + expected: &["a", "b", "c"], + }, + CsvHeaderTestCase { + description: "skip initial space", + text: "a, b, c\n", + settings: CsvSettings { + skipinitialspace: Some(true), + ..CsvSettings::default() + }, + expected: &["a", "b", "c"], + }, + CsvHeaderTestCase { + description: "escape", + text: "a\\,b,c\n", + settings: CsvSettings { + escapechar: Some("\\".to_owned()), + ..CsvSettings::default() + }, + expected: &["a,b", "c"], + }, + CsvHeaderTestCase { + description: "trailing delimiter", + text: "a,b,\n", + settings: CsvSettings::default(), + expected: &["a", "b", ""], + }, + CsvHeaderTestCase { + description: "unterminated quote", + text: "\"abc", + settings: quoted(), + expected: &["abc"], + }, + CsvHeaderTestCase { + description: "bom", + text: "\u{feff}id,name\n", + settings: CsvSettings::default(), + expected: &["\u{feff}id", "name"], + }, + ]; + for test_case in test_cases { + let header: Vec = csv_header(test_case.text.as_bytes(), &test_case.settings) + .unwrap_or_else(|error| panic!("{}: {error:?}", test_case.description)); + assert_eq!(header, test_case.expected, "{}", test_case.description); + } +} + +#[test] +fn given_a_header_longer_than_one_decode_chunk_when_reading_then_reads_past_the_chunk() { + let long_name: String = "c".repeat(10_000); + let text: String = format!("{long_name},id\n1,2\n"); + + let header: Vec = csv_header(text.as_bytes(), &CsvSettings::default()).unwrap(); + + assert_eq!(header, vec![long_name, "id".to_owned()]); +} + +#[test] +fn given_csv_settings_python_rejects_when_reading_then_reports_its_type_error() { + let settings = CsvSettings { + quotechar: Some(String::new()), + ..CsvSettings::default() + }; + + let error = csv_header(b"a,b\n", &settings).unwrap_err(); + + assert!( + matches!(&error, HeaderError::Setting(message) if message == "\"quotechar\" must be a 1-character string"), + "{error:?}" + ); +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs new file mode 100644 index 0000000000..6634532437 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_validate_project_inputs.rs @@ -0,0 +1,294 @@ +use crate::models::{DiscoveryFailure, FailureKind}; +use crate::project_validation::main::validate_project_inputs::validate_project_inputs; +use crate::project_validation::models::{ + DependencyFact, ModelFact, NamedPath, NodeFact, NodeKind, ProjectFacts, SchemaFileFact, + SeedEntryFact, SeedFileFact, SourceFact, SourceFileFact, ValidationFailure, +}; + +fn named(name: &str, path: &str) -> NamedPath { + NamedPath { + name: name.to_owned(), + path: path.to_owned(), + } +} + +fn model(name: &str) -> ModelFact { + ModelFact { + file: named(name, &format!("models/{name}.sql")), + local_declarations: Vec::new(), + } +} + +fn node(kind: NodeKind, name: &str, depends_on: Vec) -> NodeFact { + NodeFact { + kind, + name: name.to_owned(), + path: format!("python/{name}.py"), + decorator_config: false, + depends_on, + } +} + +fn on(owner: usize, name: &str) -> DependencyFact { + DependencyFact::Object { + owner: Some(owner), + decorated_name: Some(name.to_owned()), + } +} + +fn failure(facts: &ProjectFacts) -> Option<(FailureKind, String)> { + match validate_project_inputs(facts) { + Ok(()) => None, + Err(ValidationFailure::Discovery(DiscoveryFailure { kind, message, .. })) => { + Some((kind, message)) + } + Err(other) => panic!("unexpected failure: {other:?}"), + } +} + +struct ValidationTestCase { + description: &'static str, + facts: ProjectFacts, + expected: Option<(FailureKind, &'static str)>, +} + +#[test] +fn given_discovered_facts_when_validating_then_reports_python_first_conflict() { + let test_cases = [ + ValidationTestCase { + description: "a valid project passes", + facts: ProjectFacts { + models: vec![model("orders"), model("customers")], + ..ProjectFacts::default() + }, + expected: None, + }, + ValidationTestCase { + description: "a non-canonical model identity is rejected first", + facts: ProjectFacts { + models: vec![model("Orders"), model("_chain_")], + ..ProjectFacts::default() + }, + expected: Some(( + FailureKind::ResourceIdentity, + "Invalid model identity 'Orders' in models/Orders.sql; use snake_case 'orders'", + )), + }, + ValidationTestCase { + description: "a reserved model name fails its identity check first", + facts: ProjectFacts { + models: vec![model("_chain_")], + ..ProjectFacts::default() + }, + expected: Some(( + FailureKind::ResourceIdentity, + "Invalid model identity '_chain_' in models/_chain_.sql; use snake_case 'chain'", + )), + }, + ValidationTestCase { + description: "a source and a model share a logical relation name", + facts: ProjectFacts { + models: vec![model("orders")], + sources: vec![SourceFileFact { + path: "sources/shop.yml".to_owned(), + entries: vec![SourceFact { + name: "orders".to_owned(), + ..SourceFact::default() + }], + }], + ..ProjectFacts::default() + }, + expected: Some(( + FailureKind::Conflict, + "Logical relation name 'orders' is declared as both model in models/orders.sql \ + and source in sources/shop.yml", + )), + }, + ValidationTestCase { + description: "a task cycle names its path", + facts: ProjectFacts { + nodes: vec![ + node(NodeKind::Task, "extract", vec![on(1, "load")]), + node(NodeKind::Task, "load", vec![on(0, "extract")]), + ], + ..ProjectFacts::default() + }, + expected: Some(( + FailureKind::Conflict, + "Python node dependency cycle detected: extract -> load -> extract", + )), + }, + ValidationTestCase { + description: "a check depending on a check", + facts: ProjectFacts { + nodes: vec![ + node(NodeKind::Check, "first", Vec::new()), + node(NodeKind::Check, "second", vec![on(0, "first")]), + ], + ..ProjectFacts::default() + }, + expected: Some(( + FailureKind::Conflict, + "Check 'second' depends on another check; checks may depend on loaders, tasks, \ + assets, models, sources, and seeds only", + )), + }, + ValidationTestCase { + description: "an undeclared seed CSV", + facts: ProjectFacts { + seeds: vec![SeedFileFact { + file: named("countries", "seeds/countries.csv"), + absolute_path: "/missing/seeds/countries.csv".to_owned(), + }], + ..ProjectFacts::default() + }, + expected: Some(( + FailureKind::Seed, + "Seed CSV seeds/countries.csv has no matching declaration for seed 'countries' \ + under seeds/**/*.yml", + )), + }, + ValidationTestCase { + description: "an unmatched path default lists known folders", + facts: ProjectFacts { + models: vec![ModelFact { + file: named("orders", "models/marts/orders.sql"), + local_declarations: Vec::new(), + }], + path_default_keys: vec!["staging".to_owned()], + ..ProjectFacts::default() + }, + expected: Some(( + FailureKind::Conflict, + "path_defaults['staging'] does not match any model paths. Known model folders \ + include: marts", + )), + }, + ]; + for test_case in test_cases { + let expected: Option<(FailureKind, String)> = test_case + .expected + .map(|(kind, message)| (kind, message.to_owned())); + assert_eq!( + failure(&test_case.facts), + expected, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_seed_whose_header_differs_when_validating_then_reports_python_tuple_reprs() { + let directory = tempfile::tempdir().unwrap(); + let csv_path = directory.path().join("countries.csv"); + std::fs::write(&csv_path, "code,label\nGB,United Kingdom\n").unwrap(); + let facts = ProjectFacts { + seeds: vec![SeedFileFact { + file: named("countries", "seeds/countries.csv"), + absolute_path: csv_path.to_string_lossy().into_owned(), + }], + schemas: vec![SchemaFileFact { + path: "seeds/countries.yml".to_owned(), + model_names: Vec::new(), + seeds: vec![SeedEntryFact { + name: "countries".to_owned(), + columns: vec!["code".to_owned()], + ..SeedEntryFact::default() + }], + }], + ..ProjectFacts::default() + }; + + let result = failure(&facts); + + assert_eq!( + result, + Some(( + FailureKind::Seed, + "seeds/countries.csv header ('code', 'label') does not match declared seed columns \ + ('code',) for 'countries'" + .to_owned() + )) + ); +} + +struct PathDefaultsTestCase { + description: &'static str, + model_paths: &'static [&'static str], + keys: &'static [&'static str], + expected_fragment: Option<&'static str>, +} + +#[test] +fn given_path_defaults_when_validating_then_every_key_must_match_a_model_path() { + let test_cases = [ + PathDefaultsTestCase { + description: "accepts Windows-style model paths that match path defaults", + model_paths: &["models\\staging\\orders.sql"], + keys: &["staging"], + expected_fragment: None, + }, + PathDefaultsTestCase { + description: "accepts recursive glob matching nested staging prefix", + model_paths: &["models/commerce/partner_feed/staging/orders.sql"], + keys: &["commerce/**/staging"], + expected_fragment: None, + }, + PathDefaultsTestCase { + description: "accepts recursive glob matching zero segments", + model_paths: &["models/commerce/staging/orders.sql"], + keys: &["commerce/**/staging"], + expected_fragment: None, + }, + PathDefaultsTestCase { + description: "counts broad glob as matched when a literal wins selection", + model_paths: &["models/commerce/partner_feed/staging/orders.sql"], + keys: &["commerce/**/staging", "commerce/partner_feed/staging"], + expected_fragment: None, + }, + PathDefaultsTestCase { + description: "rejects recursive glob matching no model paths", + model_paths: &["models/commerce/partner_feed/marts/orders.sql"], + keys: &["commerce/**/staging"], + expected_fragment: Some( + "path_defaults['commerce/**/staging'] does not match any model paths", + ), + }, + PathDefaultsTestCase { + description: "rejects equally specific matching globs during discovery", + model_paths: &["models/commerce/eu/staging/orders.sql"], + keys: &["commerce/*/staging", "commerce/eu/*"], + expected_fragment: Some("matches equally specific path_defaults keys"), + }, + ]; + for test_case in test_cases { + let facts = ProjectFacts { + models: test_case + .model_paths + .iter() + .map(|path| ModelFact { + file: named("orders", path), + local_declarations: Vec::new(), + }) + .collect(), + path_default_keys: test_case.keys.iter().map(|key| (*key).to_owned()).collect(), + ..ProjectFacts::default() + }; + + let result = failure(&facts); + + match test_case.expected_fragment { + None => assert_eq!(result, None, "{}", test_case.description), + Some(fragment) => { + let (kind, message) = result.unwrap_or_else(|| panic!("{}", test_case.description)); + assert_eq!(kind, FailureKind::Conflict, "{}", test_case.description); + assert!( + message.contains(fragment), + "{}: {message}", + test_case.description + ); + } + } + } +} diff --git a/crates/sqlbuild-model-config/src/path_defaults/main/matched_path_default_keys.rs b/crates/sqlbuild-model-config/src/path_defaults/main/matched_path_default_keys.rs new file mode 100644 index 0000000000..73afea3a66 --- /dev/null +++ b/crates/sqlbuild-model-config/src/path_defaults/main/matched_path_default_keys.rs @@ -0,0 +1,32 @@ +//! `select_path_default(...).matched_keys`: every key matching a model path, sorted. + +use crate::errors::ConfigError; +use crate::path_defaults::_helpers::matching::matches_prefix; +use crate::path_defaults::constants::MODELS_PREFIX; +use crate::path_defaults::main::select_path_default::select_path_default; +use crate::path_defaults::models::PathDefaultChoice; + +/// The sorted keys matching `model_path`, or the conflict selecting among them raises. +pub fn matched_path_default_keys( + model_path: &str, + keys: &[String], +) -> Result, ConfigError> { + if let PathDefaultChoice::Conflict(error) = select_path_default(model_path, keys) { + return Err(error); + } + let normalized: String = model_path.replace('\\', "/"); + let normalized: &str = normalized + .strip_prefix(MODELS_PREFIX) + .unwrap_or(&normalized); + let path_parts: Vec<&str> = normalized + .split('/') + .filter(|part| !part.is_empty()) + .collect(); + let mut matched: Vec = keys + .iter() + .filter(|key| matches_prefix(&path_parts, key)) + .cloned() + .collect(); + matched.sort(); + Ok(matched) +} diff --git a/crates/sqlbuild-model-config/src/path_defaults/main/mod.rs b/crates/sqlbuild-model-config/src/path_defaults/main/mod.rs index ac0e1e3b10..775de8bc48 100644 --- a/crates/sqlbuild-model-config/src/path_defaults/main/mod.rs +++ b/crates/sqlbuild-model-config/src/path_defaults/main/mod.rs @@ -1 +1,2 @@ +pub mod matched_path_default_keys; pub mod select_path_default; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index e965bf94db..e2050b597f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -5,5 +5,6 @@ pub(crate) mod key_checks; pub(crate) mod locations; pub(crate) mod model_file_record; pub(crate) mod project_files; +pub(crate) mod project_validation; pub(crate) mod registration; pub(crate) mod sql_test_blocks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index 79951291d9..8837e4e9b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -265,7 +265,7 @@ impl NativeModelFile { let other: &NativeModelFile = other.get(); for (left, right) in self.fields(py)?.iter().zip(other.fields(py)?) { if !left.eq(right)? { - return Ok(false.into_py_any(py)?); + return false.into_py_any(py); } } true.into_py_any(py) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs new file mode 100644 index 0000000000..7e9a97e4f3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_validation.rs @@ -0,0 +1,255 @@ +//! Cross-file validation of one discovery's inputs, from the facts the Python facade lists. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_discovery::project_validation::main::validate_project_inputs::validate_project_inputs; +use sqlbuild_discovery::project_validation::models::{ + AuditFileFact, CsvSettings, DependencyFact, IntegrationLoaderFact, ModelFact, NamedPath, + NodeFact, NodeKind, ProjectFacts, SchemaFileFact, ScopedDeclarationsFact, SeedEntryFact, + SeedFileFact, SourceFact, SourceFileFact, TestBlockFact, TestFileFact, ValidationFailure, +}; + +use crate::bindings::_helpers::discovery::project_files::{ + PyObject, failure_object, object, read_object, tuple_object, +}; + +const CSV_ERROR_TAG: &str = "csv"; +const CSV_SETTING_TAG: &str = "csv_setting"; +const UNREADABLE_TAG: &str = "unreadable"; + +type Pair = (String, String); +/// `(stem, path, blocks)` of a SQL test file; a block is `(name, case names)`. +type TestFileRow = (String, String, Vec<(Option, Vec)>); + +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct SeedEntryRow { + name: String, + columns: Vec, + delimiter: Option, + quotechar: Option, + escapechar: Option, + doublequote: Option, + skipinitialspace: Option, +} + +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct SourceRow { + name: String, + managed: bool, + loader: Option, + integration_kind: Option, +} + +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct NodeRow { + kind: String, + name: String, + path: String, + decorator_config: bool, + /// `(SQL resource name, owning node index, decorated name)` per dependency. + depends_on: Vec<(Option, Option, Option)>, +} + +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct FactRows { + models: Vec<(String, String, Vec)>, + sql_functions: Vec, + python_functions: Vec, + sql_hooks: Vec, + seeds: Vec<(String, String, String)>, + schemas: Vec<(String, Vec, Vec)>, + sources: Vec<(String, Vec)>, + tests: Vec, + scenarios: Vec, + audits: Vec<(Pair, bool, Vec>)>, + declaration_files: Vec<(bool, Vec)>, + model_schemas: Vec, + materializations: Vec, + nodes: Vec, + hook_functions: Vec, + event_exporters: Vec, + providers: Vec, + path_default_keys: Vec, +} + +fn named((name, path): Pair) -> NamedPath { + NamedPath { name, path } +} + +fn all_named(rows: Vec) -> Vec { + rows.into_iter().map(named).collect() +} + +fn node_kind(kind: &str) -> PyResult { + match kind { + "loader" => Ok(NodeKind::Loader), + "task" => Ok(NodeKind::Task), + "asset" => Ok(NodeKind::Asset), + "check" => Ok(NodeKind::Check), + other => Err(pyo3::exceptions::PyValueError::new_err(format!( + "unknown Python node kind {other:?}" + ))), + } +} + +fn project_facts(rows: FactRows) -> PyResult { + let nodes: Vec = rows + .nodes + .into_iter() + .map(|node| { + Ok(NodeFact { + kind: node_kind(&node.kind)?, + name: node.name, + path: node.path, + decorator_config: node.decorator_config, + depends_on: node + .depends_on + .into_iter() + .map(|(sql_name, owner, decorated_name)| match sql_name { + Some(name) => DependencyFact::SqlResource { name }, + None => DependencyFact::Object { + owner, + decorated_name, + }, + }) + .collect(), + }) + }) + .collect::>()?; + Ok(ProjectFacts { + models: rows + .models + .into_iter() + .map(|(name, path, local)| ModelFact { + file: NamedPath { name, path }, + local_declarations: all_named(local), + }) + .collect(), + sql_functions: all_named(rows.sql_functions), + python_functions: all_named(rows.python_functions), + sql_hooks: all_named(rows.sql_hooks), + seeds: rows + .seeds + .into_iter() + .map(|(name, path, absolute_path)| SeedFileFact { + file: NamedPath { name, path }, + absolute_path, + }) + .collect(), + schemas: rows + .schemas + .into_iter() + .map(|(path, model_names, seeds)| SchemaFileFact { + path, + model_names, + seeds: seeds + .into_iter() + .map(|seed| SeedEntryFact { + name: seed.name, + columns: seed.columns, + csv: CsvSettings { + delimiter: seed.delimiter, + quotechar: seed.quotechar, + escapechar: seed.escapechar, + doublequote: seed.doublequote, + skipinitialspace: seed.skipinitialspace, + }, + }) + .collect(), + }) + .collect(), + sources: rows + .sources + .into_iter() + .map(|(path, entries)| SourceFileFact { + path, + entries: entries + .into_iter() + .map(|entry| SourceFact { + name: entry.name, + managed: entry.managed, + loader: entry.loader, + integration_loader: entry + .integration_kind + .map(|kind| IntegrationLoaderFact { kind }), + }) + .collect(), + }) + .collect(), + tests: rows + .tests + .into_iter() + .map(|(name, path, blocks)| TestFileFact { + file: NamedPath { name, path }, + blocks: blocks + .into_iter() + .map(|(name, cases)| TestBlockFact { name, cases }) + .collect(), + }) + .collect(), + scenarios: all_named(rows.scenarios), + audits: rows + .audits + .into_iter() + .map(|(file, generic, block_names)| AuditFileFact { + file: named(file), + generic, + block_names, + }) + .collect(), + declaration_files: rows + .declaration_files + .into_iter() + .map(|(private, declarations)| ScopedDeclarationsFact { + private, + declarations: all_named(declarations), + }) + .collect(), + model_schemas: all_named(rows.model_schemas), + materializations: all_named(rows.materializations), + nodes, + hook_functions: all_named(rows.hook_functions), + event_exporters: all_named(rows.event_exporters), + providers: all_named(rows.providers), + path_default_keys: rows.path_default_keys, + }) +} + +/// `None` when the inputs are consistent, else the payload of the first error Python raises. +#[pyfunction] +fn validate_discovered_inputs(py: Python<'_>, rows: FactRows) -> PyResult> { + let facts: ProjectFacts = project_facts(rows)?; + let failure: ValidationFailure = match validate_project_inputs(&facts) { + Ok(()) => return Ok(None), + Err(failure) => failure, + }; + let payload: PyObject = match failure { + ValidationFailure::Discovery(failure) => failure_object(py, failure)?, + ValidationFailure::Unreadable { + absolute_path, + failure, + } => tuple_object( + py, + vec![ + object(py, UNREADABLE_TAG)?, + object(py, absolute_path)?, + read_object(py, failure)?, + ], + )?, + ValidationFailure::Csv(message) => { + tuple_object(py, vec![object(py, CSV_ERROR_TAG)?, object(py, message)?])? + } + ValidationFailure::CsvSetting(message) => { + tuple_object(py, vec![object(py, CSV_SETTING_TAG)?, object(py, message)?])? + } + }; + Ok(Some(payload)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(validate_discovered_inputs, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index d4c6034ae8..21c91bebfb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -4,7 +4,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::discovery::{ declaration_files, globbing, header_declarations, key_checks, locations, project_files, - sql_test_blocks, + project_validation, sql_test_blocks, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -15,5 +15,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { header_declarations::register(module)?; key_checks::register(module)?; globbing::register(module)?; + project_validation::register(module)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index d2c88aac9b..162f0f9d47 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 146269 +remaining_lines = 145515 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8cf1a6b0e9..a6ab3f56ed 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -872,6 +872,7 @@ class DiscoveredSqlModelFile: @property def extract_implicit_alias_columns(self) -> bool: ... +def validate_discovered_inputs(facts: dict[str, object]) -> tuple[object, ...] | None: ... def discover_model_files( request: dict[str, object], tree: NativeProjectTree, project_dir: Path ) -> list[tuple[Path, object]] | tuple[object, ...]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py index 2cf3ff2943..bb40b05506 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/model_files.py @@ -69,9 +69,7 @@ def discover_native_model_files( ) -def _model_file( - *, project_dir: Path, relative_path: Path, item: object -) -> DiscoveredSqlModelFile: +def _model_file(*, project_dir: Path, relative_path: Path, item: object) -> DiscoveredSqlModelFile: if isinstance(item, DiscoveredSqlModelFile): return item payload: tuple[object, ...] = cast(tuple[object, ...], item) diff --git a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py index 2606887e43..b6255bcc15 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py +++ b/src/sqlbuild/compiler/discovery/_helpers/native/payloads.py @@ -16,10 +16,12 @@ ) from sqlbuild.compiler.discovery.exceptions import ( DeclarationParseError, + DiscoveryConflictError, DiscoveryError, ModelSqlParseError, ProjectPathError, SchemaParseError, + SeedDiscoveryError, SourceParseError, SqlAuditParseError, SqlHookParseError, @@ -58,6 +60,8 @@ "sql_hook": SqlHookParseError, "sql_audit": SqlAuditParseError, "resource_identity": ResourceIdentityError, + "conflict": DiscoveryConflictError, + "seed": SeedDiscoveryError, } diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index ee1920118e..7993c6f15b 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -1,275 +1,198 @@ -"""Cross-file discovery validation helpers.""" +"""Cross-file discovery validation, run natively over the facts of one discovery.""" from __future__ import annotations import csv +from collections.abc import Callable, Iterable from pathlib import Path +from typing import Protocol, cast -from sqlbuild.compiler.discovery._helpers.integrations.loaders import integration_loader_name -from sqlbuild.compiler.discovery._helpers.validation.identities import ( - validate_discovered_resource_identities, -) -from sqlbuild.compiler.discovery.constants import CURRENT_DIRECTORY_PATH, RESERVED_MODEL_NAMES -from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError, SeedDiscoveryError +import sqlbuild._native as _native +from sqlbuild.compiler.discovery._helpers.native.payloads import native_failure, native_read_error from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, - DiscoveredHookFunction, DiscoveredLoaderFunction, DiscoveredProjectInputs, - DiscoveredProvider, - DiscoveredPythonFunctionFile, - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSourceFile, - DiscoveredSqlFunctionFile, - DiscoveredSqlHookFile, - DiscoveredSqlModelFile, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, DiscoveredTaskFunction, ) -from sqlbuild.compiler.path_defaults.main._select import select_path_default +from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_check_definition import read_check_definition from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition from sqlbuild.python_nodes.main.read_task_definition import read_task_definition -from sqlbuild.python_nodes.models import ( - AssetDefinition, - CheckDefinition, - LoaderDefinition, - SqlResourceRef, - TaskDefinition, -) -from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry, SourceEntry - - -def validate_discovered_inputs(discovered_inputs: DiscoveredProjectInputs) -> None: - """Validate cross-file conflicts across discovered project inputs.""" - - validate_discovered_resource_identities(discovered_inputs) - _validate_unique_model_file_names(discovered_inputs.model_files) - _validate_unique_scenario_file_names(discovered_inputs.scenario_files) - _validate_unique_source_names(discovered_inputs.source_files) - _validate_unique_loader_names(discovered_inputs.loader_functions) - _validate_unique_python_node_names( - loader_functions=discovered_inputs.loader_functions, - task_functions=discovered_inputs.task_functions, - asset_functions=discovered_inputs.asset_functions, - check_functions=discovered_inputs.check_functions, - ) - _validate_source_loader_name_collisions( - source_files=discovered_inputs.source_files, - loader_functions=discovered_inputs.loader_functions, - ) - _validate_loader_dependencies(discovered_inputs.loader_functions) - _validate_python_node_dependencies( - loader_functions=discovered_inputs.loader_functions, - task_functions=discovered_inputs.task_functions, - asset_functions=discovered_inputs.asset_functions, - ) - _validate_check_dependencies( - loader_functions=discovered_inputs.loader_functions, - task_functions=discovered_inputs.task_functions, - asset_functions=discovered_inputs.asset_functions, - check_functions=discovered_inputs.check_functions, - ) - _validate_source_loader_references( - source_files=discovered_inputs.source_files, - loader_functions=discovered_inputs.loader_functions, - ) - _validate_terminal_loader_config_ownership( - source_files=discovered_inputs.source_files, - loader_functions=discovered_inputs.loader_functions, - ) - _validate_unique_schema_model_names(discovered_inputs.schema_files) - _validate_unique_schema_seed_names(discovered_inputs.schema_files) - _validate_unique_logical_relation_names( - model_files=discovered_inputs.model_files, - source_files=discovered_inputs.source_files, - schema_files=discovered_inputs.schema_files, - ) - _validate_declared_seed_files( - schema_files=discovered_inputs.schema_files, - seed_files=discovered_inputs.seed_files, - ) - _validate_unique_project_resource_names(discovered_inputs) - _validate_path_defaults_match_models( - path_defaults=discovered_inputs.project_config.path_defaults, - model_files=discovered_inputs.model_files, - ) +from sqlbuild.python_nodes.models import SqlResourceRef +type _PythonNode = ( + DiscoveredLoaderFunction + | DiscoveredTaskFunction + | DiscoveredAssetFunction + | DiscoveredCheckFunction +) +type _Named = tuple[str, str] -def _validate_unique_model_file_names(model_files: tuple[DiscoveredSqlModelFile, ...]) -> None: - seen_names: dict[str, str] = {} - model_file: DiscoveredSqlModelFile - for model_file in model_files: - model_name: str = model_file.file_path.stem - if model_name in RESERVED_MODEL_NAMES: - raise DiscoveryConflictError( - f"Model name '{model_name}' in {model_file.relative_path} is reserved " - "by SQLBuild for compiled output structure" - ) - existing_path: str | None = seen_names.get(model_name) - if existing_path is not None: - raise DiscoveryConflictError( - f"Duplicate model file name found for '{model_name}' in " - f"{existing_path} and {model_file.relative_path}" - ) - seen_names[model_name] = str(model_file.relative_path) +class _NamedResource(Protocol): + @property + def name(self) -> str: ... + @property + def relative_path(self) -> Path: ... -def _validate_unique_scenario_file_names( - scenario_files: tuple[DiscoveredSqlScenarioFile, ...], -) -> None: - seen_names: dict[str, str] = {} - scenario_file: DiscoveredSqlScenarioFile - for scenario_file in scenario_files: - existing_path: str | None = seen_names.get(scenario_file.name) - if existing_path is not None: - raise DiscoveryConflictError( - f"Duplicate scenario file name found for '{scenario_file.name}' in " - f"{existing_path} and {scenario_file.relative_path}" - ) - seen_names[scenario_file.name] = str(scenario_file.relative_path) +class _ResourceFile(Protocol): + @property + def file_path(self) -> Path: ... + @property + def relative_path(self) -> Path: ... -def _validate_unique_source_names(source_files: tuple[DiscoveredSourceFile, ...]) -> None: - seen_names: dict[str, str] = {} - source_file: DiscoveredSourceFile - for source_file in source_files: - source_entry: SourceEntry - for source_entry in source_file.source_entries: - existing_path: str | None = seen_names.get(source_entry.name) - if existing_path is not None: - raise DiscoveryConflictError( - "Duplicate source declaration found for " - f"'{source_entry.name}' in {existing_path} and {source_file.relative_path}" - ) - seen_names[source_entry.name] = str(source_file.relative_path) +class _NodeDefinition(Protocol): + @property + def name(self) -> str: ... -def _validate_unique_loader_names(loader_functions: tuple[DiscoveredLoaderFunction, ...]) -> None: - seen_names: dict[str, str] = {} - loader_function: DiscoveredLoaderFunction - for loader_function in loader_functions: - existing_path: str | None = seen_names.get(loader_function.name) - if existing_path is not None: - raise DiscoveryConflictError( - f"Duplicate source loader found for '{loader_function.name}' in " - f"{existing_path} and {loader_function.relative_path}" - ) - seen_names[loader_function.name] = str(loader_function.relative_path) +_UNREADABLE_TAG: str = "unreadable" +_CSV_ERROR_TAG: str = "csv" +_CSV_SETTING_TAG: str = "csv_setting" +_DEFINITION_READERS: tuple[Callable[[Callable[..., object]], _NodeDefinition | None], ...] = ( + read_loader_definition, + read_task_definition, + read_asset_definition, + read_check_definition, +) -def _validate_unique_python_node_names( - *, - loader_functions: tuple[DiscoveredLoaderFunction, ...], - task_functions: tuple[DiscoveredTaskFunction, ...], - asset_functions: tuple[DiscoveredAssetFunction, ...], - check_functions: tuple[DiscoveredCheckFunction, ...], -) -> None: - seen_names: dict[str, str] = {} - for nodes in ( - loader_functions, - task_functions, - asset_functions, - check_functions, - ): - for node in nodes: - existing_path: str | None = seen_names.get(node.name) - if existing_path is not None: - raise DiscoveryConflictError( - f"Duplicate Python node found for '{node.name}' in " - f"{existing_path} and {node.relative_path}; loader, task, asset, and check " - "names must be globally unique" - ) - seen_names[node.name] = str(node.relative_path) +def validate_discovered_inputs(discovered_inputs: DiscoveredProjectInputs) -> None: + """Validate cross-file conflicts across discovered project inputs.""" -def _validate_source_loader_name_collisions( - *, - source_files: tuple[DiscoveredSourceFile, ...], - loader_functions: tuple[DiscoveredLoaderFunction, ...], -) -> None: - loader_paths_by_name: dict[str, str] = { - loader.name: str(loader.relative_path) for loader in loader_functions + payload: tuple[object, ...] | None = _native.validate_discovered_inputs( + _facts(discovered_inputs) + ) + if payload is None: + return + tag: object = payload[0] + if tag == _UNREADABLE_TAG: + raise native_read_error( + payload=cast(tuple[object, ...], payload[2]), file_path=Path(str(payload[1])) + ) + if tag == _CSV_ERROR_TAG: + raise csv.Error(str(payload[1])) + if tag == _CSV_SETTING_TAG: + raise TypeError(str(payload[1])) + raise native_failure(payload) + + +def _facts(inputs: DiscoveredProjectInputs) -> dict[str, object]: + nodes: tuple[tuple[str, _PythonNode], ...] = ( + *(("loader", node) for node in inputs.loader_functions), + *(("task", node) for node in inputs.task_functions), + *(("asset", node) for node in inputs.asset_functions), + *(("check", node) for node in inputs.check_functions), + ) + owners: dict[object, int] = {node.function: index for index, (_, node) in enumerate(nodes)} + return { + "models": [ + ( + model.file_path.stem, + str(model.relative_path), + _named_paths((*model.enum_declarations, *model.constant_declarations)), + ) + for model in inputs.model_files + ], + "sql_functions": _stems(inputs.sql_function_files), + "python_functions": _stems(inputs.python_function_files), + "sql_hooks": _named_paths(inputs.sql_hook_files), + "seeds": [ + (seed.file_path.stem, str(seed.relative_path), str(seed.file_path)) + for seed in inputs.seed_files + ], + "schemas": [ + ( + str(schema.relative_path), + [entry.name for entry in schema.model_entries], + [ + { + "name": entry.name, + "columns": [column.name for column in entry.columns], + "delimiter": entry.csv_settings.delimiter, + "quotechar": entry.csv_settings.quotechar, + "escapechar": entry.csv_settings.escapechar, + "doublequote": entry.csv_settings.doublequote, + "skipinitialspace": entry.csv_settings.skipinitialspace, + } + for entry in schema.seed_entries + ], + ) + for schema in inputs.schema_files + ], + "sources": [ + ( + str(source.relative_path), + [ + { + "name": entry.name, + "managed": entry.managed, + "loader": entry.loader, + "integration_kind": ( + None + if entry.integration_loader is None + else entry.integration_loader.kind + ), + } + for entry in source.source_entries + ], + ) + for source in inputs.source_files + ], + "tests": [ + ( + test.relative_path.stem, + str(test.relative_path), + [(block.name, [case.name for case in block.cases]) for block in test.blocks], + ) + for test in inputs.test_files + ], + "scenarios": _named_paths(inputs.scenario_files), + "audits": [ + ( + (audit.relative_path.stem, str(audit.relative_path)), + audit.declaration_kind is DeclarationKind.AUDIT, + [block.name for block in audit.blocks], + ) + for audit in inputs.audit_files + ], + "declaration_files": [ + (file.scope_kind is ScopeKind.PRIVATE, _named_paths(file.declarations)) + for file in (*inputs.enum_files, *inputs.constant_files) + ], + "model_schemas": _named_paths( + declaration + for schema in inputs.model_schema_files + for declaration in schema.declarations + ), + "materializations": _named_paths(inputs.materialization_files), + "nodes": [_node(kind=kind, node=node, owners=owners) for kind, node in nodes], + "hook_functions": _named_paths(inputs.hook_functions), + "event_exporters": _named_paths(inputs.event_exporters), + "providers": _named_paths(inputs.providers), + "path_default_keys": list(inputs.project_config.path_defaults), } - source_file: DiscoveredSourceFile - for source_file in source_files: - source_entry: SourceEntry - for source_entry in source_file.source_entries: - loader_path: str | None = loader_paths_by_name.get(source_entry.name) - if loader_path is None: - continue - if source_entry.managed and source_entry.loader == source_entry.name: - continue - raise DiscoveryConflictError( - f"Source '{source_entry.name}' in {source_file.relative_path} conflicts with " - f"loader '{source_entry.name}' in {loader_path}; source and loader names must " - "be globally unique unless the source is managed" - ) -def _validate_source_loader_references( - *, - source_files: tuple[DiscoveredSourceFile, ...], - loader_functions: tuple[DiscoveredLoaderFunction, ...], -) -> None: - loader_names: frozenset[str] = frozenset(loader.name for loader in loader_functions) - source_file: DiscoveredSourceFile - for source_file in source_files: - source_entry: SourceEntry - for source_entry in source_file.source_entries: - expected_loader: str | None = source_entry.name - if source_entry.integration_loader is not None: - expected_loader = integration_loader_name( - kind=source_entry.integration_loader.kind, - source_name=source_entry.name, - ) - if source_entry.managed and source_entry.loader != expected_loader: - raise DiscoveryConflictError( - f"Managed source '{source_entry.name}' in {source_file.relative_path} must " - f"use terminal loader '{expected_loader}'" - ) - if source_entry.managed and expected_loader not in loader_names: - raise DiscoveryConflictError( - f"Managed source '{source_entry.name}' in {source_file.relative_path} " - f"requires loader '{expected_loader}'" - ) - if source_entry.loader is None or source_entry.loader in loader_names: - continue - raise DiscoveryConflictError( - f"Source '{source_entry.name}' in {source_file.relative_path} references " - f"unknown loader '{source_entry.loader}'" - ) - - -def _validate_terminal_loader_config_ownership( - *, - source_files: tuple[DiscoveredSourceFile, ...], - loader_functions: tuple[DiscoveredLoaderFunction, ...], -) -> None: - loader_by_name: dict[str, DiscoveredLoaderFunction] = { - loader.name: loader for loader in loader_functions +def _node(*, kind: str, node: _PythonNode, owners: dict[object, int]) -> dict[str, object]: + return { + "kind": kind, + "name": node.name, + "path": str(node.relative_path), + "decorator_config": isinstance(node, DiscoveredLoaderFunction) + and _loader_has_decorator_config(node), + "depends_on": [ + (dependency.name, None, None) + if isinstance(dependency, SqlResourceRef) + else (None, owners.get(dependency), _decorated_name(dependency)) + for dependency in node.depends_on + ], } - source_file: DiscoveredSourceFile - for source_file in source_files: - source_entry: SourceEntry - for source_entry in source_file.source_entries: - if source_entry.loader is None: - continue - loader_function: DiscoveredLoaderFunction | None = loader_by_name.get( - source_entry.loader - ) - if loader_function is None or not _loader_has_decorator_config(loader_function): - continue - raise DiscoveryConflictError( - f"Source '{source_entry.name}' references loader '{source_entry.loader}', " - "but terminal source loader write and schema config must be declared in " - "source YAML, not the @loader decorator" - ) def _loader_has_decorator_config(loader_function: DiscoveredLoaderFunction) -> bool: @@ -285,574 +208,19 @@ def _loader_has_decorator_config(loader_function: DiscoveredLoaderFunction) -> b ) -def _validate_loader_dependencies(loader_functions: tuple[DiscoveredLoaderFunction, ...]) -> None: - loader_by_function: dict[object, DiscoveredLoaderFunction] = { - loader.function: loader for loader in loader_functions - } - visiting: set[str] = set() - visited: set[str] = set() - - def visit( - *, - loader_function: DiscoveredLoaderFunction, - path: tuple[str, ...], - visiting_names: set[str], - visited_names: set[str], - ) -> tuple[set[str], set[str]]: - if loader_function.name in visited_names: - return visiting_names, visited_names - if loader_function.name in visiting_names: - cycle_path: str = " -> ".join((*path, loader_function.name)) - raise DiscoveryConflictError(f"Loader dependency cycle detected: {cycle_path}") - visiting_names = visiting_names | {loader_function.name} - dependency: object - for dependency in loader_function.depends_on: - if dependency in loader_by_function: - visiting_names, visited_names = visit( - loader_function=loader_by_function[dependency], - path=(*path, loader_function.name), - visiting_names=visiting_names, - visited_names=visited_names, - ) - visiting_names = visiting_names - {loader_function.name} - visited_names = visited_names | {loader_function.name} - return visiting_names, visited_names - - for loader_function in loader_functions: - visiting, visited = visit( - loader_function=loader_function, - path=(), - visiting_names=visiting, - visited_names=visited, - ) - - -def _validate_python_node_dependencies( - *, - loader_functions: tuple[DiscoveredLoaderFunction, ...], - task_functions: tuple[DiscoveredTaskFunction, ...], - asset_functions: tuple[DiscoveredAssetFunction, ...], -) -> None: - nodes: tuple[ - DiscoveredLoaderFunction | DiscoveredTaskFunction | DiscoveredAssetFunction, ... - ] = ( - *loader_functions, - *task_functions, - *asset_functions, - ) - node_by_dependency_key: dict[ - object | tuple[str, str], - DiscoveredLoaderFunction | DiscoveredTaskFunction | DiscoveredAssetFunction, - ] = {node.function: node for node in nodes} - for node in nodes: - node_by_dependency_key[("name", node.name)] = node - node: DiscoveredLoaderFunction | DiscoveredTaskFunction | DiscoveredAssetFunction - for node in nodes: - dependency: object - for dependency in node.depends_on: - if isinstance(dependency, SqlResourceRef): - if isinstance(node, DiscoveredLoaderFunction): - raise DiscoveryConflictError( - f"Loader '{node.name}' depends on SQL resource " - f"'{dependency.name}'; loaders " - "may depend on Python loader/task/asset functions only" - ) - continue - if _python_node_dependency_key(dependency) not in node_by_dependency_key: - if isinstance(node, DiscoveredLoaderFunction): - raise DiscoveryConflictError( - f"Loader '{node.name}' depends on an unknown loader" - ) - raise DiscoveryConflictError( - f"Python node '{node.name}' depends on an unknown Python node; " - "use function references to decorated loaders, tasks, or assets" - ) - - visiting: set[str] = set() - visited: set[str] = set() - - def visit( - *, - current: DiscoveredLoaderFunction | DiscoveredTaskFunction | DiscoveredAssetFunction, - path: tuple[str, ...], - visiting_names: set[str], - visited_names: set[str], - ) -> tuple[set[str], set[str]]: - if current.name in visited_names: - return visiting_names, visited_names - if current.name in visiting_names: - cycle_path: str = " -> ".join((*path, current.name)) - raise DiscoveryConflictError(f"Python node dependency cycle detected: {cycle_path}") - visiting_names = visiting_names | {current.name} - dependency: object - for dependency in current.depends_on: - if isinstance(dependency, SqlResourceRef): - continue - visiting_names, visited_names = visit( - current=node_by_dependency_key[_python_node_dependency_key(dependency)], - path=(*path, current.name), - visiting_names=visiting_names, - visited_names=visited_names, - ) - visiting_names = visiting_names - {current.name} - visited_names = visited_names | {current.name} - return visiting_names, visited_names - - for node in nodes: - visiting, visited = visit( - current=node, - path=(), - visiting_names=visiting, - visited_names=visited, - ) - +def _decorated_name(dependency: object) -> str | None: + if not callable(dependency): + return None + for read_definition in _DEFINITION_READERS: + definition: _NodeDefinition | None = read_definition(dependency) + if definition is not None: + return definition.name + return None -def _validate_check_dependencies( - *, - loader_functions: tuple[DiscoveredLoaderFunction, ...], - task_functions: tuple[DiscoveredTaskFunction, ...], - asset_functions: tuple[DiscoveredAssetFunction, ...], - check_functions: tuple[DiscoveredCheckFunction, ...], -) -> None: - allowed_dependency_keys: frozenset[object | tuple[str, str]] = frozenset( - ("name", node.name) for node in (*loader_functions, *task_functions, *asset_functions) - ) - check_dependency_keys: frozenset[object | tuple[str, str]] = frozenset( - ("name", check_function.name) for check_function in check_functions - ) - check_function: DiscoveredCheckFunction - for check_function in check_functions: - dependency: object - for dependency in check_function.depends_on: - if isinstance(dependency, SqlResourceRef): - continue - dependency_key: object | tuple[str, str] = _python_node_dependency_key(dependency) - if dependency_key in check_dependency_keys: - raise DiscoveryConflictError( - f"Check '{check_function.name}' depends on another check; checks may " - "depend on loaders, tasks, assets, models, sources, and seeds only" - ) - if dependency_key not in allowed_dependency_keys: - raise DiscoveryConflictError( - f"Check '{check_function.name}' depends on an unknown Python node; " - "use function references to decorated loaders, tasks, or assets" - ) - -def _python_node_dependency_key(dependency: object) -> object | tuple[str, str]: - loader_definition: LoaderDefinition | None = ( - read_loader_definition(dependency) if callable(dependency) else None - ) - if loader_definition is not None: - return ("name", loader_definition.name) - task_definition: TaskDefinition | None = ( - read_task_definition(dependency) if callable(dependency) else None - ) - if task_definition is not None: - return ("name", task_definition.name) - asset_definition: AssetDefinition | None = ( - read_asset_definition(dependency) if callable(dependency) else None - ) - if asset_definition is not None: - return ("name", asset_definition.name) - check_definition: CheckDefinition | None = ( - read_check_definition(dependency) if callable(dependency) else None - ) - if check_definition is not None: - return ("name", check_definition.name) - return dependency - - -def _validate_unique_schema_model_names(schema_files: tuple[DiscoveredSchemaFile, ...]) -> None: - seen_names: dict[str, str] = {} - schema_file: DiscoveredSchemaFile - for schema_file in schema_files: - model_entry: SchemaModelEntry - for model_entry in schema_file.model_entries: - existing_path: str | None = seen_names.get(model_entry.name) - if existing_path is not None: - raise DiscoveryConflictError( - "Duplicate schema.yml model declaration found for " - f"'{model_entry.name}' in {existing_path} and {schema_file.relative_path}" - ) - seen_names[model_entry.name] = str(schema_file.relative_path) - - -def _validate_unique_schema_seed_names(schema_files: tuple[DiscoveredSchemaFile, ...]) -> None: - seen_names: dict[str, str] = {} - schema_file: DiscoveredSchemaFile - for schema_file in schema_files: - seed_entry: SchemaSeedEntry - for seed_entry in schema_file.seed_entries: - existing_path: str | None = seen_names.get(seed_entry.name) - if existing_path is not None: - raise DiscoveryConflictError( - "Duplicate seed declaration found for " - f"'{seed_entry.name}' in {existing_path} and {schema_file.relative_path}" - ) - seen_names[seed_entry.name] = str(schema_file.relative_path) - - -def _validate_unique_logical_relation_names( - *, - model_files: tuple[DiscoveredSqlModelFile, ...], - source_files: tuple[DiscoveredSourceFile, ...], - schema_files: tuple[DiscoveredSchemaFile, ...], -) -> None: - seen_names: dict[str, tuple[str, str]] = {} - model_file: DiscoveredSqlModelFile - for model_file in model_files: - seen_names[model_file.file_path.stem] = ("model", str(model_file.relative_path)) - - source_file: DiscoveredSourceFile - for source_file in source_files: - source_entry: SourceEntry - for source_entry in source_file.source_entries: - seen_names[source_entry.name] = _validated_logical_relation_entry( - seen_names=seen_names, - name=source_entry.name, - kind="source", - path=str(source_file.relative_path), - ) - - schema_file: DiscoveredSchemaFile - for schema_file in schema_files: - seed_entry: SchemaSeedEntry - for seed_entry in schema_file.seed_entries: - seen_names[seed_entry.name] = _validated_logical_relation_entry( - seen_names=seen_names, - name=seed_entry.name, - kind="seed", - path=str(schema_file.relative_path), - ) +def _named_paths(resources: Iterable[_NamedResource]) -> list[_Named]: + return [(resource.name, str(resource.relative_path)) for resource in resources] -def _validated_logical_relation_entry( - *, seen_names: dict[str, tuple[str, str]], name: str, kind: str, path: str -) -> tuple[str, str]: - existing_entry: tuple[str, str] | None = seen_names.get(name) - if existing_entry is not None: - raise DiscoveryConflictError( - f"Logical relation name '{name}' is declared as both {existing_entry[0]} " - f"in {existing_entry[1]} and {kind} in {path}" - ) - return (kind, path) - - -def _validate_unique_project_resource_names( - discovered_inputs: DiscoveredProjectInputs, -) -> None: - seen_names: dict[str, tuple[str, str]] = {} - - model_file: DiscoveredSqlModelFile - for model_file in discovered_inputs.model_files: - seen_names[model_file.file_path.stem] = _validated_project_resource_entry( - seen_names=seen_names, - name=model_file.file_path.stem, - kind="model", - path=str(model_file.relative_path), - ) - - source_file: DiscoveredSourceFile - managed_source_names: set[str] = set() - for source_file in discovered_inputs.source_files: - source_entry: SourceEntry - for source_entry in source_file.source_entries: - if source_entry.managed: - managed_source_names.add(source_entry.name) - seen_names[source_entry.name] = _validated_project_resource_entry( - seen_names=seen_names, - name=source_entry.name, - kind="source", - path=str(source_file.relative_path), - ) - - schema_file: DiscoveredSchemaFile - for schema_file in discovered_inputs.schema_files: - seed_entry: SchemaSeedEntry - for seed_entry in schema_file.seed_entries: - seen_names[seed_entry.name] = _validated_project_resource_entry( - seen_names=seen_names, - name=seed_entry.name, - kind="seed", - path=str(schema_file.relative_path), - ) - - sql_function_file: DiscoveredSqlFunctionFile - for sql_function_file in discovered_inputs.sql_function_files: - seen_names[sql_function_file.file_path.stem] = _validated_project_resource_entry( - seen_names=seen_names, - name=sql_function_file.file_path.stem, - kind="function", - path=str(sql_function_file.relative_path), - ) - - python_function_file: DiscoveredPythonFunctionFile - for python_function_file in discovered_inputs.python_function_files: - seen_names[python_function_file.file_path.stem] = _validated_project_resource_entry( - seen_names=seen_names, - name=python_function_file.file_path.stem, - kind="function", - path=str(python_function_file.relative_path), - ) - - node: ( - DiscoveredLoaderFunction - | DiscoveredTaskFunction - | DiscoveredAssetFunction - | DiscoveredCheckFunction - ) - for node in ( - *discovered_inputs.loader_functions, - *discovered_inputs.task_functions, - *discovered_inputs.asset_functions, - *discovered_inputs.check_functions, - ): - if isinstance(node, DiscoveredLoaderFunction) and node.name in managed_source_names: - continue - seen_names[node.name] = _validated_project_resource_entry( - seen_names=seen_names, - name=node.name, - kind=node.__class__.__name__.removeprefix("Discovered") - .removesuffix("Function") - .lower(), - path=str(node.relative_path), - ) - - provider: DiscoveredProvider - for provider in discovered_inputs.providers: - seen_names[provider.name] = _validated_project_resource_entry( - seen_names=seen_names, - name=provider.name, - kind="provider", - path=str(provider.relative_path), - ) - - sql_hook_file: DiscoveredSqlHookFile - for sql_hook_file in discovered_inputs.sql_hook_files: - seen_names[sql_hook_file.name] = _validated_project_resource_entry( - seen_names=seen_names, - name=sql_hook_file.name, - kind="sql hook", - path=str(sql_hook_file.relative_path), - ) - - hook_function: DiscoveredHookFunction - for hook_function in discovered_inputs.hook_functions: - seen_names[hook_function.name] = _validated_project_resource_entry( - seen_names=seen_names, - name=hook_function.name, - kind="python hook", - path=str(hook_function.relative_path), - ) - - scenario_file: DiscoveredSqlScenarioFile - for scenario_file in discovered_inputs.scenario_files: - seen_names[scenario_file.name] = _validated_project_resource_entry( - seen_names=seen_names, - name=scenario_file.name, - kind="scenario", - path=str(scenario_file.relative_path), - ) - - test_names: set[str] = set() - test_file: DiscoveredSqlTestFile - for test_file in discovered_inputs.test_files: - block: DiscoveredSqlTestBlock - for block in test_file.blocks: - test_name: str = block.name or test_file.relative_path.stem - if test_name in test_names: - continue - test_names.add(test_name) - seen_names[test_name] = _validated_project_resource_entry( - seen_names=seen_names, - name=test_name, - kind="unit test", - path=str(test_file.relative_path), - ) - - -def _validated_project_resource_entry( - *, seen_names: dict[str, tuple[str, str]], name: str, kind: str, path: str -) -> tuple[str, str]: - existing_entry: tuple[str, str] | None = seen_names.get(name) - if existing_entry is not None: - raise DiscoveryConflictError( - f"Project resource name '{name}' is declared as both {existing_entry[0]} " - f"in {existing_entry[1]} and {kind} in {path}; model, source, seed, function, " - "loader, task, asset, check, provider, hook, unit test, and scenario names must be " - "globally unique" - ) - return (kind, path) - - -def _validate_declared_seed_files( - *, - schema_files: tuple[DiscoveredSchemaFile, ...], - seed_files: tuple[DiscoveredSeedFile, ...], -) -> None: - _validate_unique_seed_csv_names(seed_files) - declared_seed_entries: list[tuple[SchemaSeedEntry, str]] = [] - schema_file: DiscoveredSchemaFile - for schema_file in schema_files: - seed_entry: SchemaSeedEntry - for seed_entry in schema_file.seed_entries: - declared_seed_entries.append((seed_entry, str(schema_file.relative_path))) - seed_files_by_name: dict[str, list[DiscoveredSeedFile]] = {} - for seed_file in seed_files: - seed_files_by_name.setdefault(seed_file.file_path.stem, []).append(seed_file) - - seed_entry_and_path: tuple[SchemaSeedEntry, str] - for seed_entry_and_path in declared_seed_entries: - seed_entry: SchemaSeedEntry = seed_entry_and_path[0] - declaration_path: str = seed_entry_and_path[1] - matching_seed_files: tuple[DiscoveredSeedFile, ...] = tuple( - seed_files_by_name.get(seed_entry.name, ()) - ) - if not matching_seed_files: - raise SeedDiscoveryError( - "Seed declaration " - f"'{seed_entry.name}' in {declaration_path} has no matching CSV file under seeds/" - ) - if len(matching_seed_files) > 1: - matching_paths: str = ", ".join( - str(seed_file.relative_path) for seed_file in matching_seed_files - ) - raise SeedDiscoveryError( - f"Seed declaration '{seed_entry.name}' matches multiple CSV files: {matching_paths}" - ) - - _validate_seed_csv_header(seed_entry=seed_entry, seed_file=matching_seed_files[0]) - - declared_names: set[str] = {seed_entry.name for seed_entry, _ in declared_seed_entries} - seed_file: DiscoveredSeedFile - for seed_file in seed_files: - if seed_file.file_path.stem not in declared_names: - raise SeedDiscoveryError( - f"Seed CSV {seed_file.relative_path} has no matching declaration for seed " - f"'{seed_file.file_path.stem}' under seeds/**/*.yml" - ) - - -def _validate_unique_seed_csv_names(seed_files: tuple[DiscoveredSeedFile, ...]) -> None: - seen_paths: dict[str, Path] = {} - seed_file: DiscoveredSeedFile - for seed_file in seed_files: - seed_name: str = seed_file.file_path.stem - existing_path: Path | None = seen_paths.get(seed_name) - if existing_path is not None: - raise SeedDiscoveryError( - f"Duplicate seed CSV name '{seed_name}' found: {existing_path}, " - f"{seed_file.relative_path}. Seed CSV filenames must be unique under seeds/." - ) - seen_paths[seed_name] = seed_file.relative_path - - -def _validate_seed_csv_header( - *, seed_entry: SchemaSeedEntry, seed_file: DiscoveredSeedFile -) -> None: - header_columns: tuple[str, ...] = _load_seed_csv_header( - file_path=seed_file.file_path, - delimiter=seed_entry.csv_settings.delimiter, - quotechar=seed_entry.csv_settings.quotechar, - escapechar=seed_entry.csv_settings.escapechar, - doublequote=seed_entry.csv_settings.doublequote, - skipinitialspace=seed_entry.csv_settings.skipinitialspace, - ) - if not header_columns: - raise SeedDiscoveryError(f"{seed_file.relative_path} must contain a CSV header row") - - seen_columns: set[str] = set() - column_name: str - for column_name in header_columns: - if column_name in seen_columns: - raise SeedDiscoveryError( - f"{seed_file.relative_path} contains duplicate CSV header column '{column_name}'" - ) - seen_columns.add(column_name) - - declared_columns: tuple[str, ...] = tuple(column.name for column in seed_entry.columns) - if header_columns != declared_columns: - raise SeedDiscoveryError( - f"{seed_file.relative_path} header {header_columns} does not match " - "declared seed columns " - f"{declared_columns} for '{seed_entry.name}'" - ) - - -def _load_seed_csv_header( - *, - file_path: Path, - delimiter: str | None, - quotechar: str | None, - escapechar: str | None, - doublequote: bool | None, - skipinitialspace: bool | None, -) -> tuple[str, ...]: - reader_kwargs: dict[str, object] = {} - if delimiter is not None: - reader_kwargs["delimiter"] = delimiter - if quotechar is not None: - reader_kwargs["quotechar"] = quotechar - if escapechar is not None: - reader_kwargs["escapechar"] = escapechar - if doublequote is not None: - reader_kwargs["doublequote"] = doublequote - if skipinitialspace is not None: - reader_kwargs["skipinitialspace"] = skipinitialspace - - with file_path.open("r", encoding="utf-8", newline="") as handle: - try: - header_row: list[str] = next( - csv.reader( - handle, - delimiter=delimiter or ",", - quotechar=quotechar, - escapechar=escapechar, - doublequote=True if doublequote is None else doublequote, - skipinitialspace=False if skipinitialspace is None else skipinitialspace, - ) - ) - except StopIteration: - return () - return tuple(header_row) - - -def _validate_path_defaults_match_models( - *, - path_defaults: dict[str, dict[str, object]], - model_files: tuple[DiscoveredSqlModelFile, ...], -) -> None: - if not path_defaults: - return - model_paths: tuple[str, ...] = tuple( - str(model_file.relative_path).replace("\\", "/").removeprefix("models/") - for model_file in model_files - ) - matched_keys: set[str] = set() - model_path: str - for model_path in model_paths: - matched_keys.update( - select_path_default( - model_path=model_path, - path_keys=tuple(path_defaults), - ).matched_keys - ) - path_key: str - for path_key in path_defaults: - if path_key in matched_keys: - continue - model_folders: tuple[str, ...] = tuple( - sorted( - { - str(Path(model_path).parent) - for model_path in model_paths - if str(Path(model_path).parent) != CURRENT_DIRECTORY_PATH - } - ) - ) - known_folders: str = ", ".join(model_folders[:5]) if model_folders else "" - raise DiscoveryConflictError( - f"path_defaults['{path_key}'] does not match any model paths. Known model folders " - f"include: {known_folders}" - ) +def _stems(files: Iterable[_ResourceFile]) -> list[_Named]: + return [(file.file_path.stem, str(file.relative_path)) for file in files] diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/identities.py b/src/sqlbuild/compiler/discovery/_helpers/validation/identities.py deleted file mode 100644 index da6364381e..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/identities.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Canonical authored resource identity validation.""" - -from __future__ import annotations - -from collections.abc import Iterator -from dataclasses import dataclass -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.resource_names.main._validate_resource_identity import ( - validate_resource_identity, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind - - -@dataclass(frozen=True) -class _IdentityCandidate: - name: str - kind: str - path: Path - private_identity: bool = False - - -def validate_discovered_resource_identities( - discovered_inputs: DiscoveredProjectInputs, -) -> None: - """Reject non-canonical identities before graph assembly or persistence.""" - - for candidate in _identity_candidates(discovered_inputs): - validate_resource_identity( - name=candidate.name, - kind=candidate.kind, - path=candidate.path, - private_identity=candidate.private_identity, - ) - - -def _identity_candidates( - discovered_inputs: DiscoveredProjectInputs, -) -> Iterator[_IdentityCandidate]: - for model_file in discovered_inputs.model_files: - yield _IdentityCandidate(model_file.file_path.stem, "model", model_file.relative_path) - for declaration in (*model_file.enum_declarations, *model_file.constant_declarations): - yield _IdentityCandidate( - declaration.name, - "model-local declaration", - declaration.relative_path, - private_identity=True, - ) - for function_file in ( - *discovered_inputs.sql_function_files, - *discovered_inputs.python_function_files, - ): - yield _IdentityCandidate( - function_file.file_path.stem, "function", function_file.relative_path - ) - for hook_file in discovered_inputs.sql_hook_files: - yield _IdentityCandidate(hook_file.name, "SQL hook", hook_file.relative_path) - for seed_file in discovered_inputs.seed_files: - yield _IdentityCandidate(seed_file.file_path.stem, "seed", seed_file.relative_path) - for schema_file in discovered_inputs.schema_files: - for model_entry in schema_file.model_entries: - yield _IdentityCandidate(model_entry.name, "schema model", schema_file.relative_path) - for seed_entry in schema_file.seed_entries: - yield _IdentityCandidate(seed_entry.name, "schema seed", schema_file.relative_path) - for source_file in discovered_inputs.source_files: - for source_entry in source_file.source_entries: - yield _IdentityCandidate(source_entry.name, "source", source_file.relative_path) - for test_file in discovered_inputs.test_files: - for block in test_file.blocks: - yield _IdentityCandidate( - block.name or test_file.relative_path.stem, - "SQL test", - test_file.relative_path, - ) - for case in block.cases: - yield _IdentityCandidate(case.name, "SQL test case", test_file.relative_path) - for scenario_file in discovered_inputs.scenario_files: - yield _IdentityCandidate(scenario_file.name, "scenario", scenario_file.relative_path) - for audit_file in discovered_inputs.audit_files: - for block in audit_file.blocks: - is_generic: bool = audit_file.declaration_kind is DeclarationKind.AUDIT - yield _IdentityCandidate( - ( - audit_file.relative_path.stem - if is_generic - else block.name or audit_file.relative_path.stem - ), - "audit", - audit_file.relative_path, - ) - if is_generic and block.name is not None: - yield _IdentityCandidate( - block.name, - "generic audit instance", - audit_file.relative_path, - ) - for declaration_file in ( - *discovered_inputs.enum_files, - *discovered_inputs.constant_files, - ): - for declaration in declaration_file.declarations: - yield _IdentityCandidate( - declaration.name, - "declaration", - declaration.relative_path, - private_identity=declaration_file.scope_kind is ScopeKind.PRIVATE, - ) - for schema_file in discovered_inputs.model_schema_files: - for declaration in schema_file.declarations: - yield _IdentityCandidate(declaration.name, "model schema", declaration.relative_path) - for named_resources, kind in ( - (discovered_inputs.materialization_files, "materialization"), - (discovered_inputs.loader_functions, "loader"), - (discovered_inputs.task_functions, "task"), - (discovered_inputs.asset_functions, "asset"), - (discovered_inputs.check_functions, "check"), - (discovered_inputs.hook_functions, "Python hook"), - (discovered_inputs.event_exporters, "event exporter"), - (discovered_inputs.providers, "provider"), - ): - for resource in named_resources: - yield _IdentityCandidate(resource.name, kind, resource.relative_path) diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 012a87f6cf..dbba6d76bf 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -33,7 +33,6 @@ SchemaSeedEntry, SourceColumnEntry, SourceEntry, - SourceLocation, ) from sqlbuild.spec.contracts.types import EventExportSeverity, SourceWriteStrategy from sqlbuild.sql_values.models import SqlLogicalType, SqlValue diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 5a466cf6f2..6192436408 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -224,3 +224,13 @@ class DeepHeaderValuesTestCase: header: str stack_frames: int expected_max_depth: int + + +@dataclass(frozen=True) +class ProjectValidationTestCase: + """A project whose files conflict across discovery, with the error discovery raises.""" + + description: str + files: tuple[tuple[str, bytes], ...] + expected_error_type: str + expected_message: str diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_validation.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_validation.py new file mode 100644 index 0000000000..128d7d8612 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_project_validation.py @@ -0,0 +1,109 @@ +"""Native cross-file validation raises the discovery errors Python raised, with their text.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( + ProjectValidationTestCase, +) +from tests.integration.src.sqlbuild.compiler.discovery.helpers import ( + stage_outcome, + write_project, +) + +_MODEL: tuple[str, bytes] = ("models/orders.sql", b"MODEL ();\nSELECT 1 AS order_id") + + +def _seed_files(*, header: bytes, settings: bytes = b"") -> tuple[tuple[str, bytes], ...]: + return ( + _MODEL, + ("seeds/countries.csv", header + b"\nGB,United Kingdom\n"), + ( + "seeds/countries.yml", + b"seeds:\n - name: countries\n" + + settings + + b" columns:\n - name: code\n type: VARCHAR\n" + b" - name: label\n type: VARCHAR\n", + ), + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectValidationTestCase( + description="a seed header that differs from its declared columns", + files=_seed_files(header=b"code,name"), + expected_error_type="SeedDiscoveryError", + expected_message=( + "seeds/countries.csv header ('code', 'name') does not match declared seed " + "columns ('code', 'label') for 'countries'" + ), + ), + ProjectValidationTestCase( + description="a quoted seed header honours the declared quotechar", + files=_seed_files( + header=b'"code";"label";"extra"', + settings=b" csv_settings:\n delimiter: ';'\n quotechar: '\"'\n", + ), + expected_error_type="SeedDiscoveryError", + expected_message=( + "seeds/countries.csv header ('code', 'label', 'extra') does not match declared " + "seed columns ('code', 'label') for 'countries'" + ), + ), + ProjectValidationTestCase( + description="a seed CSV without a declaration", + files=(_MODEL, ("seeds/regions.csv", b"code\nEU\n")), + expected_error_type="SeedDiscoveryError", + expected_message=( + "Seed CSV seeds/regions.csv has no matching declaration for seed 'regions' under " + "seeds/**/*.yml" + ), + ), + ProjectValidationTestCase( + description="two model files with one name", + files=(_MODEL, ("models/marts/orders.sql", b"MODEL ();\nSELECT 2 AS order_id")), + expected_error_type="DiscoveryConflictError", + expected_message=( + "Duplicate model file name found for 'orders' in models/marts/orders.sql and " + "models/orders.sql" + ), + ), + ProjectValidationTestCase( + description="a non-canonical model identity", + files=(("models/Orders.sql", b"MODEL ();\nSELECT 1 AS order_id"),), + expected_error_type="ResourceIdentityError", + expected_message=( + "Invalid model identity 'Orders' in models/Orders.sql; use snake_case 'orders'" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_conflicting_project_files_when_discovering_then_raises_the_discovery_error( + test_case: ProjectValidationTestCase, tmp_path: Path +) -> None: + write_project(project_dir=tmp_path, files=test_case.files) + + outcome: tuple[object, ...] = stage_outcome(project_dir=tmp_path) + + assert outcome[1:] == (test_case.expected_error_type, test_case.expected_message) + + +def test_given_an_empty_seed_quotechar_when_discovering_then_raises_the_csv_type_error( + tmp_path: Path, +) -> None: + write_project( + project_dir=tmp_path, + files=_seed_files( + header=b"code,label", settings=b" csv_settings:\n quotechar: ''\n" + ), + ) + + with pytest.raises(TypeError, match='"quotechar" must be a 1-character string'): + discover_project_inputs(project_dir=tmp_path) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 27a33f0e20..8b11448b1c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -228,22 +228,6 @@ class DeferredModelOutputLocationTestCase: expected_extract_implicit_alias_columns: bool -@dataclass(frozen=True) -class ValidatePathDefaultsMatchModelsTestCase: - description: str - model_relative_paths: tuple[str, ...] - path_defaults: dict[str, dict[str, object]] - expected_model_file_count: int - - -@dataclass(frozen=True) -class ValidatePathDefaultsMatchModelsErrorTestCase: - description: str - model_relative_paths: tuple[str, ...] - path_defaults: dict[str, dict[str, object]] - expected_error_fragment: str - - @dataclass(frozen=True) class LoadProjectConfigTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_path_defaults.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_path_defaults.py deleted file mode 100644 index d94d9ba7a0..0000000000 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_path_defaults.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.discovery._helpers.validation.discovery import ( - _validate_path_defaults_match_models, -) -from sqlbuild.compiler.discovery.exceptions import DiscoveryConflictError -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( - ValidatePathDefaultsMatchModelsErrorTestCase, - ValidatePathDefaultsMatchModelsTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - ValidatePathDefaultsMatchModelsTestCase( - description="accepts Windows-style model paths that match path defaults", - model_relative_paths=("models\\staging\\orders.sql",), - path_defaults={"staging": {}}, - expected_model_file_count=1, - ), - ValidatePathDefaultsMatchModelsTestCase( - description="accepts recursive glob matching nested staging prefix", - model_relative_paths=("models/commerce/partner_feed/staging/orders.sql",), - path_defaults={"commerce/**/staging": {}}, - expected_model_file_count=1, - ), - ValidatePathDefaultsMatchModelsTestCase( - description="accepts recursive glob matching zero segments", - model_relative_paths=("models/commerce/staging/orders.sql",), - path_defaults={"commerce/**/staging": {}}, - expected_model_file_count=1, - ), - ValidatePathDefaultsMatchModelsTestCase( - description="counts broad glob as matched when a literal wins selection", - model_relative_paths=("models/commerce/partner_feed/staging/orders.sql",), - path_defaults={ - "commerce/**/staging": {}, - "commerce/partner_feed/staging": {}, - }, - expected_model_file_count=1, - ), - ), - ids=lambda case: case.description, -) -def test_given_windows_style_model_paths_when_validating_path_defaults_then_they_match( - test_case: ValidatePathDefaultsMatchModelsTestCase, -) -> None: - model_files: tuple[DiscoveredSqlModelFile, ...] = tuple( - DiscoveredSqlModelFile( - file_path=Path("/repo") / relative_path, - relative_path=Path(relative_path), - contents="MODEL ();\n\nselect 1\n", - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="select 1", - ) - for relative_path in test_case.model_relative_paths - ) - - _validate_path_defaults_match_models( - path_defaults=test_case.path_defaults, - model_files=model_files, - ) - - assert len(model_files) == test_case.expected_model_file_count - - -@pytest.mark.parametrize( - "test_case", - ( - ValidatePathDefaultsMatchModelsErrorTestCase( - description="rejects recursive glob matching no model paths", - model_relative_paths=("models/commerce/partner_feed/marts/orders.sql",), - path_defaults={"commerce/**/staging": {}}, - expected_error_fragment=( - r"path_defaults\['commerce/\*\*/staging'\] does not match any model paths" - ), - ), - ValidatePathDefaultsMatchModelsErrorTestCase( - description="rejects equally specific matching globs during discovery", - model_relative_paths=("models/commerce/eu/staging/orders.sql",), - path_defaults={"commerce/*/staging": {}, "commerce/eu/*": {}}, - expected_error_fragment="matches equally specific path_defaults keys", - ), - ), - ids=lambda case: case.description, -) -def test_given_invalid_glob_matches_when_validating_path_defaults_then_raises_conflict( - test_case: ValidatePathDefaultsMatchModelsErrorTestCase, -) -> None: - model_files: tuple[DiscoveredSqlModelFile, ...] = tuple( - DiscoveredSqlModelFile( - file_path=Path("/repo") / relative_path, - relative_path=Path(relative_path), - contents="MODEL ();\n\nselect 1\n", - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql="select 1", - ) - for relative_path in test_case.model_relative_paths - ) - - with pytest.raises(DiscoveryConflictError, match=test_case.expected_error_fragment): - _validate_path_defaults_match_models( - path_defaults=test_case.path_defaults, - model_files=model_files, - ) From 625226e3d99d34b3335f263cbdb1d034e4d0e7cd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:58:06 +0100 Subject: [PATCH 5/7] refactor: parse source and seed YAML declarations natively --- crates/sqlbuild-discovery/src/lib.rs | 1 + .../src/yaml_entries/_helpers/fields.rs | 139 +++ .../src/yaml_entries/_helpers/freshness.rs | 431 +++++++++ .../src/yaml_entries/_helpers/integrations.rs | 402 +++++++++ .../src/yaml_entries/_helpers/mod.rs | 3 + .../src/yaml_entries/main/mod.rs | 2 + .../main/parse_schema_document.rs | 306 +++++++ .../main/parse_sources_document.rs | 311 +++++++ .../src/yaml_entries/mod.rs | 6 + .../src/yaml_entries/models.rs | 180 ++++ .../src/yaml_entries/tests.rs | 6 + .../tests/test_parse_schema_document.rs | 107 +++ .../tests/test_parse_sources_document.rs | 122 +++ .../src/yaml_entries/tests/test_types.rs | 102 +++ .../src/header_metadata/main/mod.rs | 1 + .../header_metadata/main/parse_audit_list.rs | 28 + .../src/bindings/_helpers/discovery/mod.rs | 1 + .../_helpers/discovery/registration.rs | 3 +- .../_helpers/discovery/yaml_entries.rs | 358 ++++++++ python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 6 + .../_helpers/integrations/loaders.py | 432 ---------- .../compiler/discovery/_helpers/yml/schema.py | 423 +-------- .../discovery/_helpers/yml/sources.py | 815 +----------------- .../discovery/_helpers/yml/yaml_classes.py | 45 + src/sqlbuild/compiler/discovery/constants.py | 11 - 26 files changed, 2572 insertions(+), 1671 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/_helpers/fields.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/_helpers/freshness.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/_helpers/integrations.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/main/parse_schema_document.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/main/parse_sources_document.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/mod.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/models.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/tests.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_schema_document.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/tests/test_parse_sources_document.rs create mode 100644 crates/sqlbuild-discovery/src/yaml_entries/tests/test_types.rs create mode 100644 crates/sqlbuild-model-config/src/header_metadata/main/parse_audit_list.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs create mode 100644 src/sqlbuild/compiler/discovery/_helpers/yml/yaml_classes.py diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index 940181a53c..e68026dc37 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -12,4 +12,5 @@ pub mod models; pub mod project_validation; pub mod sql_tests; pub mod tree; +pub mod yaml_entries; pub mod yaml_files; diff --git a/crates/sqlbuild-discovery/src/yaml_entries/_helpers/fields.rs b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/fields.rs new file mode 100644 index 0000000000..df655ad1e8 --- /dev/null +++ b/crates/sqlbuild-discovery/src/yaml_entries/_helpers/fields.rs @@ -0,0 +1,139 @@ +//! Python's authored-value field readers, with the errors their YAML callers raise. + +use sqlbuild_model_config::errors::ConfigError; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; + +use crate::key_checks::main::unknown_mapping_keys::unknown_mapping_keys; + +/// Where one YAML parse reports its errors: the file path and the label the messages name. +#[derive(Clone, Copy)] +pub(crate) struct Site<'a> { + pub(crate) path: &'a str, + pub(crate) label: &'a str, +} + +impl Site<'_> { + /// The error `