diff --git a/crates/sqlbuild-core/src/json/_helpers/emitter.rs b/crates/sqlbuild-core/src/json/_helpers/emitter.rs index 3a275b282b..1b2f1dd35b 100644 --- a/crates/sqlbuild-core/src/json/_helpers/emitter.rs +++ b/crates/sqlbuild-core/src/json/_helpers/emitter.rs @@ -1,7 +1,7 @@ //! Serialize a [`JsonValue`] in the layout of one Python serializer. use crate::json::_helpers::floats::{orjson_text, python_repr}; -use crate::json::_helpers::strings::{StringEscaping, json_string, json_utf16_string}; +use crate::json::_helpers::strings::{StringEscaping, with_json_string, with_json_utf16_string}; use crate::json::constants::{ ORJSON_MAX_CONTAINER_DEPTH, ORJSON_MAX_INTEGER, ORJSON_MIN_INTEGER, PYTHON_INT_MAX_STR_DIGITS, STDLIB_MAX_CONTAINER_DEPTH, @@ -48,9 +48,14 @@ impl<'options> Layout<'options> { } } - fn newline(&self, level: usize) -> String { - self.indent - .map_or_else(String::new, |indent| format!("\n{}", indent.repeat(level))) + fn with_newline(&self, mut output: String, level: usize) -> String { + if let Some(indent) = self.indent { + output.push('\n'); + for _ in 0..level { + output.push_str(indent); + } + } + output } fn integer(&self, value: &JsonInteger) -> Result { @@ -94,67 +99,85 @@ impl<'options> Layout<'options> { counted && level >= limit } - fn value(&self, value: &JsonValue, level: usize) -> Result { + fn with_value( + &self, + mut output: String, + value: &JsonValue, + level: usize, + ) -> Result { if self.too_deep(value, level) { return Err(JsonEmitError::NestingTooDeep); } match value { - JsonValue::Null => Ok("null".to_owned()), - JsonValue::Bool(flag) => Ok(flag.to_string()), - JsonValue::Integer(number) => self.integer(number), - JsonValue::Float(number) => self.float(*number), - JsonValue::String(text) => Ok(json_string(text, self.escaping)), - JsonValue::Utf16(units) => json_utf16_string(units, self.escaping), - JsonValue::Array(items) => { - let members = items - .iter() - .map(|item| self.value(item, level + 1)) - .collect::, _>>()?; - Ok(self.container(('[', ']'), members, level)) + JsonValue::Null => output.push_str("null"), + JsonValue::Bool(flag) => output.push_str(if *flag { "true" } else { "false" }), + JsonValue::Integer(number) => output.push_str(&self.integer(number)?), + JsonValue::Float(number) => output.push_str(&self.float(*number)?), + JsonValue::String(text) => return Ok(with_json_string(output, text, self.escaping)), + JsonValue::Utf16(units) => { + return with_json_utf16_string(output, units, self.escaping); } - JsonValue::Object(entries) => self.object(entries, level), + JsonValue::Array(items) => return self.with_array(output, items, level), + JsonValue::Object(entries) => return self.with_object(output, entries, level), + } + Ok(output) + } + + /// `output` followed by the text before the member at `index` of a container at `level`. + fn with_separator(&self, mut output: String, index: usize, level: usize) -> String { + if index > 0 { + output.push_str(self.item_separator); + } + self.with_newline(output, level + 1) + } + + fn with_array( + &self, + mut output: String, + items: &[JsonValue], + level: usize, + ) -> Result { + if items.is_empty() { + output.push_str("[]"); + return Ok(output); } + output.push('['); + for (index, item) in items.iter().enumerate() { + output = self.with_separator(output, index, level); + output = self.with_value(output, item, level + 1)?; + } + output = self.with_newline(output, level); + output.push(']'); + Ok(output) } - fn object( + fn with_object( &self, + mut output: String, entries: &[(String, JsonValue)], level: usize, ) -> Result { + if entries.is_empty() { + output.push_str("{}"); + return Ok(output); + } let mut ordered: Vec<&(String, JsonValue)> = entries.iter().collect(); if self.sort_keys { ordered.sort_by(|left, right| left.0.cmp(&right.0)); } - let members = ordered - .into_iter() - .map(|(key, item)| { - Ok(format!( - "{}{}{}", - json_string(key, self.escaping), - self.key_separator, - self.value(item, level + 1)? - )) - }) - .collect::, _>>()?; - Ok(self.container(('{', '}'), members, level)) - } - - fn container(&self, brackets: (char, char), members: Vec, level: usize) -> String { - if members.is_empty() { - return format!("{}{}", brackets.0, brackets.1); + output.push('{'); + for (index, (key, item)) in ordered.into_iter().enumerate() { + output = self.with_separator(output, index, level); + output = with_json_string(output, key, self.escaping); + output.push_str(self.key_separator); + output = self.with_value(output, item, level + 1)?; } - let opening = self.newline(level + 1); - let separator = format!("{}{opening}", self.item_separator); - let closing = self.newline(level); - format!( - "{}{opening}{}{closing}{}", - brackets.0, - members.join(&separator), - brackets.1 - ) + output = self.with_newline(output, level); + output.push('}'); + Ok(output) } } pub(crate) fn emit(value: &JsonValue, dialect: &JsonDialect) -> Result { - Layout::new(dialect).value(value, 0) + Layout::new(dialect).with_value(String::new(), value, 0) } diff --git a/crates/sqlbuild-core/src/json/_helpers/strings.rs b/crates/sqlbuild-core/src/json/_helpers/strings.rs index 9152d34258..76c45d8a18 100644 --- a/crates/sqlbuild-core/src/json/_helpers/strings.rs +++ b/crates/sqlbuild-core/src/json/_helpers/strings.rs @@ -1,5 +1,7 @@ //! JSON string escaping of `json.dumps` and orjson. +use std::fmt::Write; + use crate::json::constants::FIRST_PRINTABLE_ASCII; use crate::json::errors::JsonEmitError; @@ -12,67 +14,78 @@ pub(crate) enum StringEscaping { Unicode, } -fn unicode_escape(unit: u32) -> String { - format!("\\u{unit:04x}") +fn with_unicode_escape(mut output: String, unit: u32) -> String { + let _ = write!(output, "\\u{unit:04x}"); + output } -fn escaped(character: char, escaping: StringEscaping) -> String { +fn with_escaped(mut output: String, character: char, escaping: StringEscaping) -> String { match character { - '"' => "\\\"".to_owned(), - '\\' => "\\\\".to_owned(), - '\n' => "\\n".to_owned(), - '\r' => "\\r".to_owned(), - '\t' => "\\t".to_owned(), - '\u{8}' => "\\b".to_owned(), - '\u{c}' => "\\f".to_owned(), - ' '..='~' => character.to_string(), - _ if character < FIRST_PRINTABLE_ASCII => unicode_escape(u32::from(character)), - _ if escaping == StringEscaping::Unicode => character.to_string(), - _ => character - .encode_utf16(&mut [0_u16; 2]) - .iter() - .map(|unit| unicode_escape(u32::from(*unit))) - .collect(), + '"' => output.push_str("\\\""), + '\\' => output.push_str("\\\\"), + '\n' => output.push_str("\\n"), + '\r' => output.push_str("\\r"), + '\t' => output.push_str("\\t"), + '\u{8}' => output.push_str("\\b"), + '\u{c}' => output.push_str("\\f"), + ' '..='~' => output.push(character), + _ if character < FIRST_PRINTABLE_ASCII => { + return with_unicode_escape(output, u32::from(character)); + } + _ if escaping == StringEscaping::Unicode => output.push(character), + _ => { + for unit in character.encode_utf16(&mut [0_u16; 2]).iter() { + output = with_unicode_escape(output, u32::from(*unit)); + } + } } + output +} + +/// Whether `byte` is printable ASCII that a JSON string carries unescaped. +fn plain(byte: u8) -> bool { + (b' '..=b'~').contains(&byte) && byte != b'"' && byte != b'\\' } -/// `text` as a quoted JSON string. -pub(crate) fn json_string(text: &str, escaping: StringEscaping) -> String { - let mut output = String::with_capacity(text.len() + 2); +/// `output` followed by `text` as a quoted JSON string. +pub(crate) fn with_json_string(mut output: String, text: &str, escaping: StringEscaping) -> String { + output.reserve(text.len() + 2); output.push('"'); - for character in text.chars() { - match character { - ' '..='~' if character != '"' && character != '\\' => output.push(character), - _ => output.push_str(&escaped(character, escaping)), - } + let mut rest: &str = text; + while !rest.is_empty() { + let run: usize = rest + .bytes() + .position(|byte| !plain(byte)) + .unwrap_or(rest.len()); + output.push_str(&rest[..run]); + rest = &rest[run..]; + let Some(character) = rest.chars().next() else { + break; + }; + output = with_escaped(output, character, escaping); + rest = &rest[character.len_utf8()..]; } output.push('"'); output } -/// UTF-16 `units` as a quoted JSON string; `ensure_ascii` escapes a lone surrogate as `\uXXXX`. -pub(crate) fn json_utf16_string( +/// `output` followed by UTF-16 `units` as a quoted JSON string; `ensure_ascii` escapes lone surrogates. +pub(crate) fn with_json_utf16_string( + mut output: String, units: &[u16], escaping: StringEscaping, ) -> Result { - let mut output = String::with_capacity(units.len() + 2); + output.reserve(units.len() + 2); output.push('"'); for decoded in char::decode_utf16(units.iter().copied()) { - match (decoded, escaping) { - (Ok(character), _) => output.push_str(&escaped_or_plain(character, escaping)), + output = match (decoded, escaping) { + (Ok(character), _) => with_escaped(output, character, escaping), (Err(lone), StringEscaping::Ascii) => { - output.push_str(&unicode_escape(u32::from(lone.unpaired_surrogate()))); + with_unicode_escape(output, u32::from(lone.unpaired_surrogate())) } (Err(_), StringEscaping::Unicode) => return Err(JsonEmitError::LoneSurrogate), - } + }; } output.push('"'); Ok(output) } - -fn escaped_or_plain(character: char, escaping: StringEscaping) -> String { - match character { - ' '..='~' if character != '"' && character != '\\' => character.to_string(), - _ => escaped(character, escaping), - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index 0938071139..00b92ba4c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -1,5 +1,7 @@ //! The compile-lifetime binding catalog: known relation shapes around one native project catalog. +use std::sync::atomic::{AtomicBool, Ordering}; + use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyDictMethods, PyModule, PyModuleMethods, PyResult, Python, @@ -50,12 +52,19 @@ pub(crate) struct BindingCatalog { pub(crate) expression_shapes: Py, #[pyo3(get)] pub(crate) native: Py, + /// Whether binding preparation has added relations to `schemas`. + pub(crate) extended: AtomicBool, } /// A catalog's dialect, case mode, known functions and known types. pub(crate) type CatalogKey = (String, bool, Vec, Vec); impl BindingCatalog { + /// Whether `schemas` holds exactly the relations it was created with and later given. + pub(crate) fn unextended(&self) -> bool { + !self.extended.load(Ordering::Relaxed) + } + /// A catalog for `key` that starts with `relations`. pub(crate) fn create( py: Python<'_>, @@ -80,6 +89,7 @@ impl BindingCatalog { quoted_ignore_case, analysis_shapes: PyDict::new(py).unbind(), expression_shapes: PyDict::new(py).unbind(), + extended: AtomicBool::new(false), native: Py::new(py, native)?, }) } @@ -103,6 +113,7 @@ impl BindingCatalog { } } if !additions.is_empty() { + self.extended.store(true, Ordering::Relaxed); let relations: Relations = conversions::relations(additions.as_any())?; self.native.borrow_mut(py).inner.update_relations(relations); } @@ -207,6 +218,7 @@ impl BindingCatalog { analysis_shapes: self.analysis_shapes.bind(py).copy()?.unbind(), expression_shapes: self.expression_shapes.bind(py).copy()?.unbind(), native: Py::new(py, native)?, + extended: AtomicBool::new(true), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 04d020dd94..81960c08d6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -2,23 +2,27 @@ use std::path::PathBuf; -use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::types::{IntoPyDict, PyDict, PyDictMethods, PyFrozenSet, PyString}; use sqlbuild_analysis::assembly::analysis_session::types::Shapes; +use crate::bindings::_helpers::analysis::binding_catalog::BindingCatalog; +use crate::bindings::_helpers::project_assembly::declared_relations::DeclaredShapes; + use crate::bindings::_helpers::analysis_session::profile_inputs::{ case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, }; use crate::bindings::_helpers::analysis_session::session::{ - CacheStatsRow, NativeModelAnalysisSession, RequestRow, start_model_analysis_session, + CacheStatsRow, FamilyRow, ModelRow, NativeModelAnalysisSession, RequestRow, + start_model_analysis_session, }; use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::cache::native_store::{content_digest, current_code_identity}; use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; use crate::bindings::_helpers::compiled_project::model_input_view::{ - AttachedSchema, attached_schema, input_model_name, + AttachedSchema, NativeModelInput, attached_schema, input_model_name, input_query_sql, + input_reference_rows, }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; @@ -37,7 +41,7 @@ const STORE_FILE_NAME: &str = "model-analyses.bin"; const STORE_VERSION: &str = "native-model-analysis-store-v1"; /// What one compile's model analysis reads: the analysed models and the relation facts. -pub(crate) struct ModelAnalysisRequest<'py> { +pub(crate) struct ModelAnalysisRequest<'a, 'py> { pub(crate) model_inputs: Vec>, pub(crate) profile: Bound<'py, PyAny>, pub(crate) column_types: Bound<'py, PyAny>, @@ -47,6 +51,8 @@ pub(crate) struct ModelAnalysisRequest<'py> { pub(crate) rich_type_inference: bool, /// The `AnalysisCacheContext`, or `None` when the analysis store is bypassed. pub(crate) cache: Bound<'py, PyAny>, + /// Native rows of the three declaration maps, when the caller built them alongside. + pub(crate) declared: Option<&'a DeclaredShapes>, } /// Each analysed model's position in the finished session, those that succeeded, and the session. @@ -59,7 +65,7 @@ pub(crate) struct ModelAnalyses<'py> { /// Each model's analysis by name and the finished session; a native internal failure raises. pub(crate) fn analyze_models<'py>( - request: &ModelAnalysisRequest<'py>, + request: &ModelAnalysisRequest<'_, 'py>, ) -> PyResult> { let py: Python<'py> = request.profile.py(); if request.model_inputs.is_empty() { @@ -71,7 +77,7 @@ pub(crate) fn analyze_models<'py>( } let catalog: Bound<'py, PyAny> = native_field(&request.profile, "binding_catalog")?; let (rules, adapter_rules) = nullability_rules(&request.profile)?; - let session_request: RequestRow = session_request(request, &catalog, rules)?.extract()?; + let session_request: RequestRow = session_request(request, &catalog, rules)?; let (cache, adapter) = match adapter_rules { Some((adapter, nullability)) => (None, Some((adapter, nullability))), None => (analysis_store(request)?, None), @@ -90,36 +96,71 @@ pub(crate) fn analyze_models<'py>( /// The session request row, in the order the native session reads it. fn session_request<'py>( - request: &ModelAnalysisRequest<'py>, + request: &ModelAnalysisRequest<'_, 'py>, catalog: &Bound<'py, PyAny>, rules: Vec<(String, String)>, -) -> PyResult> { - let py: Python<'py> = catalog.py(); +) -> PyResult { let profile: &Bound<'py, PyAny> = &request.profile; - let mut families: Vec<(String, Vec<_>)> = Vec::new(); + let mut families: Vec<(String, Vec)> = Vec::new(); for item in mapping_items(&request.dynamic_families)?.try_iter()? { let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; families.push((name, family_rows(&declared)?)); } - let items: Vec> = vec![ - profile_dialect(profile)?.into_bound_py_any(py)?, - case_sensitive_shapes(profile, &native_field(profile, "sql_analysis_dialect")?)? - .into_bound_py_any(py)?, - return_types(profile)?.into_bound_py_any(py)?, - rules.into_bound_py_any(py)?, - request.rich_type_inference.into_bound_py_any(py)?, - shape_rows(&request.column_types)?.into_bound_py_any(py)?, - shape_rows(&request.nullability)?.into_bound_py_any(py)?, - shape_rows(&request.complete_schemas)?.into_bound_py_any(py)?, - shape_rows(&native_field(catalog, "schemas")?)?.into_bound_py_any(py)?, - families.into_bound_py_any(py)?, - model_rows(py, &request.model_inputs)?, - ]; - Ok(PyTuple::new(py, items)?.into_any()) + let (types, nullability, complete, schemas): (Shapes, Shapes, Shapes, Shapes) = + match declared_rows(request, catalog)? { + Some(rows) => rows, + None => ( + shape_rows(&request.column_types)?, + shape_rows(&request.nullability)?, + shape_rows(&request.complete_schemas)?, + shape_rows(&native_field(catalog, "schemas")?)?, + ), + }; + Ok(( + profile_dialect(profile)?, + case_sensitive_shapes(profile, &native_field(profile, "sql_analysis_dialect")?)?, + return_types(profile)?, + Some(rules), + request.rich_type_inference, + types, + nullability, + complete, + schemas, + families, + model_rows(profile.py(), &request.model_inputs)?, + )) +} + +/// The caller's native declaration rows when they still match the maps and catalog they mirror. +fn declared_rows( + request: &ModelAnalysisRequest<'_, '_>, + catalog: &Bound<'_, PyAny>, +) -> PyResult> { + let Some(declared) = request.declared else { + return Ok(None); + }; + let Ok(binding) = catalog.downcast::() else { + return Ok(None); + }; + let complete: &Shapes = declared.complete.rows(); + let unchanged: bool = binding.borrow().unextended() + && binding.borrow().schemas.bind(catalog.py()).len() == complete.len() + && request.complete_schemas.len()? == complete.len() + && request.column_types.len()? == declared.types.rows().len() + && request.nullability.len()? == declared.nullability.rows().len(); + if !unchanged { + return Ok(None); + } + Ok(Some(( + declared.types.rows().clone(), + declared.nullability.rows().clone(), + complete.clone(), + complete.clone(), + ))) } /// The analysis store's path and environment, or None where Python bypasses its cache. -fn analysis_store(request: &ModelAnalysisRequest<'_>) -> PyResult> { +fn analysis_store(request: &ModelAnalysisRequest<'_, '_>) -> PyResult> { let context: &Bound<'_, PyAny> = &request.cache; if context.is_none() { return Ok(None); @@ -138,7 +179,7 @@ fn analysis_store(request: &ModelAnalysisRequest<'_>) -> PyResult( - request: &ModelAnalysisRequest<'py>, + request: &ModelAnalysisRequest<'_, 'py>, catalog: &Bound<'py, PyAny>, session: &Bound<'py, NativeModelAnalysisSession>, ) -> PyResult> { @@ -249,25 +290,15 @@ fn record_metrics((hits, misses, bypasses): (usize, usize, usize)) -> PyResult<( } /// Each model's native request row, in analysis order. -fn model_rows<'py>( - py: Python<'py>, - model_inputs: &[Bound<'py, PyAny>], -) -> PyResult> { +fn model_rows<'py>(py: Python<'py>, model_inputs: &[Bound<'py, PyAny>]) -> PyResult> { let enforced: Bound<'py, PyAny> = enum_member(py, RecordId::ContractPolicy, "ENFORCED")?; let snapshot: Bound<'py, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; - let mut rows: Vec> = Vec::with_capacity(model_inputs.len()); + let mut rows: Vec = Vec::with_capacity(model_inputs.len()); for model_input in model_inputs { let values: Bound<'py, PyAny> = native_field(model_input, "config")?.getattr("values")?; let get = |key: &str| mapping_get(&values, key); - let query_sql: Bound<'py, PyAny> = PyString::new( - py, - &analysis_cursor_sql( - py, - &native_field(model_input, "query_sql")?.extract::()?, - &get("cursor_type")?, - )?, - ) - .into_any(); + let query_sql: String = + analysis_cursor_sql(py, &input_query_sql(model_input)?, &get("cursor_type")?)?; let mut placeholders: Vec<(String, String)> = Vec::new(); let raw: Bound<'py, PyAny> = get("placeholders")?; if raw.is_instance_of::() { @@ -276,14 +307,7 @@ fn model_rows<'py>( placeholders.push((key.str()?.extract()?, value.str()?.extract()?)); } } - let mut references: Vec<(String, String)> = Vec::new(); - for row in native_field(model_input, "reference_rows")?.try_iter()? { - let row: Bound<'py, PyAny> = row?; - references.push(( - row.get_item(0)?.str()?.extract()?, - row.get_item(1)?.extract()?, - )); - } + let references: Vec<(String, String)> = analysed_references(model_input)?; let schema: Option> = attached_schema(model_input)?; let recovers: bool = get("contract")?.eq(&enforced)? || schema @@ -301,18 +325,36 @@ fn model_rows<'py>( Some(schema) => family_rows(&schema.dynamic_columns)?, None => Vec::new(), }; - let row: Vec> = vec![ - input_model_name(model_input)?.into_bound_py_any(py)?, + rows.push(( + input_model_name(model_input)?, query_sql, - placeholders.into_bound_py_any(py)?, - references.into_bound_py_any(py)?, - recovers.into_bound_py_any(py)?, - snapshot_columns.into_bound_py_any(py)?, - families.into_bound_py_any(py)?, - ]; - rows.push(PyTuple::new(py, row)?.into_any()); + placeholders, + references, + recovers, + snapshot_columns, + families, + )); + } + Ok(rows) +} + +/// Each reference's `(kind, name)`; a native input answers from its record. +fn analysed_references(model_input: &Bound<'_, PyAny>) -> PyResult> { + if model_input.is_instance_of::() { + return Ok(input_reference_rows(model_input)? + .into_iter() + .map(|(kind, name, _, _)| (kind, name)) + .collect()); + } + let mut references: Vec<(String, String)> = Vec::new(); + for row in native_field(model_input, "reference_rows")?.try_iter()? { + let row: Bound<'_, PyAny> = row?; + references.push(( + row.get_item(0)?.str()?.extract()?, + row.get_item(1)?.extract()?, + )); } - Ok(PyList::new(py, rows)?.into_any()) + Ok(references) } /// `str(value or default)`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs index 98391124cc..7348587e6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -64,16 +64,21 @@ pub(crate) fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { let mut rows: Shapes = Vec::new(); for item in mapping_items(shapes)?.try_iter()? { let (name, shape): (String, Bound<'_, PyAny>) = item?.extract()?; - let mut pairs: Pairs = Vec::new(); - for column in mapping_items(&shape)?.try_iter()? { - let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; - pairs.push((column, value.str()?.extract()?)); - } - rows.push((name, pairs)); + rows.push((name, text_pairs(&shape)?)); } Ok(rows) } +/// `{column: value}` as ordered pairs, each value as `str(value)`. +pub(crate) fn text_pairs(shape: &Bound<'_, PyAny>) -> PyResult { + let mut pairs: Pairs = Vec::new(); + for column in mapping_items(shape)?.try_iter()? { + let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; + pairs.push((column, value.str()?.extract()?)); + } + Ok(pairs) +} + /// A `{key: value}` text mapping's items in order. pub(crate) fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult { mapping_items(mapping)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index ae07bd9735..6d7a78e232 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -48,7 +48,7 @@ type LineageItem = (String, u8, u8, Vec<(String, String, String)>); /// `(name, pivot column, value column, aggregate, type, name pattern)`. pub(crate) type FamilyRow = (String, String, String, String, String, Option); /// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. -type ModelRow = ( +pub(crate) type ModelRow = ( String, String, Pairs, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs index fae39e9dc2..a0bcf70fb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs @@ -2,19 +2,27 @@ use std::path::PathBuf; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use crate::bindings::_helpers::compile_command::reuse_recording::{ ArtifactRecorder, artifact_written, artifact_written_at, }; +use crate::bindings::_helpers::compile_driver::compile_timings::add_compile_timing; use crate::bindings::_helpers::compile_driver::compile_timings::record_compile_timing; -use crate::bindings::_helpers::compile_outputs::artifacts::write_compiled_artifacts; +use crate::bindings::_helpers::compile_outputs::artifacts::{ + BackgroundArtifactWriter, write_compiled_artifacts, +}; + +const PHYSICAL_WRITE_PHASE: &str = "physical_write_ms"; /// The artifacts of one write group, written together by `flush`. pub(crate) struct ArtifactBatch<'a> { check_existing: bool, recorder: ArtifactRecorder<'a>, files: Vec<(PathBuf, Vec)>, + /// Earlier groups, still being written in the background. + writer: Option, } /// `contents` with platform line endings. @@ -48,6 +56,7 @@ impl<'a> ArtifactBatch<'a> { check_existing, recorder, files: Vec::new(), + writer: None, } } @@ -77,14 +86,68 @@ impl<'a> ArtifactBatch<'a> { Ok(()) } - /// Write every queued artifact natively and empty the queue. + /// Write every queued artifact natively, after any background group, and empty the queue. pub(crate) fn flush(&mut self, py: Python<'_>) -> PyResult<()> { + self.finish_pending(py)?; if self.files.is_empty() { return Ok(()); } let files: Vec<(PathBuf, Vec)> = std::mem::take(&mut self.files); - record_compile_timing("physical_write_ms", || { + record_compile_timing(PHYSICAL_WRITE_PHASE, || { write_compiled_artifacts(py, files, self.check_existing).map(|_| ()) }) } + + /// Write the queued artifacts in the background, after every earlier background group. + pub(crate) fn flush_in_background(&mut self, py: Python<'_>) -> PyResult<()> { + if self.files.is_empty() { + return Ok(()); + } + if self.writer.is_none() { + self.writer = BackgroundArtifactWriter::start(); + } + let Some(writer) = &self.writer else { + return self.flush(py); + }; + writer.send(std::mem::take(&mut self.files), self.check_existing); + Ok(()) + } + + /// `result`, except that a background group's earlier write error wins and interrupts stop it. + pub(crate) fn settle(&mut self, py: Python<'_>, result: PyResult) -> PyResult { + let error: PyErr = match result { + Ok(value) => return Ok(value), + Err(error) => error, + }; + let Some(writer) = self.writer.take() else { + return Err(error); + }; + if !error.is_instance_of::(py) { + writer.stop(py); + return Err(error); + } + writer.finish(py)?; + Err(error) + } + + /// Wait for the background groups, raising the first write error. + fn finish_pending(&mut self, py: Python<'_>) -> PyResult<()> { + let Some(writer) = self.writer.take() else { + return Ok(()); + }; + let elapsed = writer.finish(py)?; + add_compile_timing( + PHYSICAL_WRITE_PHASE, + u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX), + ); + Ok(()) + } +} + +impl Drop for ArtifactBatch<'_> { + fn drop(&mut self) { + if let Some(writer) = self.writer.take() { + Python::attach(|py| writer.stop(py)); + } + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs index 480d6346bb..929f554d35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -176,15 +176,7 @@ impl<'py> ReportInputs<'py> { } fn semantic_notice(&self, reader: &SurrogateText) -> PyResult> { - let mut reasons: Vec<(String, Vec)> = Vec::new(); - for (name, values) in self.semantic_coverage()?.iter() { - let values: Vec = values - .try_iter()? - .map(|value| reader.text(&value?)) - .collect::>()?; - reasons.push((reader.text(&name)?, values)); - } - Ok(notice_text(&reasons)) + coverage_notice(&self.semantic_coverage()?, reader) } /// `written.target_dir / "compiled"` relative to its `target` directory, POSIX style. @@ -396,27 +388,20 @@ pub(crate) fn format_compile_text<'py>( reader.python(py, &compile_text(&report)) } -/// The partial semantic-check notice of the selected models, printed beside `--json` output. -pub(crate) fn json_semantic_notice<'py>( - graph: &Bound<'py, PyAny>, - selected_keys: &Bound<'py, PyAny>, - sql_validation_enabled: bool, -) -> PyResult>> { - let py: Python<'py> = graph.py(); - let reader: SurrogateText = SurrogateText::default(); - let notice: Option = ReportInputs { - project: native_field(graph, "project")?, - graph: graph.clone(), - written: py.None().into_bound(py), - manifest: false, - lineage: py.None().into_bound(py), - diagnostics: Vec::new(), - lineage_mode: String::new(), - selected_keys: selected_keys.clone(), - sql_validation_enabled, +/// The partial semantic-check notice of these coverage reasons, or `None` when complete. +fn coverage_notice( + coverage: &Bound<'_, PyDict>, + reader: &SurrogateText, +) -> PyResult> { + let mut reasons: Vec<(String, Vec)> = Vec::new(); + for (name, values) in coverage.iter() { + let values: Vec = values + .try_iter()? + .map(|value| reader.text(&value?)) + .collect::>()?; + reasons.push((reader.text(&name)?, values)); } - .semantic_notice(&reader)?; - notice.map(|notice| reader.python(py, ¬ice)).transpose() + Ok(notice_text(&reasons)) } fn dict<'py>( @@ -929,9 +914,37 @@ pub(crate) fn format_compile_json( py: Python<'_>, options: Option>, ) -> PyResult { - let arguments: KeywordArguments<'_> = + compile_json(py, options, None::) -> PyResult<()>>) +} + +/// `format_compile_json`, first handing the selected models' partial-check notice to `notice`. +pub(crate) fn format_compile_json_with_notice<'py>( + py: Python<'py>, + options: Option>, + notice: impl FnOnce(Bound<'py, PyString>) -> PyResult<()>, +) -> PyResult { + compile_json(py, options, Some(notice)) +} + +fn compile_json<'py>( + py: Python<'py>, + options: Option>, + notice: Option) -> PyResult<()>>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read(py, "format_compile_json", options, REPORT_OPTIONS)?; - let inputs: ReportInputs<'_> = ReportInputs::read(&arguments)?; + let inputs: ReportInputs<'py> = ReportInputs::read(&arguments)?; + let coverage: Option> = match notice { + Some(notice) => { + let coverage: Bound<'py, PyDict> = inputs.semantic_coverage()?; + let reader: SurrogateText = SurrogateText::default(); + if let Some(text) = coverage_notice(&coverage, &reader)? { + notice(reader.python(py, &text)?)?; + } + Some(coverage) + } + None => None, + }; let project: &Bound<'_, PyAny> = &inputs.project; let mut has_errors: bool = false; let mut diagnostics: Vec> = Vec::new(); @@ -1019,7 +1032,11 @@ pub(crate) fn format_compile_json( ("diagnostics", PyList::new(py, diagnostics)?.into_any()), ( "semantic_checks_partial", - inputs.semantic_coverage()?.into_any(), + match coverage { + Some(coverage) => coverage, + None => inputs.semantic_coverage()?, + } + .into_any(), ), ("compile_timings", arguments.value("timings_ms")?), ("lineage_mode", any(py, inputs.lineage_mode.as_str())?), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index c65aeee020..51a6ac15b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -611,36 +611,39 @@ pub(crate) fn write_static_target<'py>( )?; managed.push(path); } - batch.flush(py)?; - for function in native_field(project, "functions")?.try_iter()? { - let function: Bound<'py, PyAny> = function?; - if native_field(&function, "destination")? - .getattr("qualified_name")? - .is_none() - { - continue; + batch.flush_in_background(py)?; + let queued: PyResult> = (|| { + for function in native_field(project, "functions")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + if native_field(&function, "destination")? + .getattr("qualified_name")? + .is_none() + { + continue; + } + let sql: String = function_sql(adapter, &function)?; + let path: Bound<'py, PyAny> = compiled.div(function_output_path( + &native_field(&function, "relative_path")?, + &native_field(&function, "language")?, + )?)?; + batch.queue_sql(&path, &sql)?; + managed.push(path); } - let sql: String = function_sql(adapter, &function)?; - let path: Bound<'py, PyAny> = compiled.div(function_output_path( - &native_field(&function, "relative_path")?, - &native_field(&function, "language")?, - )?)?; - batch.queue_sql(&path, &sql)?; - managed.push(path); - } - batch.flush(py)?; - for audit in native_field(project, "audits")?.try_iter()? { - let audit: Bound<'py, PyAny> = audit?; - let path: Bound<'py, PyAny> = compiled.div(AUDITS_DIR)?.div(audit_output_path(&audit)?)?; - batch.queue_sql( - &path, - &native_field(&audit, "sql_body")?.extract::()?, - )?; - managed.push(path); - } - batch.flush(py)?; - let (test_paths, diagnostics) = - write_static_tests(target_dir, (adapter, project), prepared, &mut batch)?; + batch.flush_in_background(py)?; + for audit in native_field(project, "audits")?.try_iter()? { + let audit: Bound<'py, PyAny> = audit?; + let path: Bound<'py, PyAny> = + compiled.div(AUDITS_DIR)?.div(audit_output_path(&audit)?)?; + batch.queue_sql( + &path, + &native_field(&audit, "sql_body")?.extract::()?, + )?; + managed.push(path); + } + batch.flush_in_background(py)?; + write_static_tests(target_dir, (adapter, project), prepared, &mut batch) + })(); + let (test_paths, diagnostics) = batch.settle(py, queued)?; managed.extend(test_paths); if remove_stale_files { let mut paths: Vec = managed_models; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 0dd7aef323..60ec1d1a43 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -1,14 +1,17 @@ //! Project assembly's inference profile, binding catalog, source shapes and model analysis. +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignores_quoted_case}; use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; +use crate::bindings::_helpers::analysis_session::profile_inputs::text_pairs; use crate::bindings::_helpers::analysis_session::stage_functions::expression_source_shapes; use crate::bindings::_helpers::compile_driver::analysis_cache::analysis_cache_context; use crate::bindings::_helpers::compile_driver::assembly::{AssemblyRequest, AssemblyStage}; @@ -20,10 +23,11 @@ use crate::bindings::_helpers::compile_driver::stage_work::{ EXPRESSION_SHAPES, MODEL_ANALYSIS, count_stage, }; use crate::bindings::_helpers::compiled_project::model_input_view::attached_schema; +use crate::bindings::_helpers::compiled_project::source_views::read_source_view; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::project_assembly::declared_relations::{ - DeclaredRelations, declared_relations, + DeclaredRelations, DeclaredShapes, declared_relations, }; use crate::bindings::_helpers::records::construct::{enum_member, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -47,6 +51,8 @@ pub(crate) struct AnalysisContext<'py> { pub(crate) dynamic_families: Bound<'py, PyAny>, /// The model analyses, or `None` when SQL analysis does not run. pub(crate) analyses: Option>, + /// The native rows of `column_types`, `nullability` and `complete_schemas`. + pub(crate) declared: DeclaredShapes, } /// Whether `model_input` is selected for analysis by `names` (`None` selects every model). @@ -87,7 +93,7 @@ pub(crate) fn analysis_context<'py>( &native_field(inputs, "effective_settings")?.getattr("sql_analysis")?, request.skip_column_inference.bind(py), )?; - let (column_types, nullability, complete_schemas): DeclaredRelations<'py> = + let (column_types, nullability, complete_schemas, declared): DeclaredRelations<'py> = declared_relations(inputs.clone())?; let (column_types, nullability, complete_schemas): ( Bound<'py, PyAny>, @@ -109,9 +115,11 @@ pub(crate) fn analysis_context<'py>( complete_schemas, dynamic_families, analyses: None, + declared, }; if context.sql_analysis_enabled.is_truthy()? { - record_expression_shapes(request, &context)?; + let declared: DeclaredShapes = std::mem::take(&mut context.declared); + context.declared = record_expression_shapes(request, &context, declared)?; } let catalog: Bound<'py, PyAny> = native_field(&context.profile, "binding_catalog")?; native_field(&catalog, "native")? @@ -252,15 +260,15 @@ fn function_return_types<'py>( fn record_expression_shapes<'py>( request: &AssemblyRequest, context: &AnalysisContext<'py>, -) -> PyResult<()> { + mut declared: DeclaredShapes, +) -> PyResult { let py: Python<'py> = context.profile.py(); let inputs: &Bound<'py, PyAny> = request.inputs.bind(py); let mut sources: Vec> = Vec::new(); let mut expressions: Vec> = Vec::new(); for source_input in native_field(inputs, "source_inputs")?.try_iter()? { let source_input: Bound<'py, PyAny> = source_input?; - let expression: Bound<'py, PyAny> = - native_field(&source_input, "source_entry")?.getattr("expression")?; + let expression: Bound<'py, PyAny> = source_expression(&source_input)?; if expression.is_truthy()? { sources.push(source_input); expressions.push(expression); @@ -295,8 +303,7 @@ fn record_expression_shapes<'py>( if shape.is_none() { continue; } - let name: Bound<'py, PyAny> = - native_field(source_input, "source_entry")?.getattr("name")?; + let name: Bound<'py, PyAny> = source_name(source_input)?; let declared_types: Bound<'py, PyAny> = mapping_get_or(&context.column_types, &name, PyDict::new(py))?; let declared_shape: Bound<'py, PyDict> = PyDict::new(py); @@ -316,9 +323,40 @@ fn record_expression_shapes<'py>( mapping_get_or(&context.nullability, &name, PyDict::new(py))?; nullability.set_item(&column, mapping_get_or(&known, &column, &unknown)?)?; } - context.nullability.set_item(&name, nullability)?; + context.nullability.set_item(&name, &nullability)?; + let key: String = name.extract()?; + let shape_pairs: Pairs = text_pairs(declared_shape.as_any())?; + declared.complete.set(key.clone(), shape_pairs.clone()); + declared.types.set(key.clone(), shape_pairs); + declared + .nullability + .set(key, text_pairs(nullability.as_any())?); + } + Ok(declared) +} + +/// `source_input.source_entry.expression`; a native source answers from its attached record. +fn source_expression<'py>(source_input: &Bound<'py, PyAny>) -> PyResult> { + match read_source_view(source_input, |source, py| { + source + .attached() + .expression + .as_deref() + .into_bound_py_any(py) + }) { + Some(expression) => expression, + None => native_field(source_input, "source_entry")?.getattr("expression"), + } +} + +/// `source_input.source_entry.name`; a native source answers from its attached record. +fn source_name<'py>(source_input: &Bound<'py, PyAny>) -> PyResult> { + match read_source_view(source_input, |source, py| { + Ok(PyString::new(py, &source.attached().name).into_any()) + }) { + Some(name) => name, + None => native_field(source_input, "source_entry")?.getattr("name"), } - Ok(()) } /// Each selected model's native analysis, timed as `model_analysis_ms`, when SQL analysis runs. @@ -366,6 +404,7 @@ fn model_analyses<'py>( dynamic_families: context.dynamic_families.clone(), rich_type_inference: rich, cache, + declared: Some(&context.declared), }) .map(Some) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 61fa9bf4f9..5942689a4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -1,14 +1,17 @@ //! The `sqb compile` command: analysis, output phases, reuse recording and printed output. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyTuple}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PyString, PyTuple, +}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use std::rc::Rc; use crate::bindings::_helpers::build_progress::transient_status::status_reporter; use crate::bindings::_helpers::compile_command::models::CompileReuseAttempt; use crate::bindings::_helpers::compile_command::report::{ - format_compile_json, format_compile_text, json_semantic_notice, + format_compile_json_with_notice, format_compile_text, }; use crate::bindings::_helpers::compile_command::reuse_recording::write_reusable_compile; use crate::bindings::_helpers::compile_command::reuse_recording::{ @@ -36,6 +39,23 @@ use crate::bindings::_helpers::rules::results::format_skipped_type_proof_rules; const BYPASS_OUTCOME: &str = "bypass"; +/// Compile state kept alive for process exit to reclaim, once a launcher defers teardown. +static RETAINED_UNTIL_EXIT: PyOnceLock> = PyOnceLock::new(); + +/// Let compiles in this process leave their project state for process exit to reclaim. +#[pyfunction] +fn defer_compile_teardown_to_exit(py: Python<'_>) { + let _ = RETAINED_UNTIL_EXIT.get_or_init(py, || PyList::empty(py).unbind()); +} + +/// Keep `value` alive until the process exits when teardown is deferred. +fn retain_until_exit(value: &Bound<'_, PyAny>) -> PyResult<()> { + match RETAINED_UNTIL_EXIT.get(value.py()) { + Some(retained) => retained.bind(value.py()).append(value), + None => Ok(()), + } +} + /// What one full compile reports for reuse recording. struct CommandOutcome<'py> { exit_code: i64, @@ -87,8 +107,9 @@ pub(crate) fn run_compile_command<'py>( (&output, &status), total_start, ); - if was_enabled { - gc.call_method0("enable")?; + let deferred: bool = RETAINED_UNTIL_EXIT.get(py).is_some(); + if deferred || was_enabled { + gc.call_method0(if deferred { "freeze" } else { "enable" })?; } exit_code } @@ -168,6 +189,7 @@ fn recorded_compile<'py>( ), ], )?; + retain_until_exit(&manager)?; within(&manager, |prepared_artifacts| { let recorders: Recorders<'py> = Recorders { detailed_timings: Rc::clone(detailed_timings), @@ -248,6 +270,7 @@ fn compiled_with_status<'py>( py, &analysis_options(request, &project_dir, recorders, status)?, )?; + retain_until_exit(&analysis)?; let rules_failed: bool = rules_failed(&analysis)?; let skip_write: bool = native_field(request, "profile_flags")? .getattr("skip_write")? @@ -435,16 +458,15 @@ fn record_output<'py>( ]; let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = shared.to_vec(); let rendered: Bound<'py, PyString> = if native_field(request, "json_output")?.is_truthy()? { - let notice: Option> = - json_semantic_notice(&graph, &selected_keys, validation_enabled.is_truthy()?)?; - if let Some(notice) = notice { - output.call_method1("print_stderr_line", (notice,))?; - } arguments.push(("timings_ms", timings.clone().into_any())); arguments.push(("sql_validation_enabled", validation_enabled)); + let print_notice = |notice: Bound<'py, PyString>| -> PyResult<()> { + output.call_method1("print_stderr_line", (notice,))?; + Ok(()) + }; PyString::new( py, - &format_compile_json(py, Some(keywords(py, &arguments)?))?, + &format_compile_json_with_notice(py, Some(keywords(py, &arguments)?), print_notice)?, ) } else { let use_color: bool = @@ -461,5 +483,6 @@ fn record_output<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(defer_compile_teardown_to_exit, module)?)?; module.add_function(wrap_pyfunction!(run_compile_command, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs index 4169bc2566..1d1d6a7af9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/integration_loaders.rs @@ -35,48 +35,50 @@ pub(crate) fn integration_loader_functions<'py>( py: Python<'py>, source_files: &Bound<'py, PyAny>, ) -> PyResult> { + let mut declared: Vec<(String, Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); + for source_file in source_files.try_iter()? { + let source_file: Bound<'py, PyAny> = source_file?; + for source_entry in native_field(&source_file, "source_entries")?.try_iter()? { + let source_entry: Bound<'py, PyAny> = source_entry?; + let loader: Bound<'py, PyAny> = native_field(&source_entry, "integration_loader")?; + if loader.is_none() { + continue; + } + let kind: String = native_field(&loader, "kind")?.extract()?; + declared.push((kind, source_file.clone(), source_entry)); + } + } + if declared.is_empty() { + return Ok(PyTuple::empty(py).into_any()); + } let external: Bound<'py, PyAny> = enum_member(py, RecordId::LoaderConnectionMode, "EXTERNAL")?; let mut loaders: Vec> = Vec::new(); for integration in &INTEGRATIONS { let kind: String = host::entry(py, integration.kind_constant)?.extract()?; let factory: Bound<'py, PyAny> = host::entry(py, integration.loader_factory)?; - for source_file in source_files.try_iter()? { - let source_file: Bound<'py, PyAny> = source_file?; - for source_entry in native_field(&source_file, "source_entries")?.try_iter()? { - let source_entry: Bound<'py, PyAny> = source_entry?; - let declared: Bound<'py, PyAny> = - native_field(&source_entry, "integration_loader")?; - if declared.is_none() - || native_field(&declared, "kind")?.extract::()? != kind - { - continue; - } - let name: String = integration_loader_name( - &kind, - &native_field(&source_entry, "name")?.extract::()?, - ); - let function: Bound<'py, PyAny> = call_with( - &factory, - &[ - ("source_entry", source_entry), - ("name", name.clone().into_pyobject(py)?.into_any()), - ], - )?; - loaders.push(new_record_with( - py, - RecordId::DiscoveredLoaderFunction, - &[ - ("file_path", native_field(&source_file, "file_path")?), - ( - "relative_path", - native_field(&source_file, "relative_path")?, - ), - ("name", name.into_pyobject(py)?.into_any()), - ("function", function), - ("connection_mode", external.clone()), - ], - )?); - } + for (_, source_file, source_entry) in declared.iter().filter(|loader| loader.0 == kind) { + let name: String = integration_loader_name( + &kind, + &native_field(source_entry, "name")?.extract::()?, + ); + let function: Bound<'py, PyAny> = call_with( + &factory, + &[ + ("source_entry", source_entry.clone()), + ("name", name.clone().into_pyobject(py)?.into_any()), + ], + )?; + loaders.push(new_record_with( + py, + RecordId::DiscoveredLoaderFunction, + &[ + ("file_path", native_field(source_file, "file_path")?), + ("relative_path", native_field(source_file, "relative_path")?), + ("name", name.into_pyobject(py)?.into_any()), + ("function", function), + ("connection_mode", external.clone()), + ], + )?); } } Ok(PyTuple::new(py, loaders)?.into_any()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs index b1718b837f..8dcc012f25 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs @@ -1,7 +1,13 @@ //! Write, clean and publish compiled artifacts natively, raising the errors Python raised. use std::collections::HashSet; +use std::panic::resume_unwind; use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::sync::{Arc, Condvar, Mutex, PoisonError}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; use pyo3::exceptions::PyOSError; use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; @@ -17,6 +23,12 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; type PublishedRow = (String, Vec<(String, String)>); +/// One write group's `(path, contents)` artifacts and whether unchanged files are kept. +type ArtifactGroup = (Vec<(PathBuf, Vec)>, bool); + +const WRITER_THREAD: &str = "sqlbuild-artifact-writer"; +const WRITER_POLL: Duration = Duration::from_millis(50); + /// Write each `(path, contents)` artifact; returns how many were written and left unchanged. pub(crate) fn write_compiled_artifacts( py: Python<'_>, @@ -31,6 +43,145 @@ pub(crate) fn write_compiled_artifacts( }) } +/// Artifact groups written one after another on their own thread, stopping at the first error. +pub(crate) struct BackgroundArtifactWriter { + groups: Sender, + /// Set to skip every group not yet started; the group being written still completes. + stop: Arc, + /// Marked once the writer returns, so a wait can check for signals meanwhile. + finished: Arc, + handle: JoinHandle<(Result<(), ArtifactError>, Duration)>, +} + +/// Write each group in order until one fails or a stop is requested between groups. +fn write_groups( + groups: &Receiver, + stop: &AtomicBool, +) -> (Result<(), ArtifactError>, Duration) { + let mut elapsed: Duration = Duration::ZERO; + for (files, check_existing) in groups { + if stop.load(Ordering::SeqCst) { + break; + } + let started: Instant = Instant::now(); + let written: Result = + write_artifacts(&files, check_existing); + elapsed += started.elapsed(); + if let Err(error) = written { + return (Err(error), elapsed); + } + } + (Ok(()), elapsed) +} + +impl BackgroundArtifactWriter { + /// Start the writer thread, or `None` when no thread could start. + pub(crate) fn start() -> Option { + let (groups, received) = channel(); + let finished: Arc = Arc::new(WriterDone::default()); + let marker: DoneMarker = DoneMarker(Arc::clone(&finished)); + let stop: Arc = Arc::new(AtomicBool::new(false)); + let stopping: Arc = Arc::clone(&stop); + let spawned = std::thread::Builder::new() + .name(WRITER_THREAD.to_owned()) + .spawn(move || { + let _marker: DoneMarker = marker; + write_groups(&received, &stopping) + }); + match spawned { + Ok(handle) => Some(Self { + groups, + stop, + finished, + handle, + }), + Err(_) => None, + } + } + + /// Queue one group behind the groups already sent; a writer that stopped drops it. + pub(crate) fn send(&self, files: Vec<(PathBuf, Vec)>, check_existing: bool) { + let _ = self.groups.send((files, check_existing)); + } + + /// Wait without the GIL for every sent group; a signal stops it after the current group. + pub(crate) fn finish(self, py: Python<'_>) -> PyResult { + let Self { + groups, + stop, + finished, + handle, + } = self; + drop(groups); + if let Err(interrupt) = wait_for_writer(py, &finished) { + stop.store(true, Ordering::SeqCst); + let _ = py.detach(|| handle.join()); + return Err(interrupt); + } + compiler_guard(|| { + let joined = py.detach(|| handle.join()); + let (outcome, elapsed) = joined.unwrap_or_else(|payload| resume_unwind(payload)); + outcome.map_err(|error| artifact_error(py, error))?; + Ok(elapsed) + }) + } + + /// Stop after the group being written and wait for it without the GIL, ignoring the outcome. + pub(crate) fn stop(self, py: Python<'_>) { + let Self { + groups, + stop, + finished, + handle, + } = self; + stop.store(true, Ordering::SeqCst); + drop(groups); + drop(finished); + let _ = py.detach(|| handle.join()); + } +} + +/// Whether the writer thread has returned, with a condition variable to wait on it. +#[derive(Default)] +struct WriterDone { + finished: Mutex, + changed: Condvar, +} + +/// Marks the writer done when the writer thread returns or unwinds. +struct DoneMarker(Arc); + +impl Drop for DoneMarker { + fn drop(&mut self) { + *self + .0 + .finished + .lock() + .unwrap_or_else(PoisonError::into_inner) = true; + self.0.changed.notify_all(); + } +} + +impl WriterDone { + /// Whether the writer returned within `timeout`. + fn finished_within(&self, timeout: Duration) -> bool { + let finished = self.finished.lock().unwrap_or_else(PoisonError::into_inner); + let (finished, _) = self + .changed + .wait_timeout_while(finished, timeout, |finished| !*finished) + .unwrap_or_else(PoisonError::into_inner); + *finished + } +} + +/// Wait without the GIL until the writer returns, checking for signals between waits. +fn wait_for_writer(py: Python<'_>, finished: &WriterDone) -> PyResult<()> { + while !py.detach(|| finished.finished_within(WRITER_POLL)) { + py.check_signals()?; + } + Ok(()) +} + /// Delete unmanaged compiled files and empty directories; returns how many files were removed. pub(crate) fn remove_stale_artifacts( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index 83db2ce746..8871aa9d1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -195,6 +195,16 @@ impl AnalysisRecord { .map(|lineage| lineage.clone_ref(py)) } + /// The number of upstream sources across the model's lineage rows, or `None` without lineage. + pub(crate) fn lineage_edge_count(&self) -> Option { + match &self.lineage { + None => None, + Some(LineageFacts::Native(rows) | LineageFacts::NativeFacts(rows)) => { + Some(rows.iter().map(|row| row.sources.len()).sum()) + } + } + } + /// The model's `CompilerDiagnostic` tuple, built once. pub(crate) fn binding_tuple( &self, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index a9f7b3142f..cd7cf04bb6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -735,6 +735,11 @@ impl NativeCompiledModel { }) } + /// The upstream source count of the model's lineage facts, `None` without them. + pub(crate) fn lineage_edge_count(&self, py: Python<'_>) -> PyResult> { + self.compiled(py, |compiled, _| Ok(compiled.analysis.lineage_edge_count())) + } + #[getter] pub(crate) fn fast_lineage_columns(&self, py: Python<'_>) -> PyResult> { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs index fb9f23ff34..3660b9432b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -389,17 +389,14 @@ pub(crate) fn model_loop_classes<'py>( )?; let record: Bound<'py, PyDict> = dict( py, - vec![ - ("replace", imported(py, "dataclasses", "replace")?), - ( - "declaration_rows", - pyo3::wrap_pyfunction!( - crate::bindings::_helpers::compiled_project::records::declaration_rows, - py - )? - .into_any(), - ), - ], + vec![( + "declaration_rows", + pyo3::wrap_pyfunction!( + crate::bindings::_helpers::compiled_project::records::declaration_rows, + py + )? + .into_any(), + )], )?; dict( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index 9b56f17be5..bc37f09b55 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -535,6 +535,16 @@ pub(crate) fn input_model_name(model_input: &Bound<'_, PyAny>) -> PyResult) -> PyResult { + if let Ok(native) = model_input.cast::() { + return native.get().read(model_input.py(), |record, _| { + Ok(record.query_sql.to_string()) + }); + } + native_field(model_input, "query_sql")?.extract() +} + /// Each reference of `model_input` as `(kind, name, package, call argument count)`. pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = model_input.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 79036fdaeb..38a437789d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -24,7 +24,7 @@ use crate::bindings::_helpers::compiled_project::project::{ DeclarationPy, config_values, declarations, }; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{field as native_field, replaced_keywords}; use crate::bindings::_helpers::render::declaration_expansion::{ ExpandedDeclarations, ScannedReference, span_rows, }; @@ -103,8 +103,6 @@ pub(crate) struct FinishedModel<'py> { #[derive(FromPyObject)] #[pyo3(from_item_all)] pub(crate) struct RecordClasses<'py> { - /// `dataclasses.replace`. - replace: Bound<'py, PyAny>, /// `declaration_rows`, the facts rows of the model's local enums and constants. declaration_rows: Bound<'py, PyAny>, } @@ -159,10 +157,7 @@ pub(crate) fn finished_record<'py>( .extract()?; let replaced: Bound<'py, PyDict> = PyDict::new(py); replaced.set_item(VALUES_ARGUMENT, &stripped)?; - let config: Bound<'py, PyAny> = inputs - .classes - .replace - .call((&finished.config,), Some(&replaced))?; + let config: Bound<'py, PyAny> = replaced_keywords(&finished.config, &replaced)?; let mut declaration_usages: Vec> = unique(py, usages)?; declaration_usages.extend(finished.named_usages); let (macro_sql, macro_spans, macro_deps, macro_usages) = match finished.macros { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs index d72ac55f30..611a222050 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/schema_facts.rs @@ -1,6 +1,6 @@ //! Native facts of declared schema columns and families, read once from their Python specs. -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{field as native_field, record_attribute_opt}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::compiled_project::models::{ DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, @@ -116,7 +116,11 @@ pub(crate) fn family_fact(family: &DynamicFamily) -> DynamicFamilyFact { /// An attribute's value, `None` when it is missing or `None`. fn present<'py>(spec: &Bound<'py, PyAny>, name: &str) -> PyResult>> { - Ok(spec.getattr_opt(name)?.filter(|value| !value.is_none())) + let value: Option> = match record_attribute_opt(spec, name) { + Some(value) => value?, + None => spec.getattr_opt(name)?, + }; + Ok(value.filter(|value| !value.is_none())) } fn optional<'py, T: pyo3::FromPyObject<'py>>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs index a665075ef2..4f3e3d1204 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs @@ -1,7 +1,7 @@ //! `build_source_inputs`: every discovered source attached natively into the compiled project. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use sqlbuild_analysis::assembly::project::models::{SyntaxCheck, SyntaxMode}; use sqlbuild_attachments::cursor_intrinsics::main::intrinsic_free::intrinsic_free; use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicCheck; @@ -415,8 +415,39 @@ fn expanded_audit( replaced_spec(audit, &changes) } +/// `dataclasses.replace(value, **changes)`, or `value` itself when every change keeps its field. fn replaced_spec(value: &Bound<'_, PyAny>, changes: &Bound<'_, PyDict>) -> PyResult> { - replaced(value.py(), value, changes) + for (name, changed) in changes.iter() { + let current: Bound<'_, PyAny> = native_field(value, &name.extract::()?)?; + if !same_value(¤t, &changed)? { + return replaced(value.py(), value, changes); + } + } + Ok(value.clone().unbind()) +} + +/// Whether `changed` is `current` or an equal value of the same type, element by element. +fn same_value(current: &Bound<'_, PyAny>, changed: &Bound<'_, PyAny>) -> PyResult { + if current.is(changed) { + return Ok(true); + } + if !current.get_type().is(changed.get_type()) { + return Ok(false); + } + if let (Ok(current), Ok(changed)) = + (current.downcast::(), changed.downcast::()) + { + if current.len() != changed.len() { + return Ok(false); + } + for (left, right) in current.iter().zip(changed.iter()) { + if !left.is(&right) { + return Ok(false); + } + } + return Ok(true); + } + current.eq(changed) } /// `validate_source_expression_syntax`: the expression parses as a FROM target. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index 59d85a8209..48690285f0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -5,15 +5,18 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyclass, pymethods}; +use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::compiled_model_view::object_key; use crate::bindings::_helpers::compiled_project::model_input_view::{ InputViewClasses, view_classes, }; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::graph::project_graph::ResourceRow; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; +use crate::bindings::_helpers::rules::relation_keys::audit_key_rows; const SOURCE_RESOURCE: &str = "source"; @@ -43,9 +46,16 @@ pub(crate) struct AttachedSource { pub(crate) usages: Py, } +/// What a source declares about its keys: its column facts and `(audit, column argument)` rows. +pub(crate) struct SourceKeyFacts { + pub(crate) columns: Vec, + pub(crate) audits: Vec<(String, Option)>, +} + /// One attached source held by the compiled project. pub(crate) struct SourceRecord { attached: AttachedSource, + key_facts: PyOnceLock, /// The namespace native project assembly resolved, once it ran. namespace: Option, attached_entry: PyOnceLock>, @@ -57,6 +67,7 @@ impl SourceRecord { pub(crate) fn new(attached: AttachedSource) -> Self { Self { attached, + key_facts: PyOnceLock::new(), namespace: None, attached_entry: PyOnceLock::new(), assembled_entry: PyOnceLock::new(), @@ -69,6 +80,16 @@ impl SourceRecord { &self.attached } + /// The source's declared column facts and audit key rows, read once from its expanded specs. + pub(crate) fn key_facts(&self, py: Python<'_>) -> PyResult<&SourceKeyFacts> { + self.key_facts.get_or_try_init(py, || { + Ok(SourceKeyFacts { + columns: python_column_facts(self.attached.columns.bind(py).as_any(), None)?, + audits: audit_key_rows(self.attached.audits.bind(py).as_any())?, + }) + }) + } + /// Keep the namespace native project assembly resolved for the source. pub(crate) fn set_namespace(&mut self, namespace: SourceNamespace) { self.namespace = Some(namespace); @@ -120,6 +141,23 @@ pub(crate) fn replaced( Ok(replaced_keywords(value, changes)?.unbind()) } +/// Read the source record behind a native source view, or `None` for any other object. +pub(crate) fn read_source_view<'py, T>( + view: &Bound<'py, PyAny>, + read: impl FnOnce(&SourceRecord, Python<'py>) -> PyResult, +) -> Option> { + let py: Python<'py> = view.py(); + if let Ok(input) = view.cast::() { + let input: &NativeSourceInput = input.get(); + return Some(read_source(&input.owner, input.index, py, read)); + } + let Ok(compiled) = view.cast::() else { + return None; + }; + let compiled: &NativeCompiledSource = compiled.get(); + Some(read_source(&compiled.owner, compiled.index, py, read)) +} + fn read_source<'py, T>( owner: &Py, index: usize, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index 66ae0dd634..fe0176e6ef 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -1,6 +1,7 @@ //! The facts cross-file validation reads, taken from Python's `DiscoveredProjectInputs`. -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; +use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use pyo3::FromPyObject; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -95,7 +96,7 @@ fn optional_text(owner: &Bound<'_, PyAny>, name: &str) -> PyResult, name: &str) -> PyResult { - native_field(&native_field(owner, name)?, "stem")?.extract() + path_stem(&native_field(owner, name)?) } /// `(resource.name, str(resource.relative_path))`. @@ -106,8 +107,15 @@ fn named(resource: &Bound<'_, PyAny>) -> PyResult { }) } -/// `(file.file_path.stem, str(file.relative_path))`. +/// `(file.file_path.stem, str(file.relative_path))`; a native model file answers from its texts. fn stem_named(file: &Bound<'_, PyAny>) -> PyResult { + if let Ok(model_file) = file.cast::() { + let texts: ModelFileTexts = model_file.get().path_texts(file.py())?; + return Ok(NamedPath { + name: texts.name, + path: texts.relative_path, + }); + } Ok(NamedPath { name: stem(file, "file_path")?, path: text(file, "relative_path")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs index 55c95a0e56..6f636c6d51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_blocks.rs @@ -1,7 +1,12 @@ //! SQL `TEST(...)` blocks validated natively and built as discovery records. +use std::collections::HashMap; +use std::sync::Mutex; + use pyo3::IntoPyObjectExt; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_model_config::sql_tests::main::sql_test_block::sql_test_block; use sqlbuild_model_config::sql_tests::main::sql_test_names_failure::sql_test_names_failure; @@ -53,6 +58,22 @@ fn case_record<'py>( ) } +/// `SqlTestMode(mode)`, the member each mode resolves to looked up once per process. +fn test_mode<'py>(py: Python<'py>, mode: &'static str) -> PyResult> { + static MODES: PyOnceLock>>> = PyOnceLock::new(); + let modes: &Mutex>> = + MODES.get_or_init(py, || Mutex::new(HashMap::new())); + let mut known = modes + .lock() + .map_err(|_| PyRuntimeError::new_err("SQL test mode cache is poisoned"))?; + if let Some(member) = known.get(mode) { + return Ok(member.bind(py).clone()); + } + let member: Bound<'py, PyAny> = host::entry(py, FacadeHost::SqlTestMode)?.call1((mode,))?; + let _ = known.insert(mode, member.clone().unbind()); + Ok(member) +} + fn block_record<'py>( py: Python<'py>, (index, native): (usize, &NativeBlock<'py>), @@ -86,10 +107,7 @@ fn block_record<'py>( keywords.set_item("header_values", &native.0)?; keywords.set_item("sql_body", native.1.as_str())?; keywords.set_item("name", block.name)?; - keywords.set_item( - "mode", - host::entry(py, FacadeHost::SqlTestMode)?.call1((block.mode,))?, - )?; + keywords.set_item("mode", test_mode(py, block.mode)?)?; keywords.set_item("parameters", PyTuple::new(py, parameters)?)?; keywords.set_item("cases", PyTuple::new(py, cases)?)?; keywords.set_item("cursor_start", block.cursor_start)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index ba0d85caeb..91df11f629 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -18,10 +18,32 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; +/// Where a compact model's facts are: given, or projected from its compiled model on first read. +pub(crate) enum CompactColumns { + Facts(Py), + /// A compiled model whose `fast_lineage_columns` are the facts, and their upstream count. + Model { + model: Py, + edges: usize, + }, +} + /// One compact model's facts and root-star flag. -struct CompactModel { - columns: Py, - has_star: bool, +pub(crate) struct CompactModel { + pub(crate) columns: CompactColumns, + pub(crate) has_star: bool, +} + +impl CompactModel { + /// The model's lineage facts as Python objects. + fn facts<'py>(&self, py: Python<'py>) -> PyResult> { + match &self.columns { + CompactColumns::Facts(columns) => Ok(columns.bind(py).clone()), + CompactColumns::Model { model, .. } => { + native_field(model.bind(py), "fast_lineage_columns") + } + } + } } /// The Python objects one edge record materialises from. @@ -103,30 +125,12 @@ impl ProjectColumnLineage { let _ = compact.insert( name.extract()?, CompactModel { - columns: columns.unbind(), + columns: CompactColumns::Facts(columns.unbind()), has_star, }, ); } - let mut edge_counts: HashMap = HashMap::new(); - for name in &model_order { - if let Some(model) = compact.get(name) { - let count: usize = fact_edge_count(&classes, model.columns.bind(cls.py()))?; - let _ = edge_counts.insert(name.clone(), count); - } - } - for item in mapping_items(models)?.try_iter()? { - let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; - let _ = edge_counts.insert(name, upstream_count(&native_field(&model, "columns")?)?); - } - Ok(Self { - models: models.clone().unbind(), - compact, - model_order, - edges: None, - records: None, - edge_counts, - }) + Self::compacted(&classes, models, compact, model_order) } /// Return model lineage, expanding compact model facts on first access. @@ -144,7 +148,7 @@ impl ProjectColumnLineage { continue; } let lineage: Bound<'_, PyAny> = - model_from_facts(&classes, name, compact.columns.bind(py), compact.has_star)?; + model_from_facts(&classes, name, &compact.facts(py)?, compact.has_star)?; models.set_item(name, lineage)?; } slf.compact.clear(); @@ -275,6 +279,38 @@ impl ProjectColumnLineage { } impl ProjectColumnLineage { + /// Indexed project lineage of built `models` and `compact` facts, in `model_order`. + pub(crate) fn compacted( + classes: &LineageClasses<'_>, + models: &Bound<'_, PyAny>, + compact: HashMap, + model_order: Vec, + ) -> PyResult { + let py: Python<'_> = models.py(); + let mut edge_counts: HashMap = HashMap::new(); + for name in &model_order { + if let Some(model) = compact.get(name) { + let count: usize = match &model.columns { + CompactColumns::Facts(columns) => fact_edge_count(classes, columns.bind(py))?, + CompactColumns::Model { edges, .. } => *edges, + }; + let _ = edge_counts.insert(name.clone(), count); + } + } + for item in mapping_items(models)?.try_iter()? { + let (name, model): (String, Bound<'_, PyAny>) = item?.extract()?; + let _ = edge_counts.insert(name, upstream_count(&native_field(&model, "columns")?)?); + } + Ok(Self { + models: models.clone().unbind(), + compact, + model_order, + edges: None, + records: None, + edge_counts, + }) + } + /// Build fast edge records from the compact facts, or the expanded models, in model order. fn ensure_records(&mut self, classes: &LineageClasses<'_>) -> PyResult<()> { if self.records.is_some() { @@ -285,7 +321,7 @@ impl ProjectColumnLineage { let models: Bound<'_, PyAny> = self.models.bind(py).clone(); for name in &self.model_order { if let Some(compact) = self.compact.get(name) { - builder.push_facts(classes, name, compact.columns.bind(py))?; + builder.push_facts(classes, name, &compact.facts(py)?)?; continue; } let model: Bound<'_, PyAny> = mapping_get(&models, name)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 8254b952d8..49c73bacfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -1,5 +1,7 @@ //! Project column lineage in FAST or RICH mode, and direct semantic uses, from a project view. +use std::collections::HashMap; + use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; @@ -18,7 +20,10 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis::model_reads::star_flags; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::lineage::column_lineage_graph::ProjectColumnLineage; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::lineage::column_lineage_graph::{ + CompactColumns, CompactModel, ProjectColumnLineage, +}; use crate::bindings::_helpers::lineage::lineage_objects::{LineageClasses, log_lineage_event}; use crate::bindings::_helpers::lineage::project_schema::{ fast_schema, inferred_names, rich_schema, @@ -139,11 +144,18 @@ fn selected_models<'py>( /// One model's FAST lineage work: its compact facts and whether its root star expands. struct FastModel<'py> { name: String, - facts: Option>, + facts: FastFacts<'py>, has_star: bool, expands_star: bool, } +/// A model's lineage facts: none, Python facts, or a compiled model's native facts and edge count. +enum FastFacts<'py> { + Missing, + Python(Bound<'py, PyAny>), + Native(Bound<'py, PyAny>, usize), +} + fn fast_lineage<'py>( classes: &LineageClasses<'py>, project: &Bound<'py, PyAny>, @@ -154,9 +166,22 @@ fn fast_lineage<'py>( let mut work: Vec> = Vec::with_capacity(models.len()); let mut requests: Vec = Vec::new(); for model in models { - let facts: Bound<'py, PyAny> = native_field(model, "fast_lineage_columns")?; let (has_star, star_resolved): (bool, bool) = star_flags(model)?; let expands_star: bool = has_star && !star_resolved; + let native_edges: Option = match model.downcast::() { + Ok(view) if !expands_star => view.get().lineage_edge_count(py)?, + _ => None, + }; + if let Some(edges) = native_edges { + work.push(FastModel { + name: native_field(model, "name")?.extract()?, + facts: FastFacts::Native(model.clone(), edges), + has_star, + expands_star, + }); + continue; + } + let facts: Bound<'py, PyAny> = native_field(model, "fast_lineage_columns")?; if facts.is_none() { requests.push(FastLineageModel::Parse { query_sql: native_field(model, "query_sql")?.extract()?, @@ -170,7 +195,11 @@ fn fast_lineage<'py>( } work.push(FastModel { name: native_field(model, "name")?.extract()?, - facts: (!facts.is_none()).then_some(facts), + facts: if facts.is_none() { + FastFacts::Missing + } else { + FastFacts::Python(facts) + }, has_star, expands_star, }); @@ -196,56 +225,70 @@ fn fast_lineage<'py>( }; let mut outcomes = outcomes.into_iter(); let built: Bound<'py, PyDict> = PyDict::new(py); - let compact: Bound<'py, PyDict> = PyDict::new(py); + let mut compact: HashMap = HashMap::new(); let mut model_order: Vec = Vec::new(); for model in work { - if let Some(mut columns) = model.facts { - if model.expands_star { - let star_columns: Vec = outcomes - .next() - .map(|outcome| outcome.into_parts().1) - .unwrap_or_default(); - let mut expanded: Vec> = - columns.try_iter()?.collect::>()?; - for column in &star_columns { - expanded.push(classes.column_fact(column)?); + let columns: CompactColumns = match model.facts { + FastFacts::Native(view, edges) => CompactColumns::Model { + model: view.unbind(), + edges, + }, + FastFacts::Python(mut columns) => { + if model.expands_star { + let star_columns: Vec = outcomes + .next() + .map(|outcome| outcome.into_parts().1) + .unwrap_or_default(); + let mut expanded: Vec> = + columns.try_iter()?.collect::>()?; + for column in &star_columns { + expanded.push(classes.column_fact(column)?); + } + columns = PyTuple::new(py, expanded)?.into_any(); } - columns = PyTuple::new(py, expanded)?.into_any(); + CompactColumns::Facts(columns.unbind()) } - compact.set_item(&model.name, (columns, model.has_star))?; - model_order.push(model.name); - continue; - } - match outcomes.next() { - Some(FastLineageOutcome::Built { columns, has_star }) => { - let lineage_columns: Vec> = columns - .iter() - .map(|column| classes.column(column, &classes.unknown_nullability)) - .collect::>()?; - built.set_item( - &model.name, - classes.model(&model.name, lineage_columns, has_star)?, - )?; - model_order.push(model.name); - } - Some(FastLineageOutcome::Unparsed(detail)) => log_lineage_event( - py, - "fast column lineage parse failed; falling back", - &model.name, - Some(&detail), - )?, - Some(FastLineageOutcome::UnknownDialect(detail)) => { - return Err(unknown_dialect_error(py, &detail)); + FastFacts::Missing => { + match outcomes.next() { + Some(FastLineageOutcome::Built { columns, has_star }) => { + let lineage_columns: Vec> = columns + .iter() + .map(|column| classes.column(column, &classes.unknown_nullability)) + .collect::>()?; + built.set_item( + &model.name, + classes.model(&model.name, lineage_columns, has_star)?, + )?; + model_order.push(model.name); + } + Some(FastLineageOutcome::Unparsed(detail)) => log_lineage_event( + py, + "fast column lineage parse failed; falling back", + &model.name, + Some(&detail), + )?, + Some(FastLineageOutcome::UnknownDialect(detail)) => { + return Err(unknown_dialect_error(py, &detail)); + } + _ => {} + } + continue; } - _ => {} - } + }; + let _ = compact.insert( + model.name.clone(), + CompactModel { + columns, + has_star: model.has_star, + }, + ); + model_order.push(model.name); } - let fields: Bound<'py, PyDict> = PyDict::new(py); - fields.set_item("models", built)?; - fields.set_item("compact_models", compact)?; - fields.set_item("model_order", PyTuple::new(py, model_order)?)?; - py.get_type::() - .call_method("from_fast_facts", (), Some(&fields)) + Bound::new( + py, + ProjectColumnLineage::compacted(classes, built.as_any(), compact, model_order)?, + ) + .map(Bound::into_any) } /// The output column names of compact facts or plain column facts. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs index c08d164649..926ac60583 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_selection/unit_tests.rs @@ -10,9 +10,10 @@ use sqlbuild_analysis::planner_selection::models::{ SelectorGroups, UnitTestSelectorNames, UnitTestSelectorSplit, }; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::planner_selection::carrier::{Carrier, py_strings}; use crate::bindings::_helpers::records::construct::new_record_with; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use crate::bindings::_helpers::records::ids::RecordId; const RESOURCE_ENTRY_FILES: [(&str, &str); 3] = [ @@ -34,7 +35,7 @@ const RESOURCE_NODES: [&str; 4] = [ ]; fn stem(path: &Bound<'_, PyAny>) -> PyResult { - native_field(path, "stem")?.extract() + path_stem(path) } fn unit_test_names(discovered: &Bound<'_, PyAny>) -> PyResult> { @@ -57,7 +58,11 @@ fn resource_names(discovered: &Bound<'_, PyAny>) -> PyResult> { let mut names: BTreeSet = BTreeSet::new(); for field in RESOURCE_PATH_FILES { for file in native_field(discovered, field)?.try_iter()? { - names.insert(stem(&native_field(&file?, "file_path")?)?); + let file: Bound<'_, PyAny> = file?; + names.insert(match file.cast::() { + Ok(native) => native.get().path_texts(file.py())?.name, + Err(_) => stem(&native_field(&file, "file_path")?)?, + }); } } for (field, entries) in RESOURCE_ENTRY_FILES { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index e2c2c9a375..effacd0eb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -1,15 +1,20 @@ //! What each model, seed, source and table function declares about its columns before analysis. +use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods}; +use std::collections::HashMap; + +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::model_input_view::{ attached_columns, attached_schema, }; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; -use crate::bindings::_helpers::records::construct::{enum_member, record_class}; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::compiled_project::source_views::{AttachedSource, read_source_view}; +use crate::bindings::_helpers::records::construct::enum_member; +use crate::bindings::_helpers::records::fields::{enum_member_value, field as native_field}; use crate::bindings::_helpers::records::ids::RecordId; const UNKNOWN_TYPE: &str = "UNKNOWN"; @@ -19,10 +24,152 @@ const CONTRACT_KEY: &str = "contract"; const TABLE_FUNCTION_PREFIX: &str = "__sqlbuild_table_function_"; const NON_NULL_MEMBER: &str = "NON_NULL"; const UNKNOWN_MEMBER: &str = "UNKNOWN"; +/// `str(None)`, the text of an exact declaration without a type. +const NONE_TEXT: &str = "None"; /// The three per-relation declaration maps the analysis starts from. -pub(crate) type DeclaredRelations<'py> = - (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); +pub(crate) type DeclaredRelations<'py> = ( + Bound<'py, PyDict>, + Bound<'py, PyDict>, + Bound<'py, PyDict>, + DeclaredShapes, +); + +/// Relation shapes as rows in a dict's insertion order, each value as `str(value)`. +#[derive(Debug, Clone, Default)] +pub(crate) struct ShapeRows { + rows: Shapes, + positions: HashMap, +} + +impl ShapeRows { + /// `shapes[name] = pairs`, keeping an existing name's position as a dict does. + pub(crate) fn set(&mut self, name: String, pairs: Pairs) { + match self.positions.get(&name) { + Some(&position) => self.rows[position].1 = pairs, + None => { + let _ = self.positions.insert(name.clone(), self.rows.len()); + self.rows.push((name, pairs)); + } + } + } + + /// The rows in insertion order. + pub(crate) fn rows(&self) -> &Shapes { + &self.rows + } +} + +/// `(column, value)` pairs as a dict keeps them: each column once, at its first position, last value. +pub(crate) fn dict_pairs(items: impl IntoIterator) -> Pairs { + let mut pairs: Pairs = Vec::new(); + let mut positions: HashMap = HashMap::new(); + for (column, value) in items { + match positions.get(&column) { + Some(&position) => pairs[position].1 = value, + None => { + let _ = positions.insert(column.clone(), pairs.len()); + pairs.push((column, value)); + } + } + } + pairs +} + +/// The native rows of the declared types, nullability and authoritative schemas. +#[derive(Debug, Clone, Default)] +pub(crate) struct DeclaredShapes { + pub(crate) types: ShapeRows, + pub(crate) nullability: ShapeRows, + pub(crate) complete: ShapeRows, +} + +/// The Python declaration maps being built: declared types, nullability and authoritative schemas. +type DeclaredMaps<'py> = (Bound<'py, PyDict>, Bound<'py, PyDict>, Bound<'py, PyDict>); + +/// The native rows of the maps being built and the nullability texts they hold. +struct Declared { + shapes: DeclaredShapes, + non_null_text: &'static str, + unknown_text: &'static str, +} + +impl Declared { + fn new() -> PyResult { + let text = |member: &str| -> PyResult<&'static str> { + enum_member_value(RecordId::InferredNullability.name(), member).ok_or_else(|| { + PyRuntimeError::new_err(format!("InferredNullability has no member {member}")) + }) + }; + Ok(Self { + shapes: DeclaredShapes::default(), + non_null_text: text(NON_NULL_MEMBER)?, + unknown_text: text(UNKNOWN_MEMBER)?, + }) + } + + /// Record `name`'s declared types and declared nullability. + fn declare<'py>( + &mut self, + maps: &DeclaredMaps<'py>, + name: &Bound<'py, PyAny>, + columns: &[SchemaColumnFact], + ) -> PyResult<()> { + let py: Python<'py> = name.py(); + let key: String = name.extract()?; + maps.0.set_item(name, column_types(py, columns, false)?)?; + self.shapes + .types + .set(key.clone(), type_pairs(columns, false)); + maps.1.set_item(name, column_nullability(py, columns)?)?; + let pairs: Pairs = dict_pairs(columns.iter().map(|column| { + let text: &str = if non_null(column) { + self.non_null_text + } else { + self.unknown_text + }; + (column.name.clone(), text.to_owned()) + })); + self.shapes.nullability.set(key, pairs); + Ok(()) + } + + /// Record `name`'s declared types as an authoritative schema. + fn complete<'py>( + &mut self, + maps: &DeclaredMaps<'py>, + name: &Bound<'py, PyAny>, + (columns, exact): (&[SchemaColumnFact], bool), + ) -> PyResult<()> { + maps.2 + .set_item(name, column_types(name.py(), columns, exact)?)?; + self.shapes + .complete + .set(name.extract()?, type_pairs(columns, exact)); + Ok(()) + } +} + +/// Each column's declared type text as `str` reads the declared types map. +fn type_pairs(columns: &[SchemaColumnFact], exact: bool) -> Pairs { + dict_pairs(columns.iter().map(|column| { + let text: &str = match column.data_type.as_deref() { + Some(declared) if exact || !declared.is_empty() => declared, + None if exact => NONE_TEXT, + _ => UNKNOWN_TYPE, + }; + (column.name.clone(), text.to_owned()) + })) +} + +/// Whether a column is declared non-null: `nullable` false or a `not_null` audit. +fn non_null(column: &SchemaColumnFact) -> bool { + column.nullable == Some(false) + || column + .audit_definitions + .iter() + .any(|definition| definition == NOT_NULL_AUDIT_NAME) +} /// `{column.name: column.type or "UNKNOWN"}`, or the declared type unchanged when `exact`. fn column_types<'py>( @@ -45,19 +192,16 @@ fn column_types<'py>( /// Each column's declared nullability: non-null when `nullable` is false or a `not_null` audit. fn column_nullability<'py>( + py: Python<'py>, columns: &[SchemaColumnFact], - nullability: &Bound<'py, PyAny>, ) -> PyResult> { - let facts: Bound<'py, PyDict> = PyDict::new(nullability.py()); - let non_null_member: Bound<'py, PyAny> = nullability.getattr(NON_NULL_MEMBER)?; - let unknown_member: Bound<'py, PyAny> = nullability.getattr(UNKNOWN_MEMBER)?; + let facts: Bound<'py, PyDict> = PyDict::new(py); + let non_null_member: Bound<'py, PyAny> = + enum_member(py, RecordId::InferredNullability, NON_NULL_MEMBER)?; + let unknown_member: Bound<'py, PyAny> = + enum_member(py, RecordId::InferredNullability, UNKNOWN_MEMBER)?; for column in columns { - let non_null: bool = column.nullable == Some(false) - || column - .audit_definitions - .iter() - .any(|definition| definition == NOT_NULL_AUDIT_NAME); - let state: &Bound<'py, PyAny> = if non_null { + let state: &Bound<'py, PyAny> = if non_null(column) { &non_null_member } else { &unknown_member @@ -72,24 +216,35 @@ fn table_function_name(function: &Bound<'_, PyAny>) -> PyResult { Ok(format!("{TABLE_FUNCTION_PREFIX}{name}")) } +/// Record one source's declared types and nullability, and its schema when its contract is enforced. +fn declare_source<'py>( + (declared, maps): (&mut Declared, &DeclaredMaps<'py>), + (name, columns, enforced): (&Bound<'py, PyAny>, &[SchemaColumnFact], bool), +) -> PyResult<()> { + declared.declare(maps, name, columns)?; + if enforced && !columns.is_empty() { + declared.complete(maps, name, (columns, false))?; + } + Ok(()) +} + /// Declared column types, declared nullability and the authoritative binding schemas. pub(crate) fn declared_relations<'py>( inputs: Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = inputs.py(); - let nullability: Bound<'py, PyAny> = record_class(py, RecordId::InferredNullability)?; - let types: Bound<'py, PyDict> = PyDict::new(py); - let nulls: Bound<'py, PyDict> = PyDict::new(py); - let complete: Bound<'py, PyDict> = PyDict::new(py); + let maps: DeclaredMaps<'py> = (PyDict::new(py), PyDict::new(py), PyDict::new(py)); + let mut declared: Declared = Declared::new()?; for model in native_field(&inputs, "model_inputs")?.try_iter()? { let model: Bound<'py, PyAny> = model?; let Some(schema) = attached_schema(&model)? else { continue; }; - let (name, columns): (String, Vec) = - (schema.name, attached_columns(&model)?); - types.set_item(&name, column_types(py, &columns, false)?)?; - nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; + let (name, columns): (Bound<'py, PyAny>, Vec) = ( + PyString::new(py, &schema.name).into_any(), + attached_columns(&model)?, + ); + declared.declare(&maps, &name, &columns)?; let contract: Option> = native_field(&native_field(&model, "config")?, "values")? .downcast_into::()? @@ -99,7 +254,7 @@ pub(crate) fn declared_relations<'py>( None => false, }; if enforced && !columns.is_empty() && !schema.dynamic_columns.is_truthy()? { - complete.set_item(&name, column_types(py, &columns, false)?)?; + declared.complete(&maps, &name, (&columns, false))?; } } let mut seeds: Vec<(Bound<'py, PyAny>, Vec)> = Vec::new(); @@ -108,24 +263,34 @@ pub(crate) fn declared_relations<'py>( let name: Bound<'py, PyAny> = native_field(&entry, "name")?; let columns: Vec = python_column_facts(&native_field(&entry, "columns")?, None)?; - types.set_item(&name, column_types(py, &columns, false)?)?; - nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; + declared.declare(&maps, &name, &columns)?; seeds.push((name, columns)); } for source in native_field(&inputs, "source_inputs")?.try_iter()? { - let entry: Bound<'py, PyAny> = native_field(&source?, "source_entry")?; + let source: Bound<'py, PyAny> = source?; + let native = read_source_view(&source, |record, py| { + let attached: &AttachedSource = record.attached(); + let name: Bound<'py, PyAny> = PyString::new(py, &attached.name).into_any(); + let enforced: bool = attached.contract.as_deref() == Some(ENFORCED_CONTRACT); + declare_source( + (&mut declared, &maps), + (&name, &record.key_facts(py)?.columns, enforced), + ) + }); + if let Some(outcome) = native { + outcome?; + continue; + } + let entry: Bound<'py, PyAny> = native_field(&source, "source_entry")?; let name: Bound<'py, PyAny> = native_field(&entry, "name")?; let columns: Vec = python_column_facts(&native_field(&entry, "columns")?, None)?; - types.set_item(&name, column_types(py, &columns, false)?)?; - nulls.set_item(&name, column_nullability(&columns, &nullability)?)?; - if entry.getattr(CONTRACT_KEY)?.eq(ENFORCED_CONTRACT)? && !columns.is_empty() { - complete.set_item(&name, column_types(py, &columns, false)?)?; - } + let enforced: bool = entry.getattr(CONTRACT_KEY)?.eq(ENFORCED_CONTRACT)?; + declare_source((&mut declared, &maps), (&name, &columns, enforced))?; } for (name, columns) in seeds { if !columns.is_empty() { - complete.set_item(name, column_types(py, &columns, false)?)?; + declared.complete(&maps, &name, (&columns, false))?; } } for function in native_field(&inputs, "sql_function_inputs")?.try_iter()? { @@ -135,8 +300,13 @@ pub(crate) fn declared_relations<'py>( if columns.is_empty() { continue; } - let name: String = table_function_name(&function)?; - types.set_item(&name, column_types(py, &columns, true)?)?; + let name: Bound<'py, PyAny> = + PyString::new(py, &table_function_name(&function)?).into_any(); + maps.0.set_item(&name, column_types(py, &columns, true)?)?; + declared + .shapes + .types + .set(name.extract()?, type_pairs(&columns, true)); let unknown: Bound<'py, PyDict> = PyDict::new(py); for column in &columns { unknown.set_item( @@ -144,8 +314,18 @@ pub(crate) fn declared_relations<'py>( enum_member(py, RecordId::InferredNullability, UNKNOWN_MEMBER)?, )?; } - nulls.set_item(&name, unknown)?; - complete.set_item(&name, column_types(py, &columns, true)?)?; + maps.1.set_item(&name, unknown)?; + let unknown_text: &str = declared.unknown_text; + declared.shapes.nullability.set( + name.extract()?, + dict_pairs( + columns + .iter() + .map(|column| (column.name.clone(), unknown_text.to_owned())), + ), + ); + declared.complete(&maps, &name, (&columns, true))?; } - Ok((types, nulls, complete)) + let (types, nulls, complete) = maps; + Ok((types, nulls, complete, declared.shapes)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index d537996df5..65b19fa0c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -165,6 +165,8 @@ pub(crate) const RECORDS: &[&[RecordSpec]] = &[ CONTRACT_RECORDS, ]; +/// `str(PurePath("."))`, the path whose name and stem are empty. +pub(crate) const CURRENT_DIRECTORY_PATH: &str = "."; /// Records whose bound values are normalized or validated before the record is created. pub(crate) const RECORD_FINISHERS: &[&[(&str, RecordFinish)]] = &[ &[("SourceFreshnessObservation", normalize_observation)], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index a3001a18c4..f1f5086168 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -13,20 +13,22 @@ use pyo3::types::{ PyType, PyTypeMethods, }; +use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::constants::{ - CACHED_PROPERTIES, CLASS_PROPERTIES, ENUMS, FROZEN_INSTANCE_ERROR, NATIVE_MODULE, - ORDERED_RECORDS, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, + CACHED_PROPERTIES, CLASS_METHODS, CLASS_PROPERTIES, CURRENT_DIRECTORY_PATH, ENUMS, + FROZEN_INSTANCE_ERROR, NATIVE_MODULE, ORDERED_RECORDS, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, + RECORDS, }; use crate::bindings::_helpers::records::construct::{enum_member, new_record}; use crate::bindings::_helpers::records::getters::getter_field; use crate::bindings::_helpers::records::models::{ EnumSpec, FieldDefault, NativeRecord, RecordSpec, }; -use crate::bindings::_helpers::records::types::ClassProperty; +use crate::bindings::_helpers::records::types::{ClassProperty, RecordFinish}; /// How a test run treats `field` falling back to `getattr` on a native class: unset outside tests. pub(crate) const NATIVE_FIELD_STRICT_ENV_VAR: &str = "SQLBUILD_NATIVE_FIELD_STRICT"; @@ -38,13 +40,31 @@ const ENUM_NAME_ATTRIBUTE: &str = "name"; /// Native string enums by class object, for `.value` and `.name` reads answered from their members. static ENUM_CLASSES: PyOnceLock> = PyOnceLock::new(); +/// Each registered record class's spec, by class object. +static REGISTERED_SPECS: PyOnceLock> = PyOnceLock::new(); + /// Every registered record spec, in registration order. pub(crate) fn all_specs() -> impl Iterator { RECORDS.iter().flat_map(|group| group.iter()) } +/// Keep each registered record class's spec, so records of that exact class find it natively. +pub(crate) fn register_specs( + py: Python<'_>, + classes: HashMap, +) -> PyResult<()> { + REGISTERED_SPECS.get_or_init(py, || classes); + Ok(()) +} + /// The spec registered for `cls`, read from its class attribute. pub(crate) fn record_spec(cls: &Bound<'_, PyType>) -> PyResult<&'static RecordSpec> { + if let Some(spec) = REGISTERED_SPECS + .get(cls.py()) + .and_then(|specs| specs.get(&(cls.as_ptr() as usize))) + { + return Ok(spec); + } let Ok(index) = cls.getattr(RECORD_SPEC_ATTRIBUTE) else { return Err(PyTypeError::new_err(format!( "{} cannot be instantiated directly", @@ -84,6 +104,43 @@ pub(crate) fn field<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult( + value: &Bound<'py, PyAny>, + name: &str, +) -> Option>>> { + let Ok(cell) = value.cast::() else { + return None; + }; + if name.starts_with('_') { + return None; + } + let record: &NativeRecord = cell.get(); + if let Some(found) = record + .spec + .fields + .iter() + .position(|field| field.name == name) + .and_then(|index| record.values.get(index)) + { + return Some(Ok(Some(found.bind(value.py()).clone()))); + } + match record_properties().get(&(record.spec.name, name)) { + Some(&(property, cached_name)) => Some( + property_value(cell, property, cached_name) + .map(|found| Some(found.into_bound(value.py()))), + ), + None if CLASS_METHODS + .iter() + .flat_map(|group| group.iter()) + .any(|(class, method, _)| *class == record.spec.name && *method == name) => + { + None + } + None => Some(Ok(None)), + } +} + /// A strict-mode setting: record fallbacks to a file, or fail on any fallback missing from an allowlist. enum StrictFields { Record(String, Mutex>), @@ -373,16 +430,24 @@ pub(crate) fn build_values( }; values.push(value.unbind()); } - match RECORD_FINISHERS - .iter() - .flat_map(|group| group.iter()) - .find(|(record, _)| *record == name) - { - Some((_, finish)) => finish(py, values), + match record_finishers().get(name) { + Some(finish) => finish(py, values), None => Ok(values), } } +/// Each finished record's finisher, by class name. +fn record_finishers() -> &'static HashMap<&'static str, RecordFinish> { + static FINISHERS: OnceLock> = OnceLock::new(); + FINISHERS.get_or_init(|| { + RECORD_FINISHERS + .iter() + .flat_map(|group| group.iter()) + .map(|&(record, finish)| (record, finish)) + .collect() + }) +} + fn default_value<'py>( py: Python<'py>, record: &str, @@ -417,7 +482,7 @@ pub(crate) fn compared_values<'py>( record: &Bound<'py, NativeRecord>, ) -> PyResult> { let py: Python<'py> = record.py(); - let spec: &RecordSpec = record_spec(&record.get_type())?; + let spec: &RecordSpec = record.get().spec; let values = spec .fields .iter() @@ -432,7 +497,7 @@ pub(crate) fn compared_values<'py>( pub(crate) fn repr_text(record: &Bound<'_, NativeRecord>) -> PyResult { let py: Python<'_> = record.py(); let cls: Bound<'_, PyType> = record.get_type(); - let spec: &RecordSpec = record_spec(&cls)?; + let spec: &RecordSpec = record.get().spec; let mut parts: Vec = Vec::with_capacity(spec.fields.len()); for (field, value) in spec.fields.iter().zip(record.get().values.iter()) { if field.shown { @@ -460,7 +525,7 @@ pub(crate) fn ordered_comparison( op: CompareOp, ) -> PyResult> { let py: Python<'_> = slf.py(); - let name: &str = record_spec(&slf.get_type())?.name; + let name: &str = slf.get().spec.name; let ordered: bool = ORDERED_RECORDS.iter().any(|group| group.contains(&name)); if !ordered || !other.get_type().is(slf.get_type()) { return Ok(py.NotImplemented()); @@ -512,3 +577,30 @@ pub(crate) fn path_posix(path: &Bound<'_, PyAny>) -> PyResult { let text: String = path.str()?.to_string_lossy().into_owned(); Ok(String::from_utf8_lossy(&pure_path_posix(std::path::Path::new(&text))).into_owned()) } + +/// The value of the native string enum `enum_name`'s member `member`. +pub(crate) fn enum_member_value(enum_name: &str, member: &str) -> Option<&'static str> { + ENUMS + .iter() + .flat_map(|group| group.iter()) + .find(|spec| spec.name == enum_name)? + .members + .iter() + .find(|(name, _)| *name == member) + .map(|(_, value)| *value) +} + +/// `path.stem` of a `pathlib` path, spelled natively from its text. +pub(crate) fn path_stem(path: &Bound<'_, PyAny>) -> PyResult { + let posix: String = path_posix_text(path)?; + if posix == CURRENT_DIRECTORY_PATH { + return Ok(String::new()); + } + Ok(model_file_stem(&posix).to_owned()) +} + +/// `path.as_posix()` of a `pathlib` path, its text with the platform separator spelled `/`. +pub(crate) fn path_posix_text(path: &Bound<'_, PyAny>) -> PyResult { + let text: Bound<'_, PyString> = path.str()?; + Ok(text.to_str()?.replace(std::path::MAIN_SEPARATOR, "/")) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs index 4f69782b9d..7fcecb19ff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/registration.rs @@ -13,8 +13,10 @@ use crate::bindings::_helpers::records::constants::{ use crate::bindings::_helpers::records::error_classes::{ BoundErrorInitializer, ErrorInitializer, class_value, }; -use crate::bindings::_helpers::records::fields::all_specs; +use std::collections::HashMap; + use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::{all_specs, register_specs}; use crate::bindings::_helpers::records::models::{ BoundNativeMethod, EnumSpec, ErrorSpec, NativeMethod, NativeProperty, NativeRecord, RecordAttribute, RecordSpec, @@ -47,14 +49,17 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add(spec.name, error_class(&parent, (index, spec))?)?; } let dataclasses: Bound<'_, PyModule> = py.import(DATACLASSES_MODULE)?; + let mut registered: HashMap = HashMap::new(); for (index, spec) in all_specs().enumerate() { let class: Bound<'_, PyAny> = record_class(&metaclass, &base, (index, spec))?; + let _ = registered.insert(class.as_ptr() as usize, spec); class.setattr( DATACLASS_FIELDS_ATTRIBUTE, dataclass_fields(&dataclasses, spec)?, )?; module.add(spec.name, class)?; } + register_specs(py, registered)?; for (class, name, property) in CLASS_PROPERTIES.iter().flat_map(|group| group.iter()) { let descriptor: Py = Py::new( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index 06568f07bc..f1bf047ba8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -34,7 +34,7 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, }; use crate::bindings::_helpers::rules::relation_keys::{ - RelationKeyRow, RelationKeys, relation_keys, + RelationKeyRow, RelationKeys, relation_keys, source_key_input, }; use crate::bindings::_helpers::rules::rule_cache::stored_entries; use crate::bindings::_helpers::rules::rules_config::rules_config_of; @@ -122,7 +122,7 @@ fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult> = Vec::new(); for source in record_field(inputs, "source_inputs")?.try_iter()? { - sources.push(record_field(&source?, "source_entry")?); + sources.push(source_key_input(&source?)?); } let keys: RelationKeys = relation_keys( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index d5f429d621..989f88060c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -10,6 +10,7 @@ use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; +use crate::bindings::_helpers::compiled_project::source_views::{SourceKeyFacts, read_source_view}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::lint_inputs::model_key_row; @@ -52,19 +53,7 @@ pub(crate) fn relation_key_row( column.nullable == Some(false), )); } - for audit in declared_audits.try_iter()? { - let audit: Bound<'_, PyAny> = audit?; - let argument: Option> = native_field(&audit, "arguments")? - .downcast::()? - .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; - audits.push(( - native_field(&audit, "definition_name")?.extract()?, - argument - .filter(pyo3::types::PyAnyMethods::is_instance_of::) - .map(|value| value.extract()) - .transpose()?, - )); - } + audits = audit_key_rows(declared_audits)?; } Ok(( kind.to_owned(), @@ -81,6 +70,61 @@ pub(crate) fn relation_key_row( )) } +/// Each audit spec's `(definition name, column argument when it is text)`. +pub(crate) fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult)>> { + let mut rows: Vec<(String, Option)> = Vec::new(); + for audit in audits.try_iter()? { + let audit: Bound<'_, PyAny> = audit?; + let argument: Option> = native_field(&audit, "arguments")? + .downcast::()? + .get_item(UNIQUE_AUDIT_COLUMN_ARGUMENT)?; + rows.push(( + native_field(&audit, "definition_name")?.extract()?, + argument + .filter(pyo3::types::PyAnyMethods::is_instance_of::) + .map(|value| value.extract()) + .transpose()?, + )); + } + Ok(rows) +} + +/// A source's key row from its native view's declared facts, or `None` for any other object. +pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option> { + read_source_view(view, |source, py| { + let facts: &SourceKeyFacts = source.key_facts(py)?; + Ok(( + RELATION_KIND_SOURCE.to_owned(), + source.attached().name.clone(), + String::new(), + String::new(), + String::new(), + false, + None, + facts + .columns + .iter() + .map(|column| { + ( + column.name.clone(), + column.audit_definitions.clone(), + column.nullable == Some(false), + ) + }) + .collect(), + facts.audits.clone(), + )) + }) +} + +/// What `relation_keys` reads for a source: its native view itself, or its `source_entry`. +pub(crate) fn source_key_input<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + if read_source_view(source, |_, _| Ok(())).is_some() { + return Ok(source.clone()); + } + native_field(source, "source_entry") +} + /// `str(values.get(key, ""))`. fn config_text(values: &Bound<'_, PyDict>, key: &str) -> PyResult { match values.get_item(key)? { @@ -149,6 +193,10 @@ pub(crate) fn relation_keys<'py>( for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { for entry in entries.try_iter()? { let entry: Bound<'py, PyAny> = entry?; + if let Some(row) = source_view_key_row(&entry) { + rows.push(row?); + continue; + } let name: String = native_field(&entry, "name")?.extract()?; rows.push(entry_relation_key_row( kind, @@ -201,7 +249,7 @@ pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult> = Vec::new(); for source in native_field(project, "sources")?.try_iter()? { - sources.push(native_field(&source?, "source_entry")?); + sources.push(source_key_input(&source?)?); } relation_keys( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs index 334bfbf364..84db425ada 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs @@ -20,9 +20,10 @@ use sqlbuild_scopes::scope_index::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::construct::error_class; -use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_values; +use crate::bindings::_helpers::records::fields::{field as native_field, path_stem}; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::scopes::relationship_scan::{ RelationshipScanSettings, ScenarioRow, TestFileRow, scanned_relationships, @@ -94,7 +95,7 @@ fn optional_text(item: &Bound<'_, PyAny>, name: &str) -> PyResult } fn stem(path: &Bound<'_, PyAny>) -> PyResult { - native_field(path, "stem")?.extract() + path_stem(path) } /// `value or fallback` with Python truthiness. @@ -109,6 +110,14 @@ fn or_else<'py>( } } +/// `value` when it is a non-empty string. +fn header_text(value: &Bound<'_, PyAny>) -> Option { + value + .extract::() + .into_iter() + .find(|name| !name.is_empty()) +} + /// `value if isinstance(value, str) and value else path.stem`. fn name_or_stem(value: &Bound<'_, PyAny>, path: &Bound<'_, PyAny>) -> PyResult { if let Ok(name) = value.extract::() @@ -144,6 +153,16 @@ impl<'py> Walk<'py> { kind: ResourceKind, name: String, location: (&Bound<'py, PyAny>, ResourceRoot), + ) -> PyResult<()> { + let (path, root) = location; + self.add_resource_at(kind, name, (text(path)?, root)) + } + + fn add_resource_at( + &mut self, + kind: ResourceKind, + name: String, + location: (String, ResourceRoot), ) -> PyResult<()> { let (path, root) = location; self.resource_identities @@ -154,7 +173,7 @@ impl<'py> Walk<'py> { self.resource_roots.push((kind, root)); self.inputs.resources.push(ResourceInput { identity: ResourceIdentity { kind, name }, - path: text(path)?, + path, root, }); Ok(()) @@ -289,13 +308,37 @@ impl<'py> Walk<'py> { fn add_models(&mut self, inputs: &Bound<'py, PyAny>) -> PyResult<()> { for model_file in items(inputs, "model_files")? { - let path: Bound<'py, PyAny> = native_field(&model_file, "relative_path")?; - let model_name: String = name_or_stem(&header_name(&model_file)?, &path)?; - self.add_resource( + let header: Bound<'py, PyAny> = header_name(&model_file)?; + let (model_name, path, folder): (String, String, String) = + match model_file.cast::() { + Ok(native) => { + let native: &NativeModelFile = native.get(); + let texts: ModelFileTexts = native.path_texts(model_file.py())?; + let name: String = match header_text(&header) { + Some(name) => name, + None => texts.name, + }; + (name, texts.relative_path, native.folder(model_file.py())?) + } + Err(_) => { + let path: Bound<'py, PyAny> = native_field(&model_file, "relative_path")?; + ( + name_or_stem(&header, &path)?, + text(&path)?, + text(&native_field(&path, "parent")?)?, + ) + } + }; + self.add_resource_at( ResourceKind::Model, model_name.clone(), - (&path, ResourceRoot::Kind), + (path.clone(), ResourceRoot::Kind), )?; + let enums: Vec> = items(&model_file, "enum_declarations")?; + let constants: Vec> = items(&model_file, "constant_declarations")?; + if enums.is_empty() && constants.is_empty() { + continue; + } let owner: Bound<'py, PyAny> = self.classes.model(RecordId::ResourceIdentity)?.call1(( self.classes @@ -305,7 +348,7 @@ impl<'py> Walk<'py> { let private: Bound<'py, PyAny> = self .classes .member(RecordId::ScopeKind, ScopeKind::Private.as_str())?; - for declaration in items(&model_file, "enum_declarations")? { + for declaration in enums { let identity: Bound<'py, PyAny> = self.declaration_identity( DeclarationKind::Enum, &native_field(&declaration, "name")?, @@ -313,9 +356,9 @@ impl<'py> Walk<'py> { )?; let metadata = [None, Some(self.enum_metadata(&declaration)?), None]; let values = self.values(identity, private.clone(), metadata); - self.add_private(values, &path)?; + self.add_private(values, (&path, &folder))?; } - for declaration in items(&model_file, "constant_declarations")? { + for declaration in constants { let identity: Bound<'py, PyAny> = self.declaration_identity( DeclarationKind::Constant, &native_field(&declaration, "name")?, @@ -323,7 +366,7 @@ impl<'py> Walk<'py> { )?; let metadata = [None, None, Some(self.constant_metadata(&declaration)?)]; let values = self.values(identity, private.clone(), metadata); - self.add_private(values, &path)?; + self.add_private(values, (&path, &folder))?; } } Ok(()) @@ -332,15 +375,15 @@ impl<'py> Walk<'py> { fn add_private( &mut self, values: DeclarationValues<'py>, - path: &Bound<'py, PyAny>, + (path, folder): (&str, &str), ) -> PyResult<()> { self.add_declaration( values, Placement { - path: text(path)?, + path: path.to_owned(), ownership_root: Some(PRIVATE_ROOT.to_owned()), root_fallback: PRIVATE_ROOT.to_owned(), - owning_path: Some(text(&native_field(path, "parent")?)?), + owning_path: Some(folder.to_owned()), }, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 08bd61504e..6ea7f7db18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -13,7 +13,10 @@ use crate::bindings::_helpers::analysis::model_reads::{ use crate::bindings::_helpers::analysis::normalization::normalize_analysis_sql; use crate::bindings::_helpers::analysis_session::stage_functions::infer_native_expression_source_shapes; use crate::bindings::_helpers::boundary::keyword_request::Request; -use crate::bindings::_helpers::compiled_project::compiled_model_view::{DeclaredRow, InferredRow}; +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + DeclaredRow, InferredRow, NativeCompiledModel, +}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::enum_member; @@ -34,11 +37,21 @@ const TABLE_FUNCTION_KIND: &str = "table_fn"; /// The relation names `(kind, name, package, count)` reference rows bind, UDFs excluded. pub(crate) fn relation_row_names<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { - let mut names: Vec = Vec::new(); + let mut pairs: Vec<(String, String)> = Vec::new(); for row in rows.try_iter()? { let row: Bound<'py, PyAny> = row?; - let kind: String = row.get_item(0)?.str()?.extract()?; - let name: String = row.get_item(1)?.extract()?; + pairs.push(( + row.get_item(0)?.str()?.extract()?, + row.get_item(1)?.extract()?, + )); + } + Ok(relation_names(pairs)) +} + +/// The relations `(kind, name)` reference rows bind, each once, table functions by binding name. +fn relation_names(rows: Vec<(String, String)>) -> Vec { + let mut names: Vec = Vec::new(); + for (kind, name) in rows { if kind == UDF_KIND { continue; } @@ -51,7 +64,23 @@ pub(crate) fn relation_row_names<'py>(rows: &Bound<'py, PyAny>) -> PyResult) -> PyResult<(Vec, bool)> { + if let Ok(view) = model.downcast::() { + let rows: Vec = view.get().reference_rows(model.py())?; + let referenced: bool = !rows.is_empty(); + let names: Vec = relation_names( + rows.into_iter() + .map(|(kind, name, _, _)| (kind, name)) + .collect(), + ); + return Ok((names, referenced)); + } + let rows: Bound<'_, PyAny> = native_field(model, "reference_rows")?; + Ok((relation_row_names(&rows)?, rows.is_truthy()?)) } /// `{name: shapes.get(name, {})}` for each relation a model's reference rows bind. @@ -61,7 +90,7 @@ pub(crate) fn model_inputs<'py>( ) -> PyResult> { let py: Python<'py> = model.py(); let inputs: Bound<'py, PyDict> = PyDict::new(py); - for name in relation_row_names(&native_field(model, "reference_rows")?)? { + for name in model_relation_names(model)?.0 { let shape: Bound<'py, PyAny> = match shapes.get_item(&name)? { Some(shape) => shape, None => PyDict::new(py).into_any(), @@ -293,8 +322,7 @@ fn model_shapes<'py>( for model in &pending { let declared: Option> = declared_rows(model)?; let inferred: Option> = inferred_rows(model)?; - let rows: Bound<'py, PyAny> = native_field(model, "reference_rows")?; - let relations: Vec = relation_row_names(&rows)?; + let (relations, referenced): (Vec, bool) = model_relation_names(model)?; let contract: Bound<'py, PyAny> = native_field(&native_field(model, "config")?, "values")? .call_method1("get", ("contract",))?; @@ -317,8 +345,7 @@ fn model_shapes<'py>( } let (has_star, star_resolved): (bool, bool) = star_flags(model)?; let closes: bool = inferred.as_ref().is_some_and(|rows| !rows.is_empty()) - && (!has_star - || (star_resolved && rows.is_truthy()? && all_known(&relations, shapes)?)); + && (!has_star || (star_resolved && referenced && all_known(&relations, shapes)?)); if let (true, Some(inferred)) = (closes, &inferred) { let inputs: Bound<'py, PyDict> = model_inputs(model, shapes)?; let columns: Bound<'py, PyDict> = typed_columns(py, inferred)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index 55b7a7de1f..9dcd6a4f87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -13,7 +13,10 @@ use sqlbuild_analysis::compiler::models::{ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::bindings::_helpers::boundary::panics::value_error; -use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::records::fields::{ + field as native_field, path_posix_text, path_stem, +}; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; use crate::bindings::types::CompilerDetach; @@ -69,6 +72,29 @@ pub(crate) fn assemble_compiled_sql_tests( } fn assembly_model(model: &Bound<'_, PyAny>) -> PyResult { + if let Ok(native) = model.cast::() { + return native.get().read(model.py(), |record, _| { + let unscanned_macro_source = if record.macro_deps.is_empty() { + Some(record.macro_source_sql.to_string()).filter(|sql| sql.contains(MACRO_TOKEN)) + } else { + None + }; + Ok(SqlTestAssemblyModel { + name: record.name.clone(), + macro_deps: record.macro_deps.clone(), + unscanned_macro_source, + references: record + .references + .iter() + .map(|(kind, name, package, _)| SqlTestAssemblyReference { + kind: kind.clone(), + name: name.clone(), + package: package.clone(), + }) + .collect(), + }) + }); + } let macro_deps: Vec = native_field(model, "macro_deps")?.extract()?; let unscanned_macro_source = if macro_deps.is_empty() { Some(native_field(model, "macro_source_sql")?.extract::()?) @@ -123,9 +149,9 @@ fn assembly_test(test: &Bound<'_, PyAny>) -> PyResult { }; Ok(SqlTestAssemblyTest { block_name: native_field(&test_block, "name")?.extract()?, - file_stem: native_field(&native_field(&test_file, "file_path")?, "stem")?.extract()?, - relative_path: relative_path.call_method0("as_posix")?.extract()?, - relative_stem: native_field(&relative_path, "stem")?.extract()?, + file_stem: path_stem(&native_field(&test_file, "file_path")?)?, + relative_path: path_posix_text(&relative_path)?, + relative_stem: path_stem(&relative_path)?, contents: native_field(&test_file, "contents")?.extract()?, block_sql: native_field(&test_block, "sql_body")?.extract()?, block_index: native_field(&test_block, "test_index")?.extract()?, diff --git a/fensu.toml b/fensu.toml index d9bfe33c54..356ddab89e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -607,7 +607,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs=4", @@ -616,7 +616,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_named.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=5", @@ -790,7 +790,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_command.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=17", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/load/ingestr_command.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=17", @@ -929,7 +929,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/style.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_checks.rs=1", @@ -1044,7 +1044,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/coverage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/source_bindings.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/authored_ctes.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/constants.rs=1", @@ -1151,7 +1151,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs=3", @@ -1220,10 +1220,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9fcce9f310..959e55faa1 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4268,6 +4268,7 @@ def build_compile_inputs( resolved_connection: dict[str, object] | None = None, no_cache: bool = False, ) -> CompileInputsView: ... +def defer_compile_teardown_to_exit() -> None: ... def run_compile_command( request: CompileCommandRequest, reuse_attempt: CompileReuseAttempt | None = None ) -> int: ... diff --git a/src/sqlbuild/cli/entry/main/launched.py b/src/sqlbuild/cli/entry/main/launched.py index b544413b74..4925ae9ab4 100644 --- a/src/sqlbuild/cli/entry/main/launched.py +++ b/src/sqlbuild/cli/entry/main/launched.py @@ -11,13 +11,14 @@ def main() -> int: - """Run the CLI with the `sqb` arguments, setting the allocator purge delay first.""" + """Run the CLI with the `sqb` arguments in a process that exits once the command returns.""" os.environ.setdefault( NATIVE_ALLOCATOR_PURGE_DELAY_VARIABLE, NATIVE_ALLOCATOR_PURGE_DELAY_MILLISECONDS ) - from sqlbuild._native import LAUNCH_RECORD_OPTION + from sqlbuild._native import LAUNCH_RECORD_OPTION, defer_compile_teardown_to_exit + _ = defer_compile_teardown_to_exit() arguments: list[str] = sys.argv[1:] offered: bool = bool(arguments) and arguments[0].startswith(LAUNCH_RECORD_OPTION) sys.argv = [sys.argv[0], *arguments[int(offered) :]] diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index 71438db67b..2f6143377e 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -73,3 +73,12 @@ class SelectorCompileTestCase: description: str args: tuple[str, ...] expected_model: bytes + + +@dataclass(frozen=True) +class LaunchedTeardownTestCase: + """One compile command line and the reuse hits its repeated launched run reports.""" + + description: str + args: tuple[str, ...] + expected_hits: int diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index f43dbaea72..e81c78ba3a 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -189,3 +189,13 @@ def startup_install(directory: Path) -> tuple[Path, Path, Path]: startup: Path = site_packages / "catalog.pth" _ = startup.write_text(str(old) + "\n", encoding="utf-8") return program, startup, new + + +def compiled_files(project_dir: Path) -> dict[str, bytes]: + """Every compiled SQL artifact under the project's target, by relative path.""" + + compiled: Path = project_dir / "target" / "compiled" + return { + path.relative_to(compiled).as_posix(): path.read_bytes() + for path in sorted(compiled.rglob("*.sql")) + } diff --git a/tests/integration/src/sqlbuild/cli/entry/test_launched_teardown.py b/tests/integration/src/sqlbuild/cli/entry/test_launched_teardown.py new file mode 100644 index 0000000000..ef18282ea5 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_launched_teardown.py @@ -0,0 +1,48 @@ +import sys +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.entry._test_types import LaunchedTeardownTestCase +from tests.integration.src.sqlbuild.cli.entry.helpers import ( + PYTHON_CLI, + LauncherRun, + compile_timings, + compiled_files, + run, + without_compile_timings, + write_project, +) + +IN_PROCESS_ENTRY: str = ( + "import sys\nfrom sqlbuild.cli.entry.main.entry import main\nsys.exit(main(sys.argv[1:]))\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + LaunchedTeardownTestCase( + description="json report", args=("compile", "--json"), expected_hits=1 + ) + ], + ids=lambda case: case.description, +) +def test_given_launched_cli_when_compiling_then_output_matches_entry_and_cache_is_reused( + test_case: LaunchedTeardownTestCase, tmp_path: Path +) -> None: + launched_dir: Path = write_project(tmp_path / "launched") + entry_dir: Path = write_project(tmp_path / "entry") + + launched: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=launched_dir) + entry: LauncherRun = run( + Path(sys.executable), args=("-c", IN_PROCESS_ENTRY, *test_case.args), cwd=entry_dir + ) + repeated: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=launched_dir) + + assert launched[0] == entry[0] == repeated[0] == 0 + assert without_compile_timings(launched[1]) == without_compile_timings(entry[1]) + assert compiled_files(launched_dir) == compiled_files(entry_dir) + assert compiled_files(launched_dir) + assert compile_timings(repeated[1])["project_reuse_hits"] == test_case.expected_hits + assert without_compile_timings(repeated[1]) == without_compile_timings(launched[1]) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 681d5d2618..bfb3ca0eab 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -85,6 +85,15 @@ class NativeSourceAttachmentTestCase: expected_usages: tuple[str, ...] +@dataclass(frozen=True) +class NativeSourceKeysTestCase: + """Sources whose declared keys native views answer as their attached entries do.""" + + description: str + source_yml: str + expected_keys: dict[str, tuple[tuple[str, ...], ...]] + + @dataclass(frozen=True) class NativeSourceErrorTestCase: """A source the native loop rejects while attaching it.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py index 694f8f2939..409c1e5f39 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_sources.py @@ -11,6 +11,7 @@ from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( NativeSourceAttachmentTestCase, NativeSourceErrorTestCase, + NativeSourceKeysTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( column_metadata, @@ -99,6 +100,44 @@ def test_given_source_when_compiling_then_native_project_holds_its_attached_entr ) +@pytest.mark.parametrize( + "test_case", + [ + NativeSourceKeysTestCase( + description="column and table unique audits with templated column metadata", + source_yml=( + "sources:\n" + " - name: raw_orders\n" + " audits:\n - unique:\n column: order_id\n" + " columns:\n" + " - name: order_id\n type: INTEGER\n nullable: false\n" + " description: Order in ${region}.\n" + " - name: customer_id\n type: INTEGER\n" + " audits:\n - unique\n - not_null\n" + " - name: raw_events\n" + " columns:\n - name: event_id\n type: INTEGER\n" + ), + expected_keys={"source:raw_orders": (("customer_id",), ("order_id",))}, + ), + ], + ids=lambda case: case.description, +) +def test_given_keyed_sources_when_deriving_relation_keys_then_native_views_match_entries( + test_case: NativeSourceKeysTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: CompiledProjectView + project, _ = compile_source_project( + project_dir=tmp_path, source_yml=test_case.source_yml, monkeypatch=monkeypatch + ) + + from_entries: dict[str, tuple[tuple[str, ...], ...]] = _native.declared_relation_keys( + model_rows=(), seeds=(), sources=tuple(source.source_entry for source in project.sources) + ) + compiled: dict[str, tuple[tuple[str, ...], ...]] = _native.compiled_relation_keys(project) + assert from_entries == test_case.expected_keys + assert {identity: compiled[identity] for identity in from_entries} == from_entries + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 17410ca9a1..bad4b27454 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -609,3 +609,51 @@ def staged_then_published( _ = artifacts.planning_diagnostics() before_publish(target) return artifacts.publish(target_dir=target, manifest=None) + + +class FunctionRenderFailingAdapter(DuckDbAdapter): + """A DuckDB adapter whose function rendering fails after the model group is handed off.""" + + def render_create_function( + self, + *, + destination: str, + arguments: tuple[Any, ...], + returns: str, + body_sql: str, + return_columns: tuple[Any, ...] = (), + language: FunctionLanguage = FunctionLanguage.SQL, + runtime_version: str | None = None, + entry_point: str | None = None, + packages: tuple[str, ...] = (), + ) -> tuple[str, ...]: + raise ValueError(f"function rendering failed for {destination}") + + +class FunctionRenderInterruptedAdapter(DuckDbAdapter): + """A DuckDB adapter interrupted while rendering a function after the model group.""" + + def render_create_function( + self, + *, + destination: str, + arguments: tuple[Any, ...], + returns: str, + body_sql: str, + return_columns: tuple[Any, ...] = (), + language: FunctionLanguage = FunctionLanguage.SQL, + runtime_version: str | None = None, + entry_point: str | None = None, + packages: tuple[str, ...] = (), + ) -> tuple[str, ...]: + raise KeyboardInterrupt(destination) + + +class BlockedModelStaging(StagingEveryProject): + """Stage artifacts into a staging tree whose model directory is a file.""" + + def create_staging_directory(self, staging_dir: Path) -> None: + super().create_staging_directory(staging_dir) + blocker: Path = staging_dir / "compiled" / "models" / "staging" + blocker.parent.mkdir(parents=True) + _ = blocker.write_text("not a directory\n", encoding="utf-8") diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 335c170747..e2321f4467 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import time from dataclasses import replace from pathlib import Path @@ -35,6 +36,9 @@ TargetWriterTestCase, ) from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( + BlockedModelStaging, + FunctionRenderFailingAdapter, + FunctionRenderInterruptedAdapter, build_cached_target_writer_project, build_missing_mock_target_writer_project, build_static_target_writer_project, @@ -44,6 +48,7 @@ from tests.unit.src.sqlbuild.compiler.manifest.helpers import build_test_project SQL_TEST_ARTIFACT_PLANS: str = "plan_compiled_sql_test_artifacts" +INTERRUPT_RETURN_SECONDS: float = 10.0 SQL_TEST_MODEL_CHAINS: str = "sql_test_model_chain_names_by_key" @@ -334,6 +339,125 @@ def test_given_invalid_utf8_artifact_when_writing_again_then_existing_decode_err ) +@pytest.mark.parametrize( + "test_case", + [ + ExpectedMessageTestCase( + description="blocked model directory stops later groups", expected_message="staging" + ) + ], + ids=lambda case: case.description, +) +def test_given_unwritable_model_artifact_when_writing_static_target_then_later_groups_are_not_written( + tmp_path: Path, + test_case: ExpectedMessageTestCase, +) -> None: + project: CompiledProjectView = build_static_target_writer_project() + target_dir: Path = tmp_path / "target" + blocker: Path = target_dir / "compiled" / "models" / "staging" + blocker.parent.mkdir(parents=True) + blocker.write_text("not a directory\n", encoding="utf-8") + + with pytest.raises(OSError, match=test_case.expected_message): + write_static_compile_target( + target_dir=target_dir, + adapter=DuckDbAdapter(), + project=project, + ) + + assert blocker.read_text(encoding="utf-8") == "not a directory\n" + assert not (target_dir / "compiled" / "functions").exists() + assert not (target_dir / "compiled" / "audits").exists() + assert not (target_dir / "compiled" / "tests").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedMessageTestCase( + description="the model write error wins over a later rendering error", + expected_message="staging", + ) + ], + ids=lambda case: case.description, +) +def test_given_model_write_failure_and_later_error_when_writing_static_target_then_os_error_wins( + tmp_path: Path, + test_case: ExpectedMessageTestCase, +) -> None: + target_dir: Path = tmp_path / "target" + blocker: Path = target_dir / "compiled" / "models" / "staging" + blocker.parent.mkdir(parents=True) + blocker.write_text("not a directory\n", encoding="utf-8") + + with pytest.raises(OSError, match=test_case.expected_message): + write_static_compile_target( + target_dir=target_dir, + adapter=FunctionRenderFailingAdapter(), + project=build_static_target_writer_project(), + ) + + assert not (target_dir / "compiled" / "functions").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedMessageTestCase( + description="interrupt after the model group", expected_message="compiled/functions" + ) + ], + ids=lambda case: case.description, +) +def test_given_interrupt_after_model_group_when_writing_static_target_then_later_groups_are_not_written( + tmp_path: Path, + test_case: ExpectedMessageTestCase, +) -> None: + target_dir: Path = tmp_path / "target" + started: float = time.monotonic() + + with pytest.raises(KeyboardInterrupt): + write_static_compile_target( + target_dir=target_dir, + adapter=FunctionRenderInterruptedAdapter(), + project=build_static_target_writer_project(), + ) + + assert time.monotonic() - started < INTERRUPT_RETURN_SECONDS + assert not (target_dir / test_case.expected_message).exists() + assert not (target_dir / "compiled" / "audits").exists() + assert not (target_dir / "compiled" / "tests").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedMessageTestCase( + description="staged model write failure falls back despite a later error", + expected_message="compiled/models/staging/orders.sql", + ) + ], + ids=lambda case: case.description, +) +def test_given_staged_model_write_failure_and_later_error_when_publishing_then_staging_falls_back( + tmp_path: Path, + test_case: ExpectedMessageTestCase, +) -> None: + target_dir: Path = tmp_path / "target" + + with BlockedModelStaging(enabled=True) as artifacts: + artifacts.start( + project=build_static_target_writer_project(), + adapter=FunctionRenderFailingAdapter(), + target_dir=target_dir, + stage_artifacts=True, + ) + written: WrittenTarget | None = artifacts.publish(target_dir=target_dir, manifest=None) + + assert written is None + assert not (target_dir / test_case.expected_message).exists() + + @pytest.mark.parametrize( "test_case", (TargetWriterCacheTestCase(description="unchanged artifact reuse", expected_builder_calls=0),),