diff --git a/Cargo.lock b/Cargo.lock index 17d744c8d0..c795b3af03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1263,6 +1263,7 @@ dependencies = [ "fensu-policy", "mimalloc", "pyo3", + "rayon", "serde", "serde_json", "sqlbuild-analysis", @@ -1311,6 +1312,7 @@ dependencies = [ name = "sqlbuild-rules" version = "0.157.0" dependencies = [ + "blake3", "fensu-policy", "globset", "polyglot-sql", diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs index 3092851426..9a274defc7 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs @@ -165,6 +165,7 @@ impl SessionCatalog { let mut shared: Vec> = self.shared_queries(batch, &cleaned)?; self.shared_members += shared.iter().flatten().count(); let mut query_digests: HashMap<&str, MemoKey> = HashMap::new(); + let batch_digest: MemoKey = batch_key_digest(batch); let keys: Vec> = batch .members .iter() @@ -174,7 +175,12 @@ impl SessionCatalog { let query: MemoKey = *query_digests .entry(share.key.as_str()) .or_insert_with(|| Sha256::digest(share.key.as_bytes()).into()); - Some(shared_result_key(batch, member, share, &query)) + Some(shared_result_key( + batch, + member, + share, + (&query, &batch_digest), + )) }) .collect(); for (member, sql) in batch.members.iter().zip(&cleaned) { @@ -609,21 +615,23 @@ impl SessionCatalog { } }; query_indexes.push(query_index); - let template: Value = json!({ - "queryIndex": query_index, - "references": lineage_resources(member, share.as_ref()), - "functionReturnTypes": function_return_types, - "declaredColumnOrder": declared_column_order(member, batch.nullability), - "recoverCteFacts": member.recover_cte_facts, - "richTypeInference": batch.rich_type_inference, - }); - let template_index: usize = - *template_keys - .entry(template.to_string()) - .or_insert_with(|| { - templates.push(template); - templates.len() - 1 - }); + let references: Map = lineage_resources(member, share.as_ref()); + let declared: Value = json!(declared_column_order(member, batch.nullability)); + let member_template_identity: String = + json!([query_index, references, declared, member.recover_cte_facts]).to_string(); + let template_index: usize = *template_keys + .entry(member_template_identity) + .or_insert_with(|| { + templates.push(json!({ + "queryIndex": query_index, + "references": references, + "functionReturnTypes": function_return_types, + "declaredColumnOrder": declared, + "recoverCteFacts": member.recover_cte_facts, + "richTypeInference": batch.rich_type_inference, + })); + templates.len() - 1 + }); let resource_names: Map = member .lineage_references .iter() @@ -897,12 +905,19 @@ fn named_sources( .collect() } -/// Digest of Python's `shared_result_keys` entry, given its shared query's digest. +/// Digest of the settings every member of `batch` shares, hashed once per batch. +fn batch_key_digest(batch: &Batch<'_>) -> MemoKey { + let shared: String = + json!([batch.rich_type_inference, batch.function_return_types]).to_string(); + Sha256::digest(shared.as_bytes()).into() +} + +/// Digest of Python's `shared_result_keys` entry, given its shared query's and batch's digests. fn shared_result_key( batch: &Batch<'_>, member: &BatchMember<'_>, share: &SharedQuery, - query: &MemoKey, + (query, batch_digest): (&MemoKey, &MemoKey), ) -> MemoKey { let reference_types: Vec<(&str, &str)> = member .lineage_references @@ -911,14 +926,13 @@ fn shared_result_key( .collect(); let rest: String = json!([ member.recover_cte_facts, - batch.rich_type_inference, - batch.function_return_types, declared_column_order(member, batch.nullability), reference_types, ]) .to_string(); let mut hasher: Sha256 = Sha256::new(); hasher.update(query); + hasher.update(batch_digest); hasher.update(rest.as_bytes()); hasher.finalize().into() } diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs index a3423f5705..3f947b3376 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/models.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -1,6 +1,7 @@ //! Facts a compile's producers retain about each resource, in production order. use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; use serde_json::Value; @@ -17,11 +18,11 @@ pub struct CompiledModelFacts { pub name: String, pub relative_path: String, /// SQL after variable, declaration and macro expansion. - pub query_sql: String, + pub query_sql: Arc, /// The authored query below the `MODEL (...)` header. - pub authored_query_sql: String, + pub authored_query_sql: Arc, /// The whole authored file. - pub authored_sql: String, + pub authored_sql: Arc, pub config: ModelConfigFacts, pub references: Vec, pub schema: Option, @@ -210,9 +211,9 @@ pub struct SplicedSql { pub struct PlainModel { pub name: String, pub relative_path: String, - pub query_sql: String, - pub authored_query_sql: String, - pub authored_sql: String, + pub query_sql: Arc, + pub authored_query_sql: Arc, + pub authored_sql: Arc, pub config: ModelConfigFacts, /// The attached schema entry; present when the header declares description, columns or audits. pub schema: Option, diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs index 1cbd5c6831..6ce0157bf8 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs @@ -3,7 +3,7 @@ use crate::compiled_project::models::{CompiledModelFacts, ModelAnalysisFact}; pub(super) fn model(name: &str, query_sql: &str) -> CompiledModelFacts { CompiledModelFacts { name: name.to_owned(), - query_sql: query_sql.to_owned(), + query_sql: query_sql.into(), ..CompiledModelFacts::default() } } diff --git a/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs b/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs index d3041bb078..e236eaa94e 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs @@ -51,7 +51,7 @@ fn given_recorded_models_when_reading_then_order_and_latest_facts_are_kept() { .map(|facts| facts.name.as_str()) .collect(); let query_sql: Option<&str> = - model_facts(&project, "orders").map(|facts| facts.query_sql.as_str()); + model_facts(&project, "orders").map(|facts| &*facts.query_sql); assert_eq!(order, test_case.expected_order, "{}", test_case.description); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs new file mode 100644 index 0000000000..5c74d8b071 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs @@ -0,0 +1,2 @@ +pub(crate) const SIMPLE_TEMPLATE_DIALECTS: [&str; 2] = ["duckdb", "snowflake"]; +pub(crate) const MAX_TEMPLATE_DEPTH: usize = 32; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs index 7207851e88..eefb8cb50c 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs @@ -2,6 +2,7 @@ pub mod assembly; pub mod authored_macro_calls; pub mod case_identity; pub(crate) mod check_independence; +pub(crate) mod constants; pub mod cte_rename; pub mod cte_slices; pub mod cte_sql; @@ -19,3 +20,4 @@ pub mod reference_calls; pub mod relation_markers; pub mod rendering; pub mod scenario_extraction; +pub(crate) mod simple_template; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index 12a44c5c3e..e0d564e389 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -143,10 +143,10 @@ struct AnalysisResolvedSql { reachable_mock_names: HashSet, } -#[derive(Clone)] +#[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct AnalysisTemplate { - existing_cte_names: HashSet, - marker_calls: Vec<(String, String)>, + pub(crate) existing_cte_names: HashSet, + pub(crate) marker_calls: Arc<[(String, String)]>, } type AnalysisTemplateCell = Arc>>; @@ -1341,10 +1341,10 @@ fn analyze_and_resolve_sql( for (name, sql) in request.fixture_ctes { generated_state.insert_helper(name, sql); } - let mut replacements: HashMap<(String, String), String> = HashMap::new(); - for (function_name, referenced_name) in template.marker_calls { + let mut replacements: HashMap<(&str, &str), String> = HashMap::new(); + for (function_name, referenced_name) in template.marker_calls.iter() { let target = if function_name == REF_FUNCTION { - if let Some(chain_sql) = request.resolved_chain.get(&referenced_name) { + if let Some(chain_sql) = request.resolved_chain.get(referenced_name) { for (name, sql) in &chain_sql.generated_ctes { if request.fixtures.scope.is_top_level_helper(name) || name.starts_with(TABLE_FUNCTION_PREFIX) @@ -1364,7 +1364,7 @@ fn analyze_and_resolve_sql( } else { generated_state.mock_target(MockTargetRequest { prefix: REF_PREFIX, - referenced_name: &referenced_name, + referenced_name, mocks: &request.fixtures.mock_refs, fixtures: request.fixtures, file_label: request.file_label, @@ -1373,7 +1373,7 @@ fn analyze_and_resolve_sql( } else if function_name == SOURCE_FUNCTION { generated_state.mock_target(MockTargetRequest { prefix: SOURCE_PREFIX, - referenced_name: &referenced_name, + referenced_name, mocks: &request.fixtures.mock_sources, fixtures: request.fixtures, file_label: request.file_label, @@ -1381,7 +1381,7 @@ fn analyze_and_resolve_sql( } else if function_name == SEED_FUNCTION { generated_state.mock_target(MockTargetRequest { prefix: SEED_PREFIX, - referenced_name: &referenced_name, + referenced_name, mocks: &request.fixtures.mock_seeds, fixtures: request.fixtures, file_label: request.file_label, @@ -1389,7 +1389,7 @@ fn analyze_and_resolve_sql( } else if function_name == DBT_REF_FUNCTION { generated_state.mock_target(MockTargetRequest { prefix: DBT_REF_PREFIX, - referenced_name: &referenced_name, + referenced_name, mocks: &request.fixtures.mock_dbt_refs, fixtures: request.fixtures, file_label: request.file_label, @@ -1397,13 +1397,13 @@ fn analyze_and_resolve_sql( } else if function_name == UDF_FUNCTION || function_name == TABLE_FUNCTION { request .functions - .get(&referenced_name) + .get(referenced_name) .map(|_| referenced_name.clone()) } else { None }; if let Some(target) = target { - replacements.insert((function_name, referenced_name), target); + replacements.insert((function_name.as_str(), referenced_name.as_str()), target); } } let cte_body_sql = strip_statement_terminators( @@ -1420,29 +1420,54 @@ fn analyze_and_resolve_sql( })) } -fn analysis_template( +#[cfg(test)] +thread_local! { + static TEMPLATE_AST_PARSES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[cfg(test)] +pub(crate) fn take_template_ast_parse_count() -> usize { + TEMPLATE_AST_PARSES.with(|count| count.replace(0)) +} + +pub(crate) fn analysis_template( query_sql: &str, dialect_name: &str, templates: &AnalysisTemplateCache, ) -> Result, String> { cached_analysis_template(query_sql, templates, || { Dialect::get_by_name(dialect_name).and_then(|dialect| { - let mut statements = match dialect.parse(query_sql) { - Ok(statements) => statements, - Err(_) => return None, - }; - if statements.len() != 1 { - return None; + if let Some(template) = + super::simple_template::simple_analysis_template(query_sql, dialect_name, &dialect) + { + return Some(template); } - let expression = statements.remove(0); - Some(AnalysisTemplate { - existing_cte_names: top_level_cte_names(&expression), - marker_calls: relation_marker_calls(&expression), - }) + parsed_analysis_template(query_sql, &dialect) }) }) } +/// The full-parser path also serves as the differential oracle for the bounded recognizer. +pub(crate) fn parsed_analysis_template( + query_sql: &str, + dialect: &Dialect, +) -> Option { + #[cfg(test)] + TEMPLATE_AST_PARSES.with(|count| count.set(count.get() + 1)); + let mut statements = match dialect.parse(query_sql) { + Ok(statements) => statements, + Err(_) => return None, + }; + if statements.len() != 1 { + return None; + } + let expression = statements.remove(0); + Some(AnalysisTemplate { + existing_cte_names: top_level_cte_names(&expression), + marker_calls: relation_marker_calls(&expression).into(), + }) +} + pub(crate) fn cached_analysis_template( query_sql: &str, templates: &AnalysisTemplateCache, @@ -1453,11 +1478,13 @@ where { let cached = { let mut cache = templates.lock().map_err(|error| error.to_string())?; - Arc::clone( - cache - .entry(query_sql.to_string()) - .or_insert_with(|| Arc::new(OnceLock::new())), - ) + if let Some(cached) = cache.get(query_sql) { + Arc::clone(cached) + } else { + let cached = Arc::new(OnceLock::new()); + cache.insert(query_sql.to_owned(), Arc::clone(&cached)); + cached + } }; Ok(cached.get_or_init(initialize).clone()) } @@ -1532,7 +1559,7 @@ fn resolve_textual_sql(request: TextualResolutionRequest<'_>) -> Result) -> Result { + sql: std::borrow::Cow<'a, str>, reached: HashSet, ctes: Vec<(String, String)>, } -fn resolve_table_function_fixtures( - sql: &str, +fn resolve_table_function_fixtures<'a>( + sql: &'a str, fixtures: &TestFixtures, patterns: &SqlTestPatterns, -) -> Result { +) -> Result, String> { let mut resolved = TableFunctionFixtures { - sql: sql.to_string(), + sql: std::borrow::Cow::Borrowed(sql), reached: HashSet::new(), ctes: Vec::new(), }; @@ -1603,7 +1630,7 @@ fn resolve_table_function_fixtures( .then(|| (format!("{TABLE_FUNCTION_PREFIX}{name}"), true)) }, )?; - resolved.sql = replaced; + resolved.sql = std::borrow::Cow::Owned(replaced); for (name, body) in &fixtures.mock_table_functions { if !reached.contains(name) { continue; @@ -1662,33 +1689,39 @@ fn top_level_cte_names(expression: &Expression) -> HashSet { with.ctes.iter().map(|cte| cte.alias.name.clone()).collect() } -fn replace_relation_markers( - sql: &str, - replacements: &HashMap<(String, String), String>, +fn replace_relation_markers<'a>( + sql: &'a str, + replacements: &HashMap<(&str, &str), String>, patterns: &SqlTestPatterns, -) -> String { - let mut result = sql.to_string(); +) -> std::borrow::Cow<'a, str> { + let mut result = std::borrow::Cow::Borrowed(sql); for (function_name, pattern) in [ (REF_FUNCTION, &patterns.reference), (SOURCE_FUNCTION, &patterns.source), (SEED_FUNCTION, &patterns.seed), ] { - result = replace_named_markers(&result, pattern, &patterns.lexical, |name| { - replacements - .get(&(function_name.to_string(), name.to_string())) - .cloned() - }); + if !replacements.keys().any(|(name, _)| *name == function_name) { + continue; + } + result = std::borrow::Cow::Owned(replace_named_markers( + &result, + pattern, + &patterns.lexical, + |name| replacements.get(&(function_name, name)).cloned(), + )); } - replace_dbt_ref_markers( + if !replacements + .keys() + .any(|(name, _)| *name == DBT_REF_FUNCTION) + { + return result; + } + std::borrow::Cow::Owned(replace_dbt_ref_markers( &result, &patterns.dbt_reference, &patterns.lexical, - |name| { - replacements - .get(&(DBT_REF_FUNCTION.to_string(), name.to_string())) - .cloned() - }, - ) + |name| replacements.get(&(DBT_REF_FUNCTION, name)).cloned(), + )) } struct UnresolvedReferenceRequest<'a> { diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs index 9c36ee4206..959d41a9fc 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs @@ -9,24 +9,28 @@ pub(crate) type MarkerCall = (String, String); /// Return marker calls in node order, where Polyglot children follow declared field order. pub(crate) fn relation_marker_calls(expression: &Expression) -> Vec { - expression.dfs().flat_map(node_marker_calls).collect() + expression.dfs().fold(Vec::new(), node_marker_calls) } /// FROM then JOIN marker calls of one node, plus the serialized `on` that traversal skips. -fn node_marker_calls(node: &Expression) -> Vec { +fn node_marker_calls(mut calls: Vec, node: &Expression) -> Vec { match node { - Expression::Select(select) => select - .from - .iter() - .flat_map(from_markers) - .chain(join_markers(&select.joins)) - .collect(), - Expression::From(from) => from_markers(from).collect(), - Expression::Delete(delete) => join_markers(&delete.joins).collect(), - Expression::JoinedTable(table) => join_markers(&table.joins).collect(), - Expression::DataDeletionProperty(property) => relation_marker_calls(&property.on), - _ => Vec::new(), + Expression::Select(select) => calls.extend( + select + .from + .iter() + .flat_map(from_markers) + .chain(join_markers(&select.joins)), + ), + Expression::From(from) => calls.extend(from_markers(from)), + Expression::Delete(delete) => calls.extend(join_markers(&delete.joins)), + Expression::JoinedTable(table) => calls.extend(join_markers(&table.joins)), + Expression::DataDeletionProperty(property) => { + return property.on.dfs().fold(calls, node_marker_calls); + } + _ => {} } + calls } fn from_markers(from: &From) -> impl Iterator + '_ { @@ -40,34 +44,26 @@ fn join_markers(joins: &[Join]) -> impl Iterator + '_ { fn marker(expression: &Expression) -> Option { match expression { Expression::Alias(alias) => marker(&alias.this), - Expression::Function(function) => { - let names: Vec> = function - .args - .iter() - .map(|argument| match argument { - Expression::Column(column) => Some(column.name.name.as_str()), - _ => None, - }) - .collect(); - function_marker(&function.name, &names) - } + Expression::Function(function) => function_marker(&function.name, &function.args), _ => None, } } -fn function_marker(name: &str, args: &[Option<&str>]) -> Option { +fn function_marker(name: &str, args: &[Expression]) -> Option { let function_name = name.to_ascii_lowercase(); let referenced_name = if function_name == DBT_REF_FUNCTION { match args { - [Some(first)] => (*first).to_string(), - [Some(first), Some(second)] => format!("{first}__{second}"), + [Expression::Column(first)] => first.name.name.clone(), + [Expression::Column(first), Expression::Column(second)] => { + format!("{}__{}", first.name.name, second.name.name) + } _ => return None, } } else { - let [Some(first)] = args else { + let [Expression::Column(first)] = args else { return None; }; - (*first).to_string() + first.name.name.clone() }; Some((function_name, referenced_name)) } diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs new file mode 100644 index 0000000000..6709e33764 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/simple_template.rs @@ -0,0 +1,266 @@ +//! Closed token productions for test-planning facts; unrecognized input uses the full parser. + +use std::collections::HashSet; + +use polyglot_sql::Dialect; +use polyglot_sql::parser::NO_PAREN_FUNCTION_NAMES; +use polyglot_sql::tokens::{Token, TokenType}; + +use crate::compiler::_helpers::sql_tests::constants::{ + MAX_TEMPLATE_DEPTH, SIMPLE_TEMPLATE_DIALECTS, +}; +use crate::compiler::_helpers::sql_tests::planning::{AnalysisTemplate, DBT_REF_FUNCTION}; +use crate::compiler::_helpers::sql_tests::relation_markers::MarkerCall; + +/// Recognize bounded DuckDB/Snowflake productions; other syntax retains the existing parser. +pub(crate) fn simple_analysis_template( + sql: &str, + dialect_name: &str, + dialect: &Dialect, +) -> Option { + if !SIMPLE_TEMPLATE_DIALECTS.contains(&dialect_name) || sql.contains("/*") { + return None; + } + let tokens = match dialect.tokenize(sql) { + Ok(tokens) => tokens, + Err(_) => return None, + }; + let mut reader = TemplateReader { + tokens: &tokens, + position: 0, + }; + let facts = reader.query(0)?; + reader.take(TokenType::Semicolon); + if reader.position != tokens.len() { + return None; + } + Some(AnalysisTemplate { + existing_cte_names: facts.names, + marker_calls: facts.calls.into(), + }) +} + +struct QueryFacts { + names: HashSet, + calls: Vec, +} + +struct TemplateReader<'a> { + tokens: &'a [Token], + position: usize, +} + +impl<'a> TemplateReader<'a> { + fn take(&mut self, kind: TokenType) -> bool { + if self + .tokens + .get(self.position) + .is_some_and(|token| token.token_type == kind) + { + self.position += 1; + true + } else { + false + } + } + + fn require(&mut self, kind: TokenType) -> Option<()> { + self.take(kind).then_some(()) + } + + fn identifier(&mut self) -> Option<&'a str> { + let token = self.tokens.get(self.position)?; + if !matches!( + token.token_type, + TokenType::Var | TokenType::Identifier | TokenType::QuotedIdentifier | TokenType::Final + ) { + return None; + } + self.position += 1; + Some(&token.text) + } + + fn query(&mut self, depth: usize) -> Option { + if depth > MAX_TEMPLATE_DEPTH { + return None; + } + let mut names: HashSet = HashSet::new(); + let mut cte_calls: Vec = Vec::new(); + let has_with = self.take(TokenType::With); + if has_with { + loop { + names.insert(self.identifier()?.to_owned()); + self.require(TokenType::As)?; + self.require(TokenType::LParen)?; + cte_calls.extend(self.query(depth + 1)?.calls); + self.require(TokenType::RParen)?; + if !self.take(TokenType::Comma) { + break; + } + } + } + let mut calls = self.select(depth)?; + while self.take(TokenType::Union) { + if has_with { + return None; + } + self.require(TokenType::All)?; + if self.take(TokenType::By) { + let name = self.tokens.get(self.position)?; + if name.token_type != TokenType::Var || !name.text.eq_ignore_ascii_case("name") { + return None; + } + self.position += 1; + } + calls.extend(self.select(depth)?); + } + calls.extend(cte_calls); + Some(QueryFacts { names, calls }) + } + + fn select(&mut self, depth: usize) -> Option> { + self.require(TokenType::Select)?; + loop { + if !self.take(TokenType::Star) { + self.scalar(depth + 1)?; + if self.take(TokenType::As) { + self.identifier()?; + } + } + if !self.take(TokenType::Comma) { + break; + } + } + self.require(TokenType::From)?; + let mut calls: Vec = self.relation()?.into_iter().collect(); + loop { + let left = self.take(TokenType::Left); + if !self.take(TokenType::Join) { + if left { + return None; + } + break; + } + calls.extend(self.relation()?); + self.require(TokenType::On)?; + self.condition(depth + 1)?; + } + if self.take(TokenType::Where) { + self.condition(depth + 1)?; + } + Some(calls) + } + + fn relation(&mut self) -> Option> { + let name = self.identifier()?; + let call = if self.take(TokenType::LParen) { + let function = name.to_ascii_lowercase(); + if !matches!( + function.as_str(), + "__ref" | "__source" | "__seed" | "__dbt_ref" + ) { + return None; + } + let mut target = self.marker_argument()?.to_owned(); + if function == DBT_REF_FUNCTION && self.take(TokenType::Comma) { + target.push_str("__"); + target.push_str(self.marker_argument()?); + } + self.require(TokenType::RParen)?; + Some((function, target)) + } else { + None + }; + if self.take(TokenType::As) { + self.identifier()?; + } + Some(call) + } + + /// Require quoted Columns; unquoted arguments can invoke private dialect-specific function rules. + fn marker_argument(&mut self) -> Option<&'a str> { + let token = self.tokens.get(self.position)?; + if token.token_type != TokenType::QuotedIdentifier + || NO_PAREN_FUNCTION_NAMES.contains(token.text.to_ascii_uppercase().as_str()) + { + return None; + } + self.identifier() + } + + fn condition(&mut self, depth: usize) -> Option<()> { + self.scalar(depth)?; + self.require(TokenType::Eq)?; + self.scalar(depth) + } + + fn scalar(&mut self, depth: usize) -> Option<()> { + self.atom(depth)?; + while self.take(TokenType::Plus) + || self.take(TokenType::Dash) + || self.take(TokenType::Star) + || self.take(TokenType::Slash) + { + self.atom(depth)?; + } + Some(()) + } + + fn atom(&mut self, depth: usize) -> Option<()> { + if depth > MAX_TEMPLATE_DEPTH { + return None; + } + if self.take(TokenType::LParen) { + self.scalar(depth + 1)?; + return self.require(TokenType::RParen); + } + if self.take(TokenType::Cast) { + self.require(TokenType::LParen)?; + self.scalar(depth + 1)?; + self.require(TokenType::As)?; + let data_type = self.tokens.get(self.position)?; + if !matches!( + data_type.token_type, + TokenType::Var | TokenType::Int | TokenType::Double + ) || !["int", "integer", "double"] + .iter() + .any(|name| data_type.text.eq_ignore_ascii_case(name)) + { + return None; + } + self.position += 1; + return self.require(TokenType::RParen); + } + if let Some(token) = self.tokens.get(self.position) + && token.token_type == TokenType::Number + { + if !token.text.bytes().any(|byte| byte.is_ascii_digit()) + || !token + .text + .bytes() + .all(|byte| byte.is_ascii_digit() || byte == b'.') + || token.text.bytes().filter(|byte| *byte == b'.').count() > 1 + { + return None; + } + self.position += 1; + return Some(()); + } + let name = self.identifier()?; + if self.take(TokenType::LParen) { + if !name.eq_ignore_ascii_case("coalesce") { + return None; + } + self.scalar(depth + 1)?; + while self.take(TokenType::Comma) { + self.scalar(depth + 1)?; + } + self.require(TokenType::RParen) + } else { + if self.take(TokenType::Dot) { + self.identifier()?; + } + Some(()) + } + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index 602fb830b6..5700adc60d 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/simple_template/helpers.rs"] +mod simple_template_helpers; #[path = "tests/test_sql_test_cte_rename.rs"] mod sql_test_cte_rename; #[path = "tests/test_sql_test_cte_slices.rs"] @@ -22,5 +24,7 @@ mod sql_test_planning; mod sql_test_relation_markers; #[path = "tests/test_sql_test_rendering.rs"] mod sql_test_rendering; +#[path = "tests/test_sql_test_simple_template.rs"] +mod sql_test_simple_template; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs new file mode 100644 index 0000000000..8fb1f0df7c --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/simple_template/helpers.rs @@ -0,0 +1,114 @@ +use polyglot_sql::Dialect; +use polyglot_sql::parser::NO_PAREN_FUNCTION_NAMES; +use std::collections::HashMap; +use std::sync::Mutex; + +use crate::compiler::_helpers::sql_tests::planning::{ + AnalysisTemplate, analysis_template, parsed_analysis_template, +}; +use crate::compiler::_helpers::sql_tests::simple_template::simple_analysis_template; +use crate::compiler::tests::test_types::SimpleTemplateCorpusOutcome; + +pub(crate) fn no_paren_marker_mismatches(dialect_name: &str) -> Vec { + let dialect = Dialect::get_by_name(dialect_name).expect("supported test dialect"); + let cache = Mutex::new(HashMap::new()); + NO_PAREN_FUNCTION_NAMES + .iter() + .flat_map(|name| [name.to_string(), name.to_ascii_lowercase()]) + .flat_map(|name| [name.clone(), format!("\"{name}\"")]) + .flat_map(|argument| { + [ + format!("__ref({argument})"), + format!("__source({argument})"), + format!("__seed({argument})"), + format!("__dbt_ref({argument})"), + format!("__dbt_ref({argument}, \"orders\")"), + format!("__dbt_ref(\"orders\", {argument})"), + ] + }) + .filter_map(|relation| { + let sql = format!("SELECT id FROM {relation}"); + let fast = simple_analysis_template(&sql, dialect_name, &dialect); + let expected = parsed_analysis_template(&sql, &dialect); + let actual = analysis_template(&sql, dialect_name, &cache).expect("template cache"); + (fast.is_some() + || actual != expected + || expected + .as_ref() + .is_some_and(|facts| !facts.marker_calls.is_empty())) + .then(|| format!("{sql}: fast={fast:?}, actual={actual:?}, expected={expected:?}")) + }) + .collect() +} + +pub(crate) fn simple_template_corpus_outcome(dialect_name: &str) -> SimpleTemplateCorpusOutcome { + let dialect = Dialect::get_by_name(dialect_name).expect("supported test dialect"); + let mut queries: Vec = Vec::new(); + for projection in [ + "id", + "*", + "CAST(id AS INTEGER) AS id", + "COALESCE(id, 0) + id * 2 AS id", + "CAST((id + 1) / 2 AS DOUBLE) AS id", + "1.5 AS id", + "\"__ref(ignored)\" AS id", + "\"a\"\"b\" AS id", + ] { + for relation in [ + "__ref(\"orders\")", + "__source(\"orders\")", + "__seed(\"orders\")", + "__dbt_ref(\"shop\", \"orders\")", + "\"__ref\"(\"orders\")", + "__REF(orders)", + "\"__REF\"(\"orders\")", + "__ref(final)", + "__ref(\"a.b\")", + "__ref(\"a\"\"b\")", + ] { + let body = format!("SELECT {projection} FROM {relation}"); + for sql in [ + body.clone(), + format!("WITH recent AS ({body}) SELECT id FROM recent"), + format!("{body} UNION ALL BY NAME {body}"), + format!( + "WITH recent AS ({body}) SELECT id FROM __ref(\"customers\") AS c JOIN recent AS r ON c.id = r.id" + ), + ] { + let tokens = dialect.tokenize(&sql).expect("generated SQL tokenizes"); + for token in &tokens { + queries.push(format!( + "{} {}", + &sql[..token.span.start], + &sql[token.span.end..] + )); + for inserted in [".", ")", "(", "FROM", "'__ref(\"ignored\")'"] { + queries.push(format!( + "{} {inserted} {}", + &sql[..token.span.start], + &sql[token.span.start..] + )); + } + } + queries.push(sql); + } + } + } + let recognized: Vec<(String, AnalysisTemplate)> = queries + .into_iter() + .filter_map(|sql| { + simple_analysis_template(&sql, dialect_name, &dialect).map(|facts| (sql, facts)) + }) + .collect(); + SimpleTemplateCorpusOutcome { + recognized: recognized.len(), + mismatches: recognized + .into_iter() + .filter_map(|(sql, fast)| { + let parsed = parsed_analysis_template(&sql, &dialect); + (Some(&fast) != parsed.as_ref()) + .then(|| format!("{sql}: fast={fast:?}, parsed={parsed:?}")) + }) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs index 9a9c4a061d..f054efe809 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs @@ -6,6 +6,18 @@ use crate::compiler::tests::test_types::{RelationMarkerOracleTestCase, RelationM #[test] fn given_parsed_statements_when_collecting_relation_markers_then_json_walk_order_is_reproduced() { let test_cases = [ + RelationMarkersTestCase { + description: "nested CTEs preserve relation order and ignore escaped marker lookalikes", + dialect: "postgres", + sql: r#"WITH "recent" AS (WITH nested AS (SELECT * FROM __source("orders")) SELECT * FROM nested) SELECT '__ref("ignored")''s' AS note, $$__seed("ignored")$$ AS body FROM recent JOIN __ref("customers") ON TRUE /* __ref("ignored") */"#, + expected_calls: &[("__ref", "customers"), ("__source", "orders")], + }, + RelationMarkersTestCase { + description: "function markers require column arguments even inside nested relations", + dialect: "duckdb", + sql: r#"SELECT __udf("projected") FROM __table_fn("orders") JOIN __udf("lookup") ON TRUE JOIN (SELECT * FROM __ref('literal') JOIN __ref(COALESCE("a", "b")) ON TRUE) t ON TRUE"#, + expected_calls: &[("__table_fn", "orders"), ("__udf", "lookup")], + }, RelationMarkersTestCase { description: "plain reference", dialect: "duckdb", diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs new file mode 100644 index 0000000000..a11546bfe3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_simple_template.rs @@ -0,0 +1,340 @@ +use std::collections::HashMap; +use std::sync::Mutex; + +use polyglot_sql::Dialect; + +use crate::compiler::_helpers::sql_tests::planning::{ + analysis_template, parsed_analysis_template, take_template_ast_parse_count, +}; +use crate::compiler::_helpers::sql_tests::simple_template::simple_analysis_template; +use crate::compiler::tests::simple_template_helpers::{ + no_paren_marker_mismatches, simple_template_corpus_outcome, +}; +use crate::compiler::tests::test_types::{ + RelationMarkerOracleTestCase, SimpleTemplateCorpusTestCase, SimpleTemplateTestCase, +}; + +#[test] +fn given_registered_no_paren_arguments_when_planning_then_all_marker_positions_use_the_parser() { + let test_cases = [ + RelationMarkerOracleTestCase { + description: "all registry names in quoted and unquoted marker argument positions", + dialect: "duckdb", + expected_mismatches: &[], + }, + RelationMarkerOracleTestCase { + description: "all Snowflake registry names in quoted and unquoted marker argument positions", + dialect: "snowflake", + expected_mismatches: &[], + }, + ]; + for test_case in test_cases { + assert_eq!( + no_paren_marker_mismatches(test_case.dialect), + test_case.expected_mismatches, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_bounded_and_fallback_queries_when_planning_then_facts_and_ast_counts_match() { + let test_cases = [ + SimpleTemplateTestCase { + description: "Snowflake dense projections and union remove the template AST", + dialect: "snowflake", + sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), +projected AS (SELECT CAST(b.id AS INTEGER) AS id, +CAST(COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE) AS DOUBLE) AS amount FROM imported AS b), +combined AS (SELECT id, amount FROM projected UNION ALL BY NAME SELECT id, amount FROM projected WHERE id = 0), +final AS (SELECT CAST(id AS INTEGER) AS id, CAST(amount AS DOUBLE) AS amount FROM combined) +SELECT id, amount FROM final"#, + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "Snowflake dense UDF projection retains the template AST", + dialect: "snowflake", + sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), +projected AS (SELECT CAST(b.id AS INTEGER) AS id, +CAST(__udf("amount")(b.amount) AS DOUBLE) AS amount FROM imported AS b) +SELECT id, amount FROM projected"#, + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "Snowflake QUALIFY remains on the full parser", + dialect: "snowflake", + sql: "SELECT id FROM __ref(\"orders\") QUALIFY ROW_NUMBER() OVER (ORDER BY id) = 1", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "quoted no-paren source argument retains parser Function classification", + dialect: "duckdb", + sql: "SELECT id FROM __source(\"now\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unquoted no-paren reference argument retains parser Function classification", + dialect: "duckdb", + sql: "SELECT id FROM __ref(NOW)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "first dbt argument with no-paren name retains parser classification", + dialect: "duckdb", + sql: "SELECT id FROM __dbt_ref(\"pi\", \"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "second dbt argument with no-paren name retains parser classification", + dialect: "duckdb", + sql: "SELECT id FROM __dbt_ref(\"orders\", \"getdate\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unquoted ordinary argument conservatively retains full parser", + dialect: "duckdb", + sql: "SELECT id FROM __seed(orders)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unquoted dialect-specific user argument retains full parser", + dialect: "duckdb", + sql: "SELECT id FROM __source(user)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "quoted user is an ordinary Column in the pinned parser", + dialect: "duckdb", + sql: "SELECT id FROM __source(\"user\")", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "dense CTE projections joins and union remove one cold template AST", + dialect: "duckdb", + sql: r#"WITH imported AS (SELECT * FROM __ref("orders")), +lookup_0 AS (SELECT * FROM __source("customers")), +projected AS (SELECT CAST(b.id AS INTEGER) AS id, +CAST(b.amount + COALESCE(l0.amount, 0) AS DOUBLE) AS amount, +CAST(COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE) AS DOUBLE) AS quantity +FROM imported AS b LEFT JOIN lookup_0 AS l0 ON l0.id = b.id), +combined AS (SELECT id, amount, quantity FROM projected +UNION ALL BY NAME SELECT id, amount, quantity FROM projected WHERE id = 0), +final AS (SELECT CAST(id AS INTEGER) AS id, CAST(amount AS DOUBLE) AS amount FROM combined) +SELECT id, amount FROM final"#, + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "nested CTEs follow outer relation then WITH traversal order", + dialect: "duckdb", + sql: "WITH recent AS (WITH nested AS (SELECT * FROM __source(\"orders\")) SELECT id FROM nested) SELECT id FROM recent JOIN __ref(\"customers\") AS c ON c.id = recent.id", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "root union has no Select CTE names", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") UNION ALL SELECT id FROM __source(\"orders\")", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "quoted names escapes and line-comment marker lookalikes", + dialect: "duckdb", + sql: "WITH \"recent\" AS (SELECT * FROM __ref(\"or\"\"ders\")) SELECT id FROM \"recent\" -- __seed(\"ignored\")", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "two-part dbt references precede joined seeds", + dialect: "duckdb", + sql: "SELECT id FROM __dbt_ref(\"shop\", \"orders\") AS o JOIN __seed(\"regions\") AS r ON o.id = r.id", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "scalar nested subquery retains full parser", + dialect: "duckdb", + sql: "SELECT (SELECT id FROM __ref(\"orders\")) AS id", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "string marker lookalike retains full parser", + dialect: "duckdb", + sql: "SELECT '__ref(\"ignored\")' AS note FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "table function call retains full parser", + dialect: "duckdb", + sql: "SELECT id FROM __table_fn(\"orders\")(1)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "scalar function marker retains full parser", + dialect: "duckdb", + sql: "SELECT __udf(\"amount\")(id) AS id FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "WITH over UNION retains full parser AST attachment", + dialect: "duckdb", + sql: "WITH recent AS (SELECT id FROM orders) SELECT id FROM recent UNION ALL SELECT id FROM recent", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unterminated block comment retains full parser policy", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") /* unclosed", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "nested block comments retain full parser policy", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") /* outer /* __ref(\"ignored\") */ inner */", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "missing projection retains parser failure", + dialect: "duckdb", + sql: "SELECT FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "incomplete cast retains parser failure", + dialect: "duckdb", + sql: "SELECT CAST(id AS INTEGER FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "incomplete WHERE retains parser failure", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") WHERE", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unterminated identifier retains parser failure", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "unterminated table identifier retains parser failure", + dialect: "duckdb", + sql: "SELECT id FROM \"orders", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "incomplete join retains parser failure", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\") LEFT", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "multiple statements retain textual fallback", + dialect: "duckdb", + sql: "SELECT id FROM __ref(\"orders\"); SELECT id FROM other", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "literal marker argument is not a Column", + dialect: "duckdb", + sql: "SELECT id FROM __ref('orders')", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "postgres dollar quoting retains its parser", + dialect: "postgres", + sql: "SELECT $$__ref(\"ignored\")$$ AS note FROM __ref(\"orders\")", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "snowflake ordinary quoted names use recognized facts", + dialect: "snowflake", + sql: "SELECT id FROM __ref(\"orders\")", + expected_ast_parses: 0, + }, + SimpleTemplateTestCase { + description: "bigquery backtick names retain its parser", + dialect: "bigquery", + sql: "SELECT id FROM __ref(`orders`)", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "tsql brackets retain its parser", + dialect: "tsql", + sql: "SELECT id FROM __ref([orders])", + expected_ast_parses: 1, + }, + SimpleTemplateTestCase { + description: "databricks quoting retains its parser", + dialect: "databricks", + sql: "SELECT id FROM __ref(`orders`)", + expected_ast_parses: 1, + }, + ]; + for test_case in test_cases { + let dialect = Dialect::get_by_name(test_case.dialect).expect("supported test dialect"); + take_template_ast_parse_count(); + let expected = parsed_analysis_template(test_case.sql, &dialect); + assert_eq!( + take_template_ast_parse_count(), + 1, + "{}", + test_case.description + ); + let fast = simple_analysis_template(test_case.sql, test_case.dialect, &dialect); + assert_eq!( + usize::from(fast.is_none()), + test_case.expected_ast_parses, + "{}", + test_case.description + ); + assert_eq!( + fast.as_ref(), + expected + .as_ref() + .filter(|_| test_case.expected_ast_parses == 0), + "{}", + test_case.description + ); + let cache = Mutex::new(HashMap::new()); + let actual = + analysis_template(test_case.sql, test_case.dialect, &cache).expect("template cache"); + assert_eq!(actual, expected, "{}", test_case.description); + assert_eq!( + take_template_ast_parse_count(), + test_case.expected_ast_parses, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_generated_and_mutated_queries_when_recognized_then_every_fact_matches_the_ast() { + let test_cases = [ + SimpleTemplateCorpusTestCase { + description: "combinations and token deletions preserve parser acceptance and facts", + dialect: "duckdb", + expected_min_recognized: 200, + expected_mismatches: &[], + }, + SimpleTemplateCorpusTestCase { + description: "Snowflake token productions preserve parser acceptance and facts", + dialect: "snowflake", + expected_min_recognized: 200, + expected_mismatches: &[], + }, + ]; + for test_case in test_cases { + let outcome = simple_template_corpus_outcome(test_case.dialect); + assert!( + outcome.recognized >= test_case.expected_min_recognized, + "{}: {} recognized", + test_case.description, + outcome.recognized + ); + assert_eq!( + outcome.mismatches, test_case.expected_mismatches, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index b94a1c4d0f..d3884c1bb4 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -186,3 +186,21 @@ pub(crate) struct SqlTestAssemblyTestCase { pub(crate) batch: crate::compiler::models::SqlTestAssemblyBatch, pub(crate) expected_outcomes: Vec, } +pub(crate) struct SimpleTemplateTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub sql: &'static str, + pub expected_ast_parses: usize, +} + +pub(crate) struct SimpleTemplateCorpusTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub expected_min_recognized: usize, + pub expected_mismatches: &'static [&'static str], +} + +pub(crate) struct SimpleTemplateCorpusOutcome { + pub recognized: usize, + pub mismatches: Vec, +} diff --git a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs index b1b26f57a2..7365790654 100644 --- a/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs +++ b/crates/sqlbuild-cache/src/artifacts/_helpers/files.rs @@ -5,6 +5,9 @@ use std::fs::{File, OpenOptions}; use std::io::{ErrorKind, Read, Write}; use std::path::{Path, PathBuf}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +use crate::artifacts::constants::{PARALLEL_WRITE_MIN_FILES, WRITE_WORKERS}; use crate::artifacts::errors::ArtifactError; use crate::artifacts::models::{Publication, WrittenArtifacts}; @@ -19,23 +22,74 @@ pub(crate) fn write_artifacts( files: &[(PathBuf, Vec)], check_existing: bool, ) -> Result { + if files.len() >= PARALLEL_WRITE_MIN_FILES && distinct_paths(files) { + return write_artifacts_parallel(files, check_existing); + } let mut outcome: WrittenArtifacts = WrittenArtifacts::default(); for (path, contents) in files { - if check_existing && let Some(existing) = read_regular_file(path)? { - if existing == *contents { - outcome.unchanged += 1; - continue; - } - if std::str::from_utf8(&existing).is_err() { - return Err(ArtifactError::ExistingNotUtf8 { path: path.clone() }); - } + if write_artifact(path, contents, check_existing)? { + outcome.written += 1; + } else { + outcome.unchanged += 1; } - overwrite_creating_parent(path, contents)?; - outcome.written += 1; } Ok(outcome) } +/// Write distinct files on a small pool; the first failure in file order is the one reported. +fn write_artifacts_parallel( + files: &[(PathBuf, Vec)], + check_existing: bool, +) -> Result { + let write = || -> Vec> { + files + .par_iter() + .map(|(path, contents)| write_artifact(path, contents, check_existing)) + .collect() + }; + let results: Vec> = match rayon::ThreadPoolBuilder::new() + .num_threads(WRITE_WORKERS) + .build() + { + Ok(pool) => pool.install(write), + Err(_) => write(), + }; + let mut outcome: WrittenArtifacts = WrittenArtifacts::default(); + for result in results { + if result? { + outcome.written += 1; + } else { + outcome.unchanged += 1; + } + } + Ok(outcome) +} + +fn distinct_paths(files: &[(PathBuf, Vec)]) -> bool { + let mut seen: HashSet<&Path> = HashSet::with_capacity(files.len()); + files.iter().all(|(path, _)| seen.insert(path.as_path())) +} + +/// Write one artifact; `false` when a byte-identical existing file was left untouched. +fn write_artifact( + path: &Path, + contents: &[u8], + check_existing: bool, +) -> Result { + if check_existing && let Some(existing) = read_regular_file(path)? { + if existing == contents { + return Ok(false); + } + if std::str::from_utf8(&existing).is_err() { + return Err(ArtifactError::ExistingNotUtf8 { + path: path.to_path_buf(), + }); + } + } + overwrite_creating_parent(path, contents)?; + Ok(true) +} + /// A regular file's bytes, or `None` when nothing comparable is at `path`. fn read_regular_file(path: &Path) -> Result>, ArtifactError> { let mut file: File = match File::open(path) { diff --git a/crates/sqlbuild-cache/src/artifacts/constants.rs b/crates/sqlbuild-cache/src/artifacts/constants.rs index 3feacccd00..f55814e80a 100644 --- a/crates/sqlbuild-cache/src/artifacts/constants.rs +++ b/crates/sqlbuild-cache/src/artifacts/constants.rs @@ -4,3 +4,7 @@ pub const NEW_FILE_MODE: u32 = 0o666; pub(crate) const SQL_EXTENSION: &str = "sql"; #[cfg(unix)] pub(crate) const NANOS_PER_SECOND: i64 = 1_000_000_000; +/// Below this many files a sequential write is cheaper than starting a pool. +pub(crate) const PARALLEL_WRITE_MIN_FILES: usize = 64; +/// Writer threads for one artifact batch, matching SQL test planning's fixed pool. +pub(crate) const WRITE_WORKERS: usize = 4; diff --git a/crates/sqlbuild-cache/src/artifacts/mod.rs b/crates/sqlbuild-cache/src/artifacts/mod.rs index 6b771d183b..dcdc4ec126 100644 --- a/crates/sqlbuild-cache/src/artifacts/mod.rs +++ b/crates/sqlbuild-cache/src/artifacts/mod.rs @@ -5,3 +5,6 @@ pub mod constants; pub mod errors; pub mod main; pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cache/src/artifacts/tests.rs b/crates/sqlbuild-cache/src/artifacts/tests.rs new file mode 100644 index 0000000000..991ace2b43 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_write_artifacts.rs"] +mod write_artifacts; diff --git a/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs b/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs new file mode 100644 index 0000000000..febcafc534 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs @@ -0,0 +1,19 @@ +use std::path::{Path, PathBuf}; + +/// `count` distinct SQL artifacts spread over nested directories under `root`. +pub(crate) fn batch(root: &Path, count: usize) -> Vec<(PathBuf, Vec)> { + (0..count) + .map(|index| { + ( + root.join(format!("group_{}/nested/model_{index}.sql", index % 7)), + format!("select {index}\n").into_bytes(), + ) + }) + .collect() +} + +/// Create `path` with `contents`, including its parent directories. +pub(crate) fn place(path: &Path, contents: &[u8]) { + std::fs::create_dir_all(path.parent().expect("artifact parent")).expect("artifact directory"); + std::fs::write(path, contents).expect("existing artifact"); +} diff --git a/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs b/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs new file mode 100644 index 0000000000..0d130d0c77 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs @@ -0,0 +1,26 @@ +/// One artifact batch written over a target where some files already exist. +pub(crate) struct WriteBatchTestCase { + pub(crate) description: &'static str, + pub(crate) file_count: usize, + pub(crate) check_existing: bool, + pub(crate) existing_identical: &'static [usize], + pub(crate) existing_different: &'static [usize], + pub(crate) expected_written: usize, + pub(crate) expected_unchanged: usize, +} + +/// One batch with existing non-UTF-8 files at several positions. +pub(crate) struct FirstFailureTestCase { + pub(crate) description: &'static str, + pub(crate) file_count: usize, + pub(crate) failing: &'static [usize], + pub(crate) expected_failing: usize, +} + +/// One batch that queues the same path more than once. +pub(crate) struct RepeatedPathTestCase { + pub(crate) description: &'static str, + pub(crate) file_count: usize, + pub(crate) repeated: usize, + pub(crate) expected_contents: &'static [u8], +} diff --git a/crates/sqlbuild-cache/src/artifacts/tests/test_write_artifacts.rs b/crates/sqlbuild-cache/src/artifacts/tests/test_write_artifacts.rs new file mode 100644 index 0000000000..0821004a11 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests/test_write_artifacts.rs @@ -0,0 +1,131 @@ +use crate::artifacts::errors::ArtifactError; +use crate::artifacts::main::write_artifacts::write_artifacts; +use crate::artifacts::tests::helpers::{batch, place}; +use crate::artifacts::tests::test_types::{ + FirstFailureTestCase, RepeatedPathTestCase, WriteBatchTestCase, +}; + +#[test] +fn given_artifact_batch_when_written_then_every_file_matches_and_unchanged_files_are_counted() { + let test_cases = [ + WriteBatchTestCase { + description: "small sequential batch", + file_count: 5, + check_existing: true, + existing_identical: &[1], + existing_different: &[3], + expected_written: 4, + expected_unchanged: 1, + }, + WriteBatchTestCase { + description: "large parallel batch", + file_count: 500, + check_existing: true, + existing_identical: &[0, 250, 499], + existing_different: &[10, 400], + expected_written: 497, + expected_unchanged: 3, + }, + WriteBatchTestCase { + description: "parallel batch overwriting without checks", + file_count: 200, + check_existing: false, + existing_identical: &[5], + existing_different: &[6], + expected_written: 200, + expected_unchanged: 0, + }, + WriteBatchTestCase { + description: "parallel batch into an empty target", + file_count: 64, + check_existing: true, + existing_identical: &[], + existing_different: &[], + expected_written: 64, + expected_unchanged: 0, + }, + ]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("target directory"); + let files = batch(root.path(), test_case.file_count); + for &index in test_case.existing_identical { + place(&files[index].0, &files[index].1); + } + for &index in test_case.existing_different { + place(&files[index].0, b"stale\n"); + } + let outcome = write_artifacts(&files, test_case.check_existing).expect("written batch"); + assert_eq!( + outcome.written, test_case.expected_written, + "{}", + test_case.description + ); + assert_eq!( + outcome.unchanged, test_case.expected_unchanged, + "{}", + test_case.description + ); + let matching = files + .iter() + .filter(|(path, contents)| std::fs::read(path).ok().as_ref() == Some(contents)) + .count(); + assert_eq!(matching, test_case.file_count, "{}", test_case.description); + } +} + +#[test] +fn given_several_non_utf8_existing_files_when_written_then_the_first_in_batch_order_is_reported() { + let test_cases = [ + FirstFailureTestCase { + description: "sequential batch", + file_count: 10, + failing: &[3, 9], + expected_failing: 3, + }, + FirstFailureTestCase { + description: "parallel batch", + file_count: 300, + failing: &[100, 299], + expected_failing: 100, + }, + ]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("target directory"); + let files = batch(root.path(), test_case.file_count); + for &index in test_case.failing { + place(&files[index].0, &[0xff, 0xfe]); + } + let error = write_artifacts(&files, true).expect_err("non-UTF-8 existing artifact"); + assert!( + matches!( + &error, + ArtifactError::ExistingNotUtf8 { path } if *path == files[test_case.expected_failing].0 + ), + "{}: {error:?}", + test_case.description + ); + } +} + +#[test] +fn given_repeated_artifact_path_when_written_then_the_last_queued_contents_win() { + let test_cases = [RepeatedPathTestCase { + description: "large batch repeating one path", + file_count: 100, + repeated: 2, + expected_contents: b"select 'last'\n", + }]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("target directory"); + let mut files = batch(root.path(), test_case.file_count); + let repeated = files[test_case.repeated].0.clone(); + files.push((repeated.clone(), test_case.expected_contents.to_vec())); + write_artifacts(&files, true).expect("written batch"); + assert_eq!( + std::fs::read(&repeated).expect("repeated artifact"), + test_case.expected_contents, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/compile_timings/constants.rs b/crates/sqlbuild-core/src/compile_timings/constants.rs index 00f84cac18..785f37672c 100644 --- a/crates/sqlbuild-core/src/compile_timings/constants.rs +++ b/crates/sqlbuild-core/src/compile_timings/constants.rs @@ -1,7 +1,7 @@ //! The compile phases and metrics `--json` reports, in presentation order. /// Phases whose elapsed nanoseconds are reported as whole milliseconds. -pub const COMPILE_TIMING_PHASES: [&str; 13] = [ +pub const COMPILE_TIMING_PHASES: [&str; 15] = [ "attachment_ms", "model_analysis_ms", "model_analysis_cpu_ms", @@ -15,14 +15,18 @@ pub const COMPILE_TIMING_PHASES: [&str; 13] = [ "cache_publication_ms", "physical_write_ms", "stale_traversal_ms", + "early_rules_build_ms", + "early_rules_wait_ms", ]; /// Counters reported as recorded. -pub const COMPILE_METRICS: [&str; 6] = [ +pub const COMPILE_METRICS: [&str; 8] = [ "analysis_batch_cache_hits", "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", "sql_test_scan_cache_hits", "sql_test_scan_cache_misses", + "early_rules_hits", + "early_rules_misses", ]; diff --git a/crates/sqlbuild-core/src/compile_timings/tests/test_compile_timing_milliseconds.rs b/crates/sqlbuild-core/src/compile_timings/tests/test_compile_timing_milliseconds.rs index ea3cfee116..09eddb3a4b 100644 --- a/crates/sqlbuild-core/src/compile_timings/tests/test_compile_timing_milliseconds.rs +++ b/crates/sqlbuild-core/src/compile_timings/tests/test_compile_timing_milliseconds.rs @@ -26,9 +26,9 @@ fn given_recorded_timings_when_reported_then_lists_every_phase_and_metric_in_ord ]; for test_case in test_cases { let entries = compile_timing_milliseconds(&test_case.timings); - assert_eq!(entries.len(), 19, "{}", test_case.description); + assert_eq!(entries.len(), 23, "{}", test_case.description); assert_eq!( - vec![entries[0], entries[18]], + vec![entries[0], entries[20]], test_case.expected_entries, "{}", test_case.description diff --git a/crates/sqlbuild-core/src/posix_path/constants.rs b/crates/sqlbuild-core/src/posix_path/constants.rs index 9216305dc5..20101ec03f 100644 --- a/crates/sqlbuild-core/src/posix_path/constants.rs +++ b/crates/sqlbuild-core/src/posix_path/constants.rs @@ -6,3 +6,5 @@ pub(crate) const ROOT: &str = "/"; pub(crate) const DOUBLE_ROOT: &str = "//"; pub(crate) const CURRENT: &str = "."; pub(crate) const SUFFIX_MARK: char = '.'; +/// The parent-directory component `normpath` removes lexically. +pub(crate) const PARENT: &str = ".."; diff --git a/crates/sqlbuild-core/src/posix_path/main/mod.rs b/crates/sqlbuild-core/src/posix_path/main/mod.rs index e4ab185549..d135530cc5 100644 --- a/crates/sqlbuild-core/src/posix_path/main/mod.rs +++ b/crates/sqlbuild-core/src/posix_path/main/mod.rs @@ -1,3 +1,4 @@ +pub mod posix_abspath; pub mod posix_join; pub mod posix_name; pub mod posix_parent; diff --git a/crates/sqlbuild-core/src/posix_path/main/posix_abspath.rs b/crates/sqlbuild-core/src/posix_path/main/posix_abspath.rs new file mode 100644 index 0000000000..8647b2a3ee --- /dev/null +++ b/crates/sqlbuild-core/src/posix_path/main/posix_abspath.rs @@ -0,0 +1,49 @@ +//! `posixpath.abspath(path)` for a process working directory `cwd`. + +use crate::posix_path::constants::{CURRENT, DOUBLE_ROOT, PARENT, ROOT, SEPARATOR}; + +/// `path` joined to `cwd` when relative, then normalised exactly as `posixpath.normpath` does. +#[must_use] +pub fn posix_abspath(cwd: &str, path: &str) -> String { + if path.starts_with(SEPARATOR) { + return posix_normpath(path); + } + let joined: String = if cwd.is_empty() || cwd.ends_with(SEPARATOR) { + format!("{cwd}{path}") + } else { + format!("{cwd}{ROOT}{path}") + }; + posix_normpath(&joined) +} + +fn posix_normpath(path: &str) -> String { + if path.is_empty() { + return CURRENT.to_owned(); + } + let root: &str = if path.starts_with(DOUBLE_ROOT) && !path.starts_with("///") { + DOUBLE_ROOT + } else if path.starts_with(SEPARATOR) { + ROOT + } else { + "" + }; + let mut kept: Vec<&str> = Vec::new(); + for part in path.split(SEPARATOR) { + if part.is_empty() || part == CURRENT { + continue; + } + let keeps_parent: bool = + (root.is_empty() && kept.is_empty()) || kept.last() == Some(&PARENT); + if part != PARENT || keeps_parent { + kept.push(part); + } else if !kept.is_empty() { + let _ = kept.pop(); + } + } + let normalised: String = format!("{root}{}", kept.join(ROOT)); + if normalised.is_empty() { + CURRENT.to_owned() + } else { + normalised + } +} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs index 4722f0737b..93fcae39f3 100644 --- a/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs +++ b/crates/sqlbuild-core/src/posix_path/tests/test_posix_path.rs @@ -1,9 +1,10 @@ +use crate::posix_path::main::posix_abspath::posix_abspath; use crate::posix_path::main::posix_join::posix_join; use crate::posix_path::main::posix_name::posix_name; use crate::posix_path::main::posix_parent::posix_parent; use crate::posix_path::main::posix_parts::posix_parts; use crate::posix_path::main::posix_suffix::posix_suffix; -use crate::posix_path::tests::test_types::PosixPathTestCase; +use crate::posix_path::tests::test_types::{PosixAbspathTestCase, PosixPathTestCase}; #[test] fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { @@ -140,3 +141,120 @@ fn given_paths_when_reading_pure_posix_parts_then_they_match_pathlib() { ); } } + +#[test] +fn given_relative_and_absolute_paths_when_making_them_absolute_then_they_match_posixpath_abspath() { + let test_cases = [ + PosixAbspathTestCase { + description: "a relative artifact path", + cwd: "/home/u/proj", + path: "target/compiled/a.sql", + expected_path: "/home/u/proj/target/compiled/a.sql", + }, + PosixAbspathTestCase { + description: "an absolute path ignores the working directory", + cwd: "/home/u/proj", + path: "/abs/x.sql", + expected_path: "/abs/x.sql", + }, + PosixAbspathTestCase { + description: "current and empty components are dropped", + cwd: "/home/u/proj", + path: "./a//b/", + expected_path: "/home/u/proj/a/b", + }, + PosixAbspathTestCase { + description: "a parent component climbs out of the working directory", + cwd: "/home/u/proj", + path: "../up/x", + expected_path: "/home/u/up/x", + }, + PosixAbspathTestCase { + description: "parents stop at the root", + cwd: "/home/u/proj", + path: "a/../../../../x", + expected_path: "/x", + }, + PosixAbspathTestCase { + description: "a root working directory", + cwd: "/", + path: "rel", + expected_path: "/rel", + }, + PosixAbspathTestCase { + description: "a parent of the root stays at the root", + cwd: "/", + path: "../x", + expected_path: "/x", + }, + PosixAbspathTestCase { + description: "a working directory with a trailing slash", + cwd: "/home/u/", + path: "x", + expected_path: "/home/u/x", + }, + PosixAbspathTestCase { + description: "exactly two leading slashes are kept", + cwd: "/home/u/proj", + path: "//net/x", + expected_path: "//net/x", + }, + PosixAbspathTestCase { + description: "three leading slashes collapse to one", + cwd: "/home/u/proj", + path: "///three/x", + expected_path: "/three/x", + }, + PosixAbspathTestCase { + description: "a double-slash working directory keeps its root", + cwd: "//net/share", + path: "x/../y", + expected_path: "//net/share/y", + }, + PosixAbspathTestCase { + description: "an empty path is the working directory", + cwd: "/home/u/proj", + path: "", + expected_path: "/home/u/proj", + }, + PosixAbspathTestCase { + description: "a dot is the working directory", + cwd: "/home/u/proj", + path: ".", + expected_path: "/home/u/proj", + }, + PosixAbspathTestCase { + description: "dots between components are dropped", + cwd: "/home/u/proj", + path: "a/./b/./", + expected_path: "/home/u/proj/a/b", + }, + PosixAbspathTestCase { + description: "climbing past the root gives the root", + cwd: "/home/u/proj", + path: "/a/b/../../..", + expected_path: "/", + }, + PosixAbspathTestCase { + description: "climbing past a double-slash root keeps it", + cwd: "/home/u/proj", + path: "//a/../..", + expected_path: "//", + }, + PosixAbspathTestCase { + description: "three dots are an ordinary name", + cwd: "/home/u/proj", + path: "x/.../y", + expected_path: "/home/u/proj/x/.../y", + }, + ]; + + for test_case in test_cases { + assert_eq!( + posix_abspath(test_case.cwd, test_case.path), + test_case.expected_path, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs index 126c6e31d9..69f579487f 100644 --- a/crates/sqlbuild-core/src/posix_path/tests/test_types.rs +++ b/crates/sqlbuild-core/src/posix_path/tests/test_types.rs @@ -7,3 +7,10 @@ pub(super) struct PosixPathTestCase { /// `suffix` under Python 3.12 and under Python 3.14. pub(super) expected_suffix: &'static str, } + +pub(super) struct PosixAbspathTestCase { + pub(super) description: &'static str, + pub(super) cwd: &'static str, + pub(super) path: &'static str, + pub(super) expected_path: &'static str, +} diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/header_columns.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/header_columns.rs new file mode 100644 index 0000000000..8991ccd664 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/header_columns.rs @@ -0,0 +1,54 @@ +//! Natively parsed header columns as their projected text, flags and counts. + +use sqlbuild_model_config::header_metadata::models::{ParsedAudit, ParsedColumn}; +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::model_files::_helpers::native_nodes::NativeNode; +use crate::model_files::models::{NativeAudit, NativeColumn}; + +pub(crate) const COLUMNS_KEY: &str = "columns"; + +pub(crate) static NULL_VALUE: AuthoredValue = AuthoredValue::Null; + +fn text(node: Option<&NativeNode<'_>>) -> Option { + node.and_then(AuthoredNode::text) +} + +fn flag(node: Option<&NativeNode<'_>>) -> Option { + match node.map(AuthoredNode::kind) { + Some(NodeKind::Bool(value)) => Some(value), + _ => None, + } +} + +/// A parsed column's leaves as the projected text, flags and counts. +pub(crate) fn native_column(column: &ParsedColumn>) -> NativeColumn { + NativeColumn { + name: column.name.text().unwrap_or_default(), + column_type: text(column.column_type.as_ref()), + nullable: flag(column.nullable.as_ref()), + description: text(column.description.as_ref()), + audits: column.audits.iter().map(native_audit).collect(), + migrate_from: text(column.migrate_from.as_ref()), + } +} + +fn native_audit(audit: &ParsedAudit>) -> NativeAudit { + NativeAudit { + definition_name: audit.definition_name.text().unwrap_or_default(), + arguments: audit + .arguments + .iter() + .map(|(key, value)| (key.text().unwrap_or_default(), value.authored())) + .collect(), + name: text(audit.name.as_ref()), + description: text(audit.description.as_ref()), + severity: text(audit.severity.as_ref()), + run_scope: text(audit.run_scope.as_ref()), + always_run: flag(audit.always_run.as_ref()), + thresholds: audit.thresholds, + minimum_samples: audit.minimum_samples.as_ref().map(NativeNode::authored), + evidence_limit: audit.evidence_limit.as_ref().map(NativeNode::authored), + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs index a39813b4a5..a6f8930a19 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/mod.rs @@ -1,2 +1,4 @@ +pub(crate) mod header_columns; +pub(crate) mod native_nodes; pub(crate) mod output_columns; pub(crate) mod parsing; diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs new file mode 100644 index 0000000000..7807df5a27 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/native_nodes.rs @@ -0,0 +1,217 @@ +//! Native authored header values read as the Python objects discovery projects them to. + +use std::cell::Cell; +use std::collections::HashMap; + +use sqlbuild_model_config::types::{AuthoredNode, NodeKind}; +use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; +use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; + +thread_local! { + /// Whether a read needed Python's `str` or `repr` of a value only Python can spell. + static INEXACT: Cell = const { Cell::new(false) }; +} + +/// One authored header value, or a mapping key, as its Python projection reads. +#[derive(Clone, Copy, Debug)] +pub(crate) enum NativeNode<'a> { + Value(&'a AuthoredValue), + Key(&'a str), +} + +/// Run `read` and report whether every Python spelling it needed was exact. +pub(crate) fn exactly(read: impl FnOnce() -> T) -> (T, bool) { + INEXACT.with(|flag| flag.set(false)); + let result: T = read(); + (result, !INEXACT.with(Cell::get)) +} + +fn inexact() -> String { + INEXACT.with(|flag| flag.set(true)); + String::new() +} + +/// A bare word's projection: Python's `_parse_word_value`. +fn word(text: &str) -> HeaderWord { + header_word_value(text) +} + +/// Whether integer `digits` (with an optional sign) is below zero, as `int(digits) < 0`. +fn negative(digits: &str) -> bool { + digits.starts_with('-') && digits[1..].bytes().any(|byte| byte != b'0') +} + +/// Python's `str(int(digits))`. +fn integer_text(digits: &str) -> String { + let unsigned: &str = digits.trim_start_matches(['+', '-']); + let trimmed: &str = unsigned.trim_start_matches('0'); + if trimmed.is_empty() { + return "0".to_owned(); + } + if negative(digits) { + format!("-{trimmed}") + } else { + trimmed.to_owned() + } +} + +impl<'a> NativeNode<'a> { + /// The text of a projected string, or `None`. + fn as_text(&self) -> Option<&'a str> { + match self { + Self::Key(key) => Some(key), + Self::Value(AuthoredValue::String(text)) => Some(text), + Self::Value(AuthoredValue::BareWord(text)) => match word(text) { + HeaderWord::Text => Some(text), + _ => None, + }, + Self::Value(_) => None, + } + } + + /// The digits of a projected integer, or `None`. + fn digits(&self) -> Option { + match self { + Self::Value(AuthoredValue::BareWord(text)) => match word(text) { + HeaderWord::Integer(digits) => Some(digits), + _ => None, + }, + _ => None, + } + } + + /// The owned authored value of a value node; a key projects as its string. + pub(crate) fn authored(&self) -> AuthoredValue { + match self { + Self::Key(key) => AuthoredValue::String((*key).to_owned()), + Self::Value(value) => (*value).clone(), + } + } +} + +impl AuthoredNode for NativeNode<'_> { + fn kind(&self) -> NodeKind { + let Self::Value(value) = self else { + return NodeKind::Str; + }; + match value { + AuthoredValue::Null => NodeKind::Null, + AuthoredValue::Boolean(flag) => NodeKind::Bool(*flag), + AuthoredValue::String(_) => NodeKind::Str, + AuthoredValue::List(_) => NodeKind::List, + AuthoredValue::Tuple(_) => NodeKind::Tuple, + AuthoredValue::Map(_) => NodeKind::Map, + AuthoredValue::BareWord(text) => match word(text) { + HeaderWord::Bool(flag) => NodeKind::Bool(flag), + HeaderWord::Null => NodeKind::Null, + HeaderWord::Integer(digits) => NodeKind::Int { + negative: negative(&digits), + }, + HeaderWord::Float(_) => NodeKind::Other, + HeaderWord::Text => NodeKind::Str, + }, + AuthoredValue::Set(_) + | AuthoredValue::TypedConstant(_) + | AuthoredValue::InlineSqlHook(_) + | AuthoredValue::NamedSqlHook(..) + | AuthoredValue::PythonHook(..) => NodeKind::Other, + } + } + + fn integer(&self) -> Option { + let digits: String = self.digits()?; + let Ok(number) = digits.parse::() else { + return None; + }; + Some(number) + } + + fn number(&self) -> Option { + if let Some(digits) = self.digits() { + let Ok(number) = digits.parse::() else { + return None; + }; + return Some(number); + } + match self { + Self::Value(AuthoredValue::BareWord(text)) => match word(text) { + HeaderWord::Float(number) => Some(number), + _ => None, + }, + _ => None, + } + } + + fn text(&self) -> Option { + self.as_text().map(str::to_owned) + } + + fn with_text(&self, read: impl FnOnce(&str) -> R) -> Option { + self.as_text().map(read) + } + + fn is_text(&self, text: &str) -> bool { + self.as_text() == Some(text) + } + + fn python_str(&self) -> String { + if let Some(text) = self.as_text() { + return text.to_owned(); + } + if let Some(digits) = self.digits() { + return integer_text(&digits); + } + match self.kind() { + NodeKind::Null => "None".to_owned(), + NodeKind::Bool(true) => "True".to_owned(), + NodeKind::Bool(false) => "False".to_owned(), + _ => inexact(), + } + } + + fn python_repr(&self) -> String { + match self.kind() { + NodeKind::Null | NodeKind::Bool(_) | NodeKind::Int { .. } => self.python_str(), + _ => inexact(), + } + } + + fn is_date_like(&self) -> bool { + false + } + + fn items(&self) -> Vec { + match self { + Self::Value(AuthoredValue::List(items) | AuthoredValue::Tuple(items)) => { + items.iter().map(Self::Value).collect() + } + _ => Vec::new(), + } + } + + fn entries(&self) -> Vec<(Self, Self)> { + match self { + Self::Value(AuthoredValue::Map(entries)) => dict_entries(entries) + .into_iter() + .map(|(key, value)| (Self::Key(key), Self::Value(value))) + .collect(), + _ => Vec::new(), + } + } +} + +/// Entries as the Python dict built from them holds them: first position, last value. +pub(crate) fn dict_entries(entries: &[(String, AuthoredValue)]) -> Vec<(&str, &AuthoredValue)> { + let mut kept: Vec<(&str, &AuthoredValue)> = Vec::with_capacity(entries.len()); + let mut positions: HashMap<&str, usize> = HashMap::with_capacity(entries.len()); + for (key, value) in entries { + match positions.get(key.as_str()) { + Some(&index) => kept[index].1 = value, + None => { + positions.insert(key, kept.len()); + kept.push((key, value)); + } + } + } + kept +} diff --git a/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs b/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs new file mode 100644 index 0000000000..2512a6e1d7 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/header_facts.rs @@ -0,0 +1,55 @@ +//! What a model's native header values show, read as their Python projection reads. + +use sqlbuild_model_config::config_presence::main::blocks_config_reuse::blocks_config_reuse; +use sqlbuild_model_config::config_presence::main::contains_template::contains_template; +use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; +use sqlbuild_model_config::header_metadata::main::parse_header_metadata::parse_header_metadata; +use sqlbuild_model_config::header_metadata::models::ParsedColumn; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::model_files::_helpers::header_columns::{COLUMNS_KEY, NULL_VALUE, native_column}; +use crate::model_files::_helpers::native_nodes::{NativeNode, dict_entries, exactly}; +use crate::model_files::models::{EntryFacts, HeaderFacts, NativeColumn}; + +/// The header facts of one model's native authored values; `path` names it in parse messages. +#[must_use] +pub fn header_facts(values: &[(String, AuthoredValue)], path: &str) -> HeaderFacts { + let entries: Vec<(&str, &AuthoredValue)> = dict_entries(values); + let facts: Vec<(String, EntryFacts)> = entries + .iter() + .map(|&(key, value)| { + let node: NativeNode<'_> = NativeNode::Value(value); + ( + key.to_owned(), + EntryFacts { + template: contains_template(&node), + macro_path: entry_macro_path(&NativeNode::Key(key), &node), + blocks_reuse: blocks_config_reuse(&node), + }, + ) + }) + .collect(); + let columns: Option> = entries + .iter() + .find(|(key, _)| *key == COLUMNS_KEY) + .map(|&(_, value)| value) + .filter(|value| **value != AuthoredValue::Null) + .and_then(|value| { + let (metadata, exact) = exactly(|| { + parse_header_metadata( + &NativeNode::Value(value), + &NativeNode::Value(&NULL_VALUE), + path, + ) + }); + let Ok(columns) = metadata.columns else { + return None; + }; + let columns: Vec>> = exact.then_some(columns)?; + Some(columns.iter().map(native_column).collect()) + }); + HeaderFacts { + entries: facts, + columns, + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/main/mod.rs b/crates/sqlbuild-discovery/src/model_files/main/mod.rs index 278eb30e86..eb6f515e35 100644 --- a/crates/sqlbuild-discovery/src/model_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/model_files/main/mod.rs @@ -1,2 +1,4 @@ pub mod discover_model_files; +pub mod header_facts; +pub mod model_file_stem; pub mod parse_model_text; diff --git a/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs b/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs new file mode 100644 index 0000000000..21db2a96a6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/main/model_file_stem.rs @@ -0,0 +1,13 @@ +//! A model's name: Python's `Path.stem` of its `/`-separated relative path. + +/// Python's `PurePath(relative_path).stem` for a `/`-separated relative path. +#[must_use] +pub fn model_file_stem(relative_path: &str) -> &str { + let name: &str = relative_path + .rsplit_once('/') + .map_or(relative_path, |(_, name)| name); + match name.rfind('.') { + Some(index) if index > 0 && index + 1 < name.len() => &name[..index], + _ => name, + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/models.rs b/crates/sqlbuild-discovery/src/model_files/models.rs index f23ad90701..a42b7646d4 100644 --- a/crates/sqlbuild-discovery/src/model_files/models.rs +++ b/crates/sqlbuild-discovery/src/model_files/models.rs @@ -1,6 +1,8 @@ //! Native results for SQL model files. use crate::models::LineColumnSpan; +use sqlbuild_model_config::config_presence::models::MacroPath; +use sqlbuild_model_config::header_metadata::models::ParsedThresholds; use sqlbuild_sqltext::compiler::models::AuthoredValue; /// What the Python facade asked for, with the header key sets Python owns during dual running. @@ -22,3 +24,60 @@ pub struct DiscoveredModelFile { pub output_column_locations: Vec<(String, LineColumnSpan)>, pub query_sql: String, } + +/// What one top-level header entry holds, as the config checks walk it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EntryFacts { + /// Whether a string in the value holds a `${...}` template. + pub template: bool, + /// Where the config macro walk finds a macro call in this entry. + pub macro_path: MacroPath, + /// Whether the value keeps the model from reusing another model's config. + pub blocks_reuse: bool, +} + +/// One model's header facts, read from its native authored values. +#[derive(Clone, Debug, PartialEq)] +pub struct HeaderFacts { + /// Each header key in the projected dict's order, with its facts. + pub entries: Vec<(String, EntryFacts)>, + /// The header's columns, when they parse exactly and without an error. + pub columns: Option>, +} + +impl HeaderFacts { + /// The facts of `key`'s value in the header. + #[must_use] + pub fn entry(&self, key: &str) -> Option<&EntryFacts> { + self.entries + .iter() + .find(|(name, _)| name == key) + .map(|(_, facts)| facts) + } +} + +/// One MODEL column parsed from native authored values. +#[derive(Clone, Debug, PartialEq)] +pub struct NativeColumn { + pub name: String, + pub column_type: Option, + pub nullable: Option, + pub description: Option, + pub audits: Vec, + pub migrate_from: Option, +} + +/// One audit instance parsed from native authored values. +#[derive(Clone, Debug, PartialEq)] +pub struct NativeAudit { + pub definition_name: String, + pub arguments: Vec<(String, AuthoredValue)>, + pub name: Option, + pub description: Option, + pub severity: Option, + pub run_scope: Option, + pub always_run: Option, + pub thresholds: Option, + pub minimum_samples: Option, + pub evidence_limit: Option, +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests.rs b/crates/sqlbuild-discovery/src/model_files/tests.rs index ab952d899c..ae2b46023d 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests.rs @@ -1,6 +1,10 @@ #[path = "tests/test_discover_model_files.rs"] mod discover_model_files; +#[path = "tests/test_header_facts.rs"] +mod header_facts; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_model_file_stem.rs"] +mod model_file_stem; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs index fc4979cb4d..6b8beef554 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/helpers.rs @@ -1,6 +1,13 @@ +use sqlbuild_model_config::config_presence::models::MacroPath; + +use crate::model_files::main::header_facts::header_facts; use crate::model_files::main::parse_model_text::parse_model_text; -use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; -use crate::model_files::tests::test_types::{Located, ModelSummary, Span}; +use crate::model_files::models::{ + DiscoveredModelFile, HeaderFacts, ModelFileOptions, NativeColumn, +}; +use crate::model_files::tests::test_types::{ + ColumnSummary, EntrySummary, Located, ModelSummary, Span, +}; use crate::models::LineColumnSpan; const FILE_PATH: &str = "/project/models/orders.sql"; @@ -73,3 +80,53 @@ fn owned(locations: &[(&str, Span)]) -> Vec { pub(super) fn failed(message: &str, help: Option<&str>) -> ModelSummary { Err((message.to_owned(), help.map(str::to_owned))) } + +/// The header facts of one model file parsed as `/project/models/orders.sql`. +pub(super) fn parsed_header_facts(contents: &str) -> HeaderFacts { + let model: DiscoveredModelFile = parse_model_text(FILE_PATH, contents.to_owned(), &options()) + .unwrap_or_else(|failure| panic!("{}", failure.message)); + header_facts(&model.header_values, "models/orders.sql") +} + +fn macro_path(path: &MacroPath) -> Option> { + let MacroPath::Found(path) = path else { + return None; + }; + Some(path.iter().map(String::as_str).collect()) +} + +/// `(key, template, macro path, blocks reuse)` of each header entry, in dict order. +pub(super) fn entry_summaries(facts: &HeaderFacts) -> Vec> { + facts + .entries + .iter() + .map(|(key, entry)| { + ( + key.as_str(), + entry.template, + macro_path(&entry.macro_path), + entry.blocks_reuse, + ) + }) + .collect() +} + +/// `(name, type, nullable, description, audit definitions)` of each parsed column. +pub(super) fn column_summaries(facts: &HeaderFacts) -> Option>> { + let columns: &Vec = facts.columns.as_ref()?; + Some(columns.iter().map(column_summary).collect()) +} + +fn column_summary(column: &NativeColumn) -> ColumnSummary<'_> { + ( + column.name.as_str(), + column.column_type.as_deref(), + column.nullable, + column.description.as_deref(), + column + .audits + .iter() + .map(|audit| audit.definition_name.as_str()) + .collect(), + ) +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs new file mode 100644 index 0000000000..7fe9587166 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_header_facts.rs @@ -0,0 +1,69 @@ +use crate::model_files::models::HeaderFacts; +use crate::model_files::tests::helpers::{column_summaries, entry_summaries, parsed_header_facts}; +use crate::model_files::tests::test_types::HeaderFactsTestCase; + +#[test] +fn given_model_headers_when_reading_native_facts_then_they_match_the_python_projection() { + let test_cases = [ + HeaderFactsTestCase { + description: "plain columns parse with their audits and block nothing", + contents: "MODEL (\n description 'Orders',\n materialized table,\n columns (\n id (type INTEGER, nullable false, audits [not_null, accepted_values (values [1, 2])]),\n amount (type DOUBLE, description 'Order amount'),\n ),\n);\nSELECT 1 AS id, 2.0 AS amount", + expected_entries: vec![ + ("description", false, None, false), + ("materialized", false, None, false), + ("columns", false, None, false), + ], + expected_columns: Some(vec![ + ( + "id", + Some("INTEGER"), + Some(false), + None, + vec!["not_null", "accepted_values"], + ), + ("amount", Some("DOUBLE"), None, Some("Order amount"), vec![]), + ]), + }, + HeaderFactsTestCase { + description: "templates and macro calls are found per entry with their path", + contents: "MODEL (\n description '${env.ORDERS}',\n columns (\n id (description '@cents(id)'),\n ),\n);\nSELECT 1 AS id", + expected_entries: vec![ + ("description", true, None, true), + ( + "columns", + false, + Some(vec!["columns", "id", "description"]), + true, + ), + ], + expected_columns: Some(vec![("id", None, None, Some("@cents(id)"), vec![])]), + }, + HeaderFactsTestCase { + description: "columns that fail to parse leave the Python parse to raise the error", + contents: "MODEL (\n columns (\n id (type INTEGER),\n ID (type INTEGER),\n ),\n);\nSELECT 1 AS id", + expected_entries: vec![("columns", false, None, false)], + expected_columns: None, + }, + HeaderFactsTestCase { + description: "a nullable column audited not_null is a parse error", + contents: "MODEL (\n columns (\n id (nullable true, audits [not_null]),\n ),\n);\nSELECT 1 AS id", + expected_entries: vec![("columns", false, None, false)], + expected_columns: None, + }, + ]; + for test_case in test_cases { + let facts: HeaderFacts = parsed_header_facts(test_case.contents); + assert_eq!( + entry_summaries(&facts), + test_case.expected_entries, + "{}", + test_case.description + ); + assert_eq!( + column_summaries(&facts), + test_case.expected_columns, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs new file mode 100644 index 0000000000..aae2d1b2c2 --- /dev/null +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_model_file_stem.rs @@ -0,0 +1,41 @@ +use crate::model_files::main::model_file_stem::model_file_stem; +use crate::model_files::tests::test_types::ModelFileStemTestCase; + +#[test] +fn given_relative_paths_when_naming_models_then_stems_match_python() { + let test_cases = [ + ModelFileStemTestCase { + description: "a final suffix is dropped", + relative_path: "models/orders.sql", + expected_stem: "orders", + }, + ModelFileStemTestCase { + description: "only the last suffix is dropped", + relative_path: "models/marts/orders.v2.sql", + expected_stem: "orders.v2", + }, + ModelFileStemTestCase { + description: "a leading dot is not a suffix", + relative_path: "models/.hidden", + expected_stem: ".hidden", + }, + ModelFileStemTestCase { + description: "a trailing dot is kept", + relative_path: "orders.", + expected_stem: "orders.", + }, + ModelFileStemTestCase { + description: "a name without a suffix is kept", + relative_path: "orders", + expected_stem: "orders", + }, + ]; + for test_case in test_cases { + assert_eq!( + model_file_stem(test_case.relative_path), + test_case.expected_stem, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs b/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs index 45db919f61..db361aab2d 100644 --- a/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/model_files/tests/test_types.rs @@ -20,3 +20,29 @@ pub(super) struct DeepModelHeaderTestCase { pub(super) depth: usize, pub(super) expected_summary: ModelSummary, } + +/// A model header and what its native header facts must read, as Python reads the header. +pub(super) struct HeaderFactsTestCase { + pub(super) description: &'static str, + pub(super) contents: &'static str, + pub(super) expected_entries: Vec>, + pub(super) expected_columns: Option>>, +} + +/// `(key, template, macro path, blocks reuse)` of one header entry. +pub(super) type EntrySummary<'a> = (&'a str, bool, Option>, bool); + +/// `(name, type, nullable, description, audit definitions)` of one parsed column. +pub(super) type ColumnSummary<'a> = ( + &'a str, + Option<&'a str>, + Option, + Option<&'a str>, + Vec<&'a str>, +); + +pub(super) struct ModelFileStemTestCase { + pub(super) description: &'static str, + pub(super) relative_path: &'static str, + pub(super) expected_stem: &'static str, +} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_walk.rs similarity index 69% rename from crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs rename to crates/sqlbuild-model-config/src/config_presence/_helpers/macro_walk.rs index d3ec7c465a..73e954912d 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_walk.rs @@ -1,16 +1,9 @@ -//! The first config field, in Python's walk order, whose string holds a macro call. +//! The walk `validate_no_macros_in_config_value` makes over nested config values. use crate::config_presence::_helpers::macro_calls::macro_call; use crate::config_presence::models::MacroPath; use crate::types::{AuthoredNode, NodeKind}; -/// Walk config values as `validate_no_macros_in_config_value` does, skipping the `skipped` keys. -pub fn first_macro_path(values: &N, skipped: &[&str]) -> MacroPath { - let mut entries = values.entries(); - entries.retain(|(key, _)| !skipped.iter().any(|name| key.is_text(name))); - walk_entries(&entries, &[]) -} - fn walk(node: &N, path: &[String]) -> MacroPath { match node.kind() { NodeKind::Str if node.with_text(macro_call).unwrap_or(false) => { @@ -30,7 +23,8 @@ fn walk(node: &N, path: &[String]) -> MacroPath { } } -fn walk_entries(entries: &[(N, N)], path: &[String]) -> MacroPath { +/// The first macro path among `entries`, string keys only, each below `path`. +pub(crate) fn walk_entries(entries: &[(N, N)], path: &[String]) -> MacroPath { for (key, value) in entries { if key.kind() != NodeKind::Str { continue; diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs index 1f894b4988..cb89f7ecbd 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod macro_calls; +pub(crate) mod macro_walk; pub(crate) mod reuse; pub(crate) mod scan; pub(crate) mod template_tokens; diff --git a/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs b/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs new file mode 100644 index 0000000000..1704dd4c74 --- /dev/null +++ b/crates/sqlbuild-model-config/src/config_presence/main/entry_macro_path.rs @@ -0,0 +1,10 @@ +//! Where Python's config walk finds a macro call within one top-level config entry. + +use crate::config_presence::_helpers::macro_walk::walk_entries; +use crate::config_presence::models::MacroPath; +use crate::types::AuthoredNode; + +/// The first macro path `validate_no_macros_in_config_value` reports for the entry `key: value`. +pub fn entry_macro_path(key: &N, value: &N) -> MacroPath { + walk_entries(&[(key.clone(), value.clone())], &[]) +} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs index e99609df7a..4a702d7a1d 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs @@ -1,4 +1,4 @@ pub mod blocks_config_reuse; pub mod contains_template; -pub mod first_macro_path; +pub mod entry_macro_path; pub mod mutable_config_value; diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs index a6fe61c021..d13d360240 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs @@ -2,8 +2,12 @@ use crate::config_presence::_helpers::macro_calls::macro_call; use crate::config_presence::_helpers::scan::scan; use crate::config_presence::main::blocks_config_reuse::blocks_config_reuse; use crate::config_presence::main::contains_template::contains_template; +use crate::config_presence::main::entry_macro_path::entry_macro_path; use crate::config_presence::main::mutable_config_value::mutable_config_value; -use crate::config_presence::tests::test_types::{PresenceTestCase, ReuseTestCase}; +use crate::config_presence::models::MacroPath; +use crate::config_presence::tests::test_types::{ + EntryMacroPathTestCase, PresenceTestCase, ReuseTestCase, +}; use crate::tests::test_types::Value; #[test] @@ -147,3 +151,51 @@ fn given_config_values_when_checking_reuse_then_python_answers_are_returned() { ); } } + +#[test] +fn given_config_entries_when_walking_for_macros_then_the_first_path_matches_python() { + let test_cases = [ + EntryMacroPathTestCase { + description: "a plain scalar holds no macro", + key: Value::Str("materialized"), + value: Value::Str("table"), + expected_path: MacroPath::Absent, + }, + EntryMacroPathTestCase { + description: "a nested mapping names every key down to the string", + key: Value::Str("columns"), + value: Value::Map(vec![( + Value::Str("id"), + Value::Map(vec![ + (Value::Str("type"), Value::Str("INTEGER")), + (Value::Str("description"), Value::Str("@cents(id)")), + ]), + )]), + expected_path: MacroPath::Found(vec![ + "columns".to_owned(), + "id".to_owned(), + "description".to_owned(), + ]), + }, + EntryMacroPathTestCase { + description: "list items report the path of the list", + key: Value::Str("tags"), + value: Value::List(vec![Value::Str("daily"), Value::Str("@later()")]), + expected_path: MacroPath::Found(vec!["tags".to_owned()]), + }, + EntryMacroPathTestCase { + description: "a mapping key that is not a string is skipped", + key: Value::Str("config"), + value: Value::Map(vec![(Value::Int(1), Value::Str("@skipped()"))]), + expected_path: MacroPath::Absent, + }, + ]; + for test_case in test_cases { + assert_eq!( + entry_macro_path(&test_case.key, &test_case.value), + test_case.expected_path, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs index 9b46ae4c75..e371ab2cef 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs @@ -1,3 +1,4 @@ +use crate::config_presence::models::MacroPath; use crate::tests::test_types::Value; /// One authored config value and what each recursive scan finds in it. @@ -15,3 +16,11 @@ pub(super) struct ReuseTestCase { pub(super) expected_blocks_reuse: bool, pub(super) expected_mutable: bool, } + +/// One top-level config entry and the macro path the walk reports for it. +pub(super) struct EntryMacroPathTestCase { + pub(super) description: &'static str, + pub(super) key: Value, + pub(super) value: Value, + pub(super) expected_path: MacroPath, +} diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 0ac50aadbd..5645b3946a 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -19,6 +19,7 @@ extension-module = ["pyo3/extension-module"] fensu-policy.workspace = true mimalloc.workspace = true pyo3.workspace = true +rayon.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-analysis.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs index f14f73c13e..e1012641f5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs @@ -204,11 +204,7 @@ pub(crate) fn model_name(model: &Bound<'_, PyAny>) -> PyResult { /// The model's compiled query SQL. pub(crate) fn query_sql<'py>(model: &Bound<'py, PyAny>) -> PyResult> { if let Ok(view) = model.downcast::() { - return Ok(view - .get() - .query_sql(model.py())? - .into_bound(model.py()) - .into_any()); + return Ok(view.get().query_sql(model.py())?.into_any()); } native_field(model, "query_sql") } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs index 9724ba59eb..fae39e9dc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use crate::bindings::_helpers::compile_command::reuse_recording::{ - ArtifactRecorder, artifact_written, + ArtifactRecorder, artifact_written, artifact_written_at, }; use crate::bindings::_helpers::compile_driver::compile_timings::record_compile_timing; use crate::bindings::_helpers::compile_outputs::artifacts::write_compiled_artifacts; @@ -17,6 +17,31 @@ pub(crate) struct ArtifactBatch<'a> { files: Vec<(PathBuf, Vec)>, } +/// `contents` with platform line endings. +fn platform_lines(contents: Vec) -> Vec { + if cfg!(windows) { + contents + .iter() + .flat_map(|byte| { + if *byte == b'\n' { + vec![b'\r', b'\n'] + } else { + vec![*byte] + } + }) + .collect() + } else { + contents + } +} + +/// `sql` with trailing whitespace replaced by one newline. +fn sql_artifact(sql: &str) -> Vec { + let trimmed: &str = + sql.trim_end_matches(sqlbuild_core::text::main::python_isspace::python_isspace); + format!("{trimmed}\n").into_bytes() +} + impl<'a> ArtifactBatch<'a> { pub(crate) fn new(check_existing: bool, recorder: ArtifactRecorder<'a>) -> Self { Self { @@ -28,20 +53,7 @@ impl<'a> ArtifactBatch<'a> { /// Queue one artifact with platform line endings, noting its bytes for compile reuse. pub(crate) fn queue(&mut self, path: &Bound<'_, PyAny>, contents: Vec) -> PyResult<()> { - let contents: Vec = if cfg!(windows) { - contents - .iter() - .flat_map(|byte| { - if *byte == b'\n' { - vec![b'\r', b'\n'] - } else { - vec![*byte] - } - }) - .collect() - } else { - contents - }; + let contents: Vec = platform_lines(contents); artifact_written(self.recorder, path, &contents)?; self.files.push((path.extract()?, contents)); Ok(()) @@ -49,9 +61,20 @@ impl<'a> ArtifactBatch<'a> { /// Queue `sql` with trailing whitespace replaced by one newline. pub(crate) fn queue_sql(&mut self, path: &Bound<'_, PyAny>, sql: &str) -> PyResult<()> { - let trimmed: &str = - sql.trim_end_matches(sqlbuild_core::text::main::python_isspace::python_isspace); - self.queue(path, format!("{trimmed}\n").into_bytes()) + self.queue(path, sql_artifact(sql)) + } + + /// `queue_sql` at a native path. + pub(crate) fn queue_sql_at( + &mut self, + py: Python<'_>, + path: PathBuf, + sql: &str, + ) -> PyResult<()> { + let contents: Vec = platform_lines(sql_artifact(sql)); + artifact_written_at(py, self.recorder, &path, &contents)?; + self.files.push((path, contents)); + Ok(()) } /// Write every queued artifact natively and empty the queue. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs index 00f2e51c8d..01eab6546c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -1,6 +1,7 @@ //! What one full compile printed and left in each artifact, and storing it for compile reuse. use std::collections::HashMap; +use std::path::{Path, PathBuf}; use std::sync::{Mutex, PoisonError}; use pyo3::exceptions::{PyException, PyOSError}; @@ -8,6 +9,8 @@ use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, P use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::posix_path::main::posix_abspath::posix_abspath; + use crate::bindings::_helpers::compile_command::constants::{ BYTES_PER_MEBIBYTE, REUSE_BYPASS_TIMING, REUSE_CHECK_TIMING, REUSE_HIT_TIMING, REUSE_LOGGER_NAME, REUSE_MISS_TIMING, REUSE_STORE_DONE_PREFIX, REUSE_STORE_NOTICE_BYTES, @@ -101,13 +104,30 @@ impl CompileArtifactWrites { /// Where a compile notes the artifacts it writes; `None` outside a recorded compile. pub(crate) type ArtifactRecorder<'a> = Option<&'a Py>; -/// `os.path.abspath(path)`. +/// `os.path.abspath(path)`, computed natively on POSIX for UTF-8 paths and working directories. fn absolute(path: &Bound<'_, PyAny>) -> PyResult { - imported(path.py(), "os.path", "abspath")? + absolute_path(path.py(), &path.extract::()?) +} + +/// `os.path.abspath(path)` of a native path. +fn absolute_path(py: Python<'_>, path: &Path) -> PyResult { + if cfg!(unix) + && let Some(found) = native_absolute(path) + { + return Ok(found); + } + imported(py, "os.path", "abspath")? .call1((path,))? .extract() } +fn native_absolute(path: &Path) -> Option { + let Ok(cwd) = std::env::current_dir() else { + return None; + }; + Some(posix_abspath(cwd.to_str()?, path.to_str()?)) +} + /// A new, empty recorder of one compile's artifact writes. pub(crate) fn new_artifact_recorder(py: Python<'_>) -> PyResult> { Py::new( @@ -134,6 +154,23 @@ pub(crate) fn artifact_written( Ok(()) } +/// `artifact_written` for a native path. +pub(crate) fn artifact_written_at( + py: Python<'_>, + recorder: ArtifactRecorder<'_>, + path: &Path, + contents: &[u8], +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder.get().record( + absolute_path(py, path)?, + RecordedArtifact::Written(artifact_digest(contents)), + ); + Ok(()) +} + /// Note an earlier artifact kept as-is after its size and mtime were validated. pub(crate) fn artifact_kept( recorder: ArtifactRecorder<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 230a79ca3e..c65aeee020 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -1,7 +1,7 @@ //! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. use std::collections::HashMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{ @@ -35,6 +35,7 @@ use crate::bindings::_helpers::compile_outputs::artifacts::{ use crate::bindings::_helpers::compile_outputs::runtime_target::{ compiled_sql_test_path, planned_sql_test_path, }; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::planner_entries::plan_output::sql_test_model_chain_names_by_key; @@ -85,6 +86,27 @@ fn model_output_path<'py>(relative_path: &Bound<'py, PyAny>) -> PyResult PathBuf { + let mut path: PathBuf = compiled.to_path_buf(); + let mut segments = relative_path + .split('/') + .filter(|segment| !segment.is_empty()) + .peekable(); + if segments.peek() != Some(&MODELS_DIR) { + path.push(MODELS_DIR); + } + for segment in segments { + path.push(segment); + } + path +} + +/// Each path object as a native path. +fn extracted(paths: &[Bound<'_, PyAny>]) -> PyResult> { + paths.iter().map(|path| path.extract()).collect() +} + /// `relative_path` under `functions//`, as a `.sql` file. fn function_output_path<'py>( relative_path: &Bound<'py, PyAny>, @@ -152,13 +174,9 @@ fn function_sql(adapter: &Bound<'_, PyAny>, function: &Bound<'_, PyAny>) -> PyRe } /// Delete compiled files no write group manages. -fn remove_stale(target_dir: &Bound<'_, PyAny>, managed: &[Bound<'_, PyAny>]) -> PyResult<()> { +fn remove_stale(target_dir: &Bound<'_, PyAny>, managed: Vec) -> PyResult<()> { let py: Python<'_> = target_dir.py(); let compiled: PathBuf = joined(target_dir, &[COMPILED_DIR])?.extract()?; - let managed: Vec = managed - .iter() - .map(|path| path.extract()) - .collect::>()?; record_compile_timing("stale_traversal_ms", || { remove_stale_artifacts(py, compiled, managed).map(|_| ()) }) @@ -289,7 +307,7 @@ fn write_planned_target<'py>( } batch.flush(py)?; if remove_stale_files { - remove_stale(target_dir, &managed)?; + remove_stale(target_dir, extracted(&managed)?)?; } write_manifest(target_dir, manifest, None)?; written_target( @@ -573,9 +591,18 @@ pub(crate) fn write_static_target<'py>( let remove_stale_files: bool = prepared_target(target_dir)?; let mut batch: ArtifactBatch = ArtifactBatch::new(remove_stale_files, prepared.recorder); let mut managed: Vec> = Vec::new(); + let mut managed_models: Vec = Vec::new(); let compiled: Bound<'py, PyAny> = joined(target_dir, &[COMPILED_DIR])?; + let compiled_dir: PathBuf = compiled.extract()?; for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; + if let Ok(native) = model.cast::() { + let (relative_path, query_sql) = native.get().output_texts(py)?; + let path: PathBuf = native_model_output_path(&compiled_dir, &relative_path); + batch.queue_sql_at(py, path.clone(), &query_sql)?; + managed_models.push(path); + continue; + } let path: Bound<'py, PyAny> = compiled.div(model_output_path(&native_field(&model, "relative_path")?)?)?; batch.queue_sql( @@ -616,7 +643,9 @@ pub(crate) fn write_static_target<'py>( write_static_tests(target_dir, (adapter, project), prepared, &mut batch)?; managed.extend(test_paths); if remove_stale_files { - remove_stale(target_dir, &managed)?; + let mut paths: Vec = managed_models; + paths.extend(extracted(&managed)?); + remove_stale(target_dir, paths)?; } write_manifest(target_dir, manifest, prepared.recorder)?; written_target( @@ -688,7 +717,7 @@ pub(crate) fn publish_staged_target<'py>( managed.push(path_class(py)?.call1((path,))?); } if existed { - remove_stale(target_dir, &managed)?; + remove_stale(target_dir, extracted(&managed)?)?; } } write_manifest(target_dir, manifest, recorder)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 9384073940..23f91269da 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::constants::RULE_FIX_AVAILABLE_NOTE; +use crate::bindings::_helpers::rules::early_builtin_rules::{EarlyRulesScope, EarlyRulesSettings}; use crate::bindings::_helpers::rules::early_sql_lint::EarlySqlLint; use crate::bindings::_helpers::rules::results::rules_progress; use crate::bindings::_helpers::rules::rule_dispatch::evaluate_rules; @@ -150,7 +151,10 @@ pub(crate) fn analyze<'py>( let enabled: bool = !(native_field(selection, "select")?.is_truthy()? || native_field(selection, "exclude")?.is_truthy()?); let early_lint: Bound<'py, EarlySqlLint> = Bound::new(py, EarlySqlLint::new(enabled))?; - let analysis: PyResult> = analyzed(py, &options, &early_lint); + let early_rules: EarlyRulesScope = EarlyRulesScope::new(); + let analysis: PyResult> = + analyzed(py, &options, (&early_lint, enabled.then_some(&early_rules))); + early_rules.stop(py); let _ = early_lint.borrow().stop(py); analysis } @@ -172,11 +176,11 @@ struct ContractPhase<'py> { cpu_ms: i64, } -/// `_analyze_compile_project`. +/// `_analyze_compile_project`; `early_rules` is armed once the dialect is known, when present. fn analyzed<'py>( py: Python<'py>, options: &AnalysisOptions<'py>, - early_lint: &Bound<'py, EarlySqlLint>, + (early_lint, early_rules): (&Bound<'py, EarlySqlLint>, Option<&EarlyRulesScope>), ) -> PyResult> { let driver: Driver<'_, 'py> = Driver { py, @@ -194,12 +198,22 @@ fn analyzed<'py>( } else { PyString::new(py, GENERIC_DIALECT).into_any() }; + if let Some(early_rules) = early_rules { + early_rules.arm(EarlyRulesSettings { + project_dir: options.project_dir.extract()?, + dialect: lint_dialect.extract()?, + no_sql_analysis: options.no_sql_validation.is_truthy()?, + }); + } let (graph, graph_ms): (Bound<'py, PyAny>, u64) = driver.graph((&discovered, &adapter), &lint_dialect)?; let project: Bound<'py, PyAny> = native_field(&graph, "project")?; let graph_failed: bool = any_error(&native_field(&project, "diagnostics")?)?; let mut early_lint_wait_ms: Bound<'py, PyAny> = 0_i64.into_pyobject(py)?.into_any(); if graph_failed { + if let Some(early_rules) = early_rules { + early_rules.stop(py); + } early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs index d02cd2cdc6..78eecf3de5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, HashMap}; +use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use pyo3::FromPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString, PyTuple, PyTupleMethods}; @@ -671,10 +672,7 @@ fn header_text(header: &Bound<'_, PyDict>, key: &str) -> PyResult pub(crate) fn model_owner<'py>( name: String, path: String, - schema: ( - &Bound<'py, PyTuple>, - (&Bound<'py, PyTuple>, &[SchemaColumnFact]), - ), + schema: (&Bound<'py, PyTuple>, (&HeaderColumns, &[SchemaColumnFact])), ) -> PyResult> { let (audits, (columns, facts)) = schema; let mut instances: Vec<(String, Option, Bound<'py, PyAny>)> = Vec::new(); @@ -686,7 +684,10 @@ pub(crate) fn model_owner<'py>( continue; } let column_owner: &str = fact.location_path.as_deref().unwrap_or(&path); - for audit in native_field(&columns.get_item(index)?, "audits")?.try_iter()? { + for audit in columns.specs[index] + .field(audits.py(), "audits")? + .try_iter()? + { let audit: Bound<'py, PyAny> = audit?; let owner_file: String = location_path(&audit)?.unwrap_or_else(|| column_owner.to_owned()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index 516f4988e8..83db2ce746 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -7,7 +7,7 @@ use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::assembly::analysis_session::main::resolved_cleaned_sql::resolved_cleaned_sql; use sqlbuild_analysis::assembly::analysis_session::models::{ ColumnFact, ContractProof, LineageFacts, LineageRow, ModelOutcome, PivotOutcome, @@ -22,10 +22,10 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ CompiledViewClasses, InferredRow, }; use crate::bindings::_helpers::compiled_project::model_input_view::ModelInputRecord; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::record_behaviours::lineage_facts::{ confidence_member, transform_kind_member, }; -use crate::bindings::_helpers::records::fields::field as native_field; const COMPILE_PHASE: &str = "compile"; const MODEL_RESOURCE: &str = "model"; @@ -135,7 +135,7 @@ pub(crate) fn analysis_record( if let PivotOutcome::Proof(proof) = contract { record.proof = Some(proof.clone()); } - record.binding = located(py, input, outcome)?; + record.binding = located(input, outcome)?; record.binding_validated = analysed.binding_validated; } if let Some(proof) = record.proof.as_ref().filter(|proof| proof.output_proven) { @@ -146,31 +146,27 @@ pub(crate) fn analysis_record( } /// The model's binding diagnostics at their authored locations. -fn located( - py: Python<'_>, - input: &ModelInputRecord, - outcome: &ModelOutcome, -) -> PyResult> { +fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult> { let rows: &[DiagnosticRow] = &outcome.analysis.binding_diagnostics; if rows.is_empty() { return Ok(Vec::new()); } let cleaned: Cow<'_, str> = resolved_cleaned_sql(outcome).map_err(compiler_error)?; - let model_file: &Bound<'_, PyAny> = input.model_file.bind(py); - let authored: String = native_field(model_file, "contents")?.extract()?; - let query: String = native_field(model_file, "query_sql")?.extract()?; + let model_file: &NativeModelFile = input.model_file.get(); + let authored: &str = model_file.contents_text(); + let query: &str = model_file.query_text(); let expansion = &input.expansion; let passes: Vec> = vec![ expansion.declaration_spans.clone(), expansion.macro_spans.clone(), ]; let source: BindingSource<'_> = BindingSource { - authored: &authored, - query: &query, + authored, + query, cleaned: &cleaned, expansion: expansion .authored - .then_some((expansion.expanded_sql.bind(py).to_str()?, passes.as_slice())), + .then_some((&*expansion.expanded_sql, passes.as_slice())), }; locate_binding_diagnostics(rows, &source).map_err(PyValueError::new_err) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index bfece36153..a9f7b3142f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -1,5 +1,7 @@ //! A compiled model, read from the native compiled project that owns it. +use std::sync::Arc; + use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, PyTupleMethods}; @@ -16,12 +18,14 @@ use crate::bindings::_helpers::compiled_project::compiled_model_analysis::{ AnalysisRecord, ProofArgument, SessionArgument, analysis_record, }; use crate::bindings::_helpers::compiled_project::model_input_view::{ - InputViewClasses, ModelInputRecord, model_relation_key_row, read_input, read_input_with_owner, - references, rejected_opt_out, schema_entry, view_classes, + InputViewClasses, ModelInputRecord, model_relation_key_row, query_sql_text, read_input, + read_input_with_owner, references, rejected_opt_out, schema_entry, view_classes, }; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::graph::project_graph::ResourceRow; +use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::relation_keys::RelationKeyRow; @@ -52,7 +56,7 @@ pub(crate) struct SqlRuleKey { } /// A schema's entry audits, column specs and each audited column's index and name. -pub(crate) type AuditedSchema = (Py, Py, Vec<(usize, String)>); +pub(crate) type AuditedSchema = (Py, Arc, Vec<(usize, String)>); /// `(name, type, nullability)` of one inferred output column. pub(crate) type InferredRow = (String, Option, String); @@ -355,10 +359,11 @@ impl NativeCompiledModel { /// The attached schema's declared `SchemaColumn`s, or `None` without a schema. pub(crate) fn declared_columns(&self, py: Python<'_>) -> PyResult>> { self.read(py, |record, py| { - Ok(record + record .schema .as_ref() - .map(|schema| schema.columns.clone_ref(py))) + .map(|schema| schema.columns.project(py).map(Bound::unbind)) + .transpose() }) } @@ -381,7 +386,7 @@ impl NativeCompiledModel { .collect(); Ok(Some(( schema.audits.clone_ref(py), - schema.columns.clone_ref(py), + schema.columns.clone(), audited, ))) }) @@ -404,8 +409,8 @@ impl NativeCompiledModel { let key = keyed .then(|| -> PyResult { Ok(SqlRuleKey { - authored_sql: facts.authored_sql.clone(), - query_sql: facts.query_sql.clone(), + authored_sql: facts.authored_sql.to_string(), + query_sql: facts.query_sql.to_string(), contract: compiled_of(record)? .analysis .proof @@ -422,6 +427,47 @@ impl NativeCompiledModel { }) } + /// The model's name, `str(relative_path)` and authored file text. + pub(crate) fn file_texts(&self, py: Python<'_>) -> PyResult<(String, String, Arc)> { + self.read(py, |record, py| { + let model_file: &NativeModelFile = record.model_file.get(); + Ok(( + model_name(record), + model_file.relative_text(py)?, + Arc::clone(model_file.contents_text()), + )) + }) + } + + /// The name, compiled query and authored file of the model, and whether it calls a function. + pub(crate) fn metadata_texts( + &self, + py: Python<'_>, + function_kinds: &[&str], + ) -> PyResult<(String, String, String, bool)> { + self.read(py, |record, _| { + Ok(( + model_name(record), + record.query_sql.to_string(), + record.model_file.get().contents_text().to_string(), + record + .references + .iter() + .any(|(kind, _, _, _)| function_kinds.contains(&kind.as_str())), + )) + }) + } + + /// `relative_path.as_posix()` and the compiled query, which compiled artifacts are written from. + pub(crate) fn output_texts(&self, py: Python<'_>) -> PyResult<(String, Arc)> { + self.read(py, |record, py| { + Ok(( + record.model_file.get().posix_path(py)?, + Arc::clone(&record.query_sql), + )) + }) + } + /// The model's authored file text and expanded query, from its recorded facts. pub(crate) fn sql_texts(&self, py: Python<'_>) -> PyResult<(String, String)> { let name: String = self.name(py)?; @@ -429,13 +475,15 @@ impl NativeCompiledModel { let facts: &CompiledModelFacts = model_facts(owner.facts(), &name).ok_or_else(|| { pyo3::exceptions::PyRuntimeError::new_err("the model was not assembled") })?; - Ok((facts.authored_sql.clone(), facts.query_sql.clone())) + Ok((facts.authored_sql.to_string(), facts.query_sql.to_string())) } - fn model_file_attribute(&self, py: Python<'_>, name: &str) -> PyResult> { - self.read(py, |record, py| { - Ok(native_field(record.model_file.bind(py), name)?.unbind()) - }) + fn model_file( + &self, + py: Python<'_>, + read: impl FnOnce(&NativeModelFile) -> PyResult, + ) -> PyResult { + self.read(py, |record, _| read(record.model_file.get())) } } @@ -506,12 +554,12 @@ impl NativeCompiledModel { #[getter] pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "relative_path") + self.model_file(py, |file| file.relative_path(py)) } #[getter] - pub(crate) fn query_sql(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, _| Ok(record.query_sql.clone_ref(py))) + pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| Ok(query_sql_text(py, record))) } #[getter] @@ -639,14 +687,14 @@ impl NativeCompiledModel { /// Inferred columns, else the attached schema's declared columns, else zero. #[getter] pub(crate) fn output_column_count(&self, py: Python<'_>) -> PyResult { - self.read(py, |record, py| { + self.read(py, |record, _py| { if let Some(rows) = &compiled_of(record)?.analysis.inferred { return Ok(rows.len()); } Ok(record .schema .as_ref() - .map_or(0, |schema| schema.columns.bind(py).len())) + .map_or(0, |schema| schema.columns.specs.len())) }) } @@ -706,23 +754,29 @@ impl NativeCompiledModel { } #[getter] - pub(crate) fn authored_sql(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "contents") + pub(crate) fn authored_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.model_file(py, |file| Ok(file.contents(py))) } #[getter] - pub(crate) fn authored_query_sql(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "query_sql") + pub(crate) fn authored_query_sql<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + self.model_file(py, |file| Ok(file.query_sql(py))) } #[getter] - pub(crate) fn output_column_locations(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "output_column_locations") + pub(crate) fn output_column_locations<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + self.model_file(py, |file| file.output_column_locations(py)) } #[getter] - pub(crate) fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult> { - self.model_file_attribute(py, "extract_implicit_alias_columns") + pub(crate) fn extract_implicit_alias_columns(&self, py: Python<'_>) -> PyResult { + self.model_file(py, |file| Ok(file.extract_implicit_alias_columns())) } #[getter] @@ -755,7 +809,7 @@ impl NativeCompiledModel { let classes: InputViewClasses<'_> = view_classes(&self.owner, py)?; self.read(py, |record, py| { let name: String = model_name(record); - let path: Bound<'_, PyAny> = native_field(record.model_file.bind(py), "relative_path")?; + let path: Bound<'_, PyAny> = record.model_file.get().path_objects(py)?.1; compiled_of(record)? .analysis .binding_tuple(py, &classes.compiled, (&name, &path)) @@ -844,10 +898,7 @@ pub(crate) fn model_graph_row( tags.push(tag?.str()?.extract()?); } } - let folder: String = native_field(record.model_file.bind(py), "relative_path")? - .getattr("parent")? - .str()? - .extract()?; + let folder: String = record.model_file.get().folder(py)?; Ok(( (MODEL_RESOURCE.to_owned(), model_name(record)), compiled.deps.clone(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index 63a4cce12f..a56686891e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -21,6 +21,7 @@ pub(crate) mod model_loop_inputs; pub(crate) mod model_loop_macros; pub(crate) mod model_loop_named; pub(crate) mod model_loop_opt_out; +pub(crate) mod model_loop_prepass; pub(crate) mod model_loop_python_hooks; pub(crate) mod model_loop_schema; pub(crate) mod model_loop_visibility; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index 0de3a2a3d4..9b56f17be5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -1,5 +1,7 @@ //! A model's compile input, read from the native compiled project that owns it. +use std::sync::Arc; + use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; @@ -12,6 +14,8 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; +use crate::bindings::_helpers::discovery::model_file_record::{NativeModelFile, projected_text}; +use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::SpanRow; use crate::bindings::_helpers::rules::relation_keys::{RelationKeyRow, relation_key_row}; @@ -33,7 +37,7 @@ pub(crate) struct InputViewClasses<'py> { /// A model's attached header schema; its specs carry user-authored audit arguments. pub(crate) struct SchemaRecord { pub(crate) description: Option, - pub(crate) columns: Py, + pub(crate) columns: Arc, pub(crate) audits: Py, pub(crate) dynamic_columns: Py, pub(crate) type_enforcement: bool, @@ -43,9 +47,9 @@ pub(crate) struct SchemaRecord { /// The expansion passes of a model's query, from authored text to expanded SQL. pub(crate) struct ExpansionRecord { /// The authored query below the header. - pub(crate) authored_sql: Py, + pub(crate) authored_sql: Arc, /// The declaration- and macro-expanded SQL before canonicalisation. - pub(crate) expanded_sql: Py, + pub(crate) expanded_sql: Arc, pub(crate) interpolation_spans: Vec, pub(crate) declaration_spans: Vec, pub(crate) macro_spans: Vec, @@ -58,12 +62,12 @@ pub(crate) struct ModelInputRecord { /// The model's name, its file stem. pub(crate) name: String, /// The discovered model file, owned by the discovery inputs. - pub(crate) model_file: Py, + pub(crate) model_file: Py, /// The resolved config, whose values hold user values. pub(crate) config: Py, - /// Immutable text shared with discovery, expansions and every Python-facing projection. - pub(crate) query_sql: Py, - pub(crate) macro_source_sql: Py, + /// Immutable text shared with discovery and expansions; Python reads build strings on demand. + pub(crate) query_sql: Arc, + pub(crate) macro_source_sql: Arc, pub(crate) references: Vec, pub(crate) schema: Option, pub(crate) sql_validation_enabled: bool, @@ -85,6 +89,7 @@ pub(crate) struct ModelInputRecord { /// A model input's Python attribute projections, each built once on first access. pub(crate) struct InputProjections { + query_sql: PyOnceLock>, references: PyOnceLock>, schema_entry: PyOnceLock>, sql_expansion: PyOnceLock>, @@ -94,6 +99,7 @@ pub(crate) struct InputProjections { impl Default for InputProjections { fn default() -> Self { Self { + query_sql: PyOnceLock::new(), references: PyOnceLock::new(), schema_entry: PyOnceLock::new(), sql_expansion: PyOnceLock::new(), @@ -166,7 +172,9 @@ pub(crate) fn view_classes<'py>( impl NativeModelInput { #[getter] pub(crate) fn model_file(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.model_file.clone_ref(py))) + self.read(py, |record, py| { + Ok(record.model_file.clone_ref(py).into_any()) + }) } #[getter] @@ -175,13 +183,15 @@ impl NativeModelInput { } #[getter] - pub(crate) fn query_sql(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.query_sql.clone_ref(py))) + pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| Ok(query_sql_text(py, record))) } #[getter] - pub(crate) fn macro_source_sql(&self, py: Python<'_>) -> PyResult> { - self.read(py, |record, py| Ok(record.macro_source_sql.clone_ref(py))) + pub(crate) fn macro_source_sql<'py>(&self, py: Python<'py>) -> PyResult> { + self.read(py, |record, py| { + Ok(PyString::new(py, &record.macro_source_sql)) + }) } #[getter] @@ -311,6 +321,18 @@ impl NativeModelInput { } } +/// The compiled query as the Python string every reader shares; the authored query's when equal. +pub(crate) fn query_sql_text<'py>( + py: Python<'py>, + record: &ModelInputRecord, +) -> Bound<'py, PyString> { + let model_file: &NativeModelFile = record.model_file.get(); + if Arc::ptr_eq(&record.query_sql, model_file.query_text()) { + return model_file.query_sql(py); + } + projected_text(py, &record.projections.query_sql, &record.query_sql) +} + /// The authored-to-expanded passes, when interpolation left the authored query unchanged. pub(crate) fn authored_expansion( classes: &InputViewClasses<'_>, @@ -417,7 +439,7 @@ pub(crate) fn schema_entry( fields.set_item("model_schema", &schema.model_schema)?; fields.set_item("description", &schema.description)?; fields.set_item("type_enforcement", schema.type_enforcement.then_some(true))?; - fields.set_item("columns", schema.columns.bind(py))?; + fields.set_item("columns", schema.columns.project(py)?)?; fields.set_item("dynamic_columns", schema.dynamic_columns.bind(py))?; fields.set_item("audits", schema.audits.bind(py))?; Ok(classes.schema_model_entry.call((), Some(&fields))?.unbind()) @@ -441,8 +463,8 @@ fn expansion_object( pass_objects.push(PyTuple::new(py, span_objects)?); } let fields: Bound<'_, PyDict> = PyDict::new(py); - fields.set_item("authored_sql", expansion.authored_sql.bind(py))?; - fields.set_item("expanded_sql", expansion.expanded_sql.bind(py))?; + fields.set_item("authored_sql", &*expansion.authored_sql)?; + fields.set_item("expanded_sql", &*expansion.expanded_sql)?; fields.set_item("passes", PyTuple::new(py, pass_objects)?)?; Ok(classes.sql_expansion.call((), Some(&fields))?.unbind()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index 01f51cf0ee..bdea620787 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -19,8 +19,8 @@ use crate::bindings::_helpers::compiled_project::model_loop_hooks::{ ExpandedHooks, HookModel, SqlHookClasses, expanded_hooks, }; use crate::bindings::_helpers::compiled_project::model_loop_host::{ - DeclarationScan, FinishedModel, InterpolationRow, ModelLoopRow, ModelPaths, RecordClasses, - RecordInputs, ReferenceRow, finished_record, + DeclarationScan, FinishedModel, InterpolationRow, ModelLoopRow, RecordClasses, RecordInputs, + ReferenceRow, finished_record, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ analysis_override, hook_presence, reference_row, @@ -34,6 +34,9 @@ use crate::bindings::_helpers::compiled_project::model_loop_named::{ use crate::bindings::_helpers::compiled_project::model_loop_opt_out::{ OptOut, OptOutModel, reject_unparseable_query, rejected_opt_out, }; +use crate::bindings::_helpers::compiled_project::model_loop_prepass::{ + NativeHeader, all_header_facts, native_header, +}; use crate::bindings::_helpers::compiled_project::model_loop_python_hooks::{ HookClasses, HookSignatures, }; @@ -54,7 +57,7 @@ use crate::bindings::_helpers::compiled_project::source_views::{ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ AttachedSqlTests, SqlTestSettings, TestFileRow, TestScope, }; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::model_config::config_errors::raised_config_exception; use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; @@ -86,11 +89,15 @@ use sqlbuild_attachments::cursor_intrinsics::main::validated_model_intrinsics::v use sqlbuild_attachments::cursor_intrinsics::models::IntrinsicModel; use sqlbuild_attachments::functions::models::KnownResources; use sqlbuild_attachments::test_targets::models::TargetCatalog; +use sqlbuild_discovery::model_files::models::HeaderFacts; use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references; use sqlbuild_sqltext::sql_references::models::ReferenceExtraction; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use std::collections::{HashMap, HashSet}; +use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::types::CompilerDetach; + const MATERIALIZED_KEY: &str = "materialized"; const CURSOR_KEY: &str = "cursor"; const INCREMENTAL_MATERIALIZATION: &str = "incremental"; @@ -267,14 +274,12 @@ fn prepared_rows<'py>( ) -> PyResult> { let hooks = &classes.sql_hooks; let mut sqls: Vec<(String, String)> = Vec::with_capacity(model_files.len()); + let mut paths: Vec = Vec::with_capacity(model_files.len()); for model_file in &model_files { let record: &NativeModelFile = model_file.get(); - let (file_path, _) = record.paths(py); - let (query_sql, _) = record.texts(py); - sqls.push(( - query_sql.to_str()?.to_owned(), - file_path.str()?.to_str()?.to_owned(), - )); + let texts: ModelFileTexts = record.path_texts(py)?; + sqls.push((record.query_text().to_string(), texts.file_path.clone())); + paths.push(texts); } let interpolated: Vec = interpolation_batch( InterpolationSources( @@ -286,8 +291,11 @@ fn prepared_rows<'py>( &sqls, )?; let mut rows: Vec = Vec::with_capacity(model_files.len()); - for ((model_file, (authored, _)), (rendered, spans, reads, error)) in - model_files.into_iter().zip(sqls).zip(interpolated) + for (((model_file, paths), (authored, _)), (rendered, spans, reads, error)) in model_files + .into_iter() + .zip(paths) + .zip(sqls) + .zip(interpolated) { let sql: String = rendered.unwrap_or(authored); let scan: Option = error.is_none().then(|| declaration_scan(&sql)); @@ -296,7 +304,7 @@ fn prepared_rows<'py>( .map(|(kind, name)| (kind.to_owned(), name)) .collect(); let interpolation: InterpolationRow = (sql, reads, error, spans); - rows.push(ModelLoopRow::new(py, (model_file, scan, interpolation))?); + rows.push(ModelLoopRow::new((model_file, paths), scan, interpolation)); } Ok(rows) } @@ -324,7 +332,7 @@ impl NativeModelLoop { let visibility: ModelVisibility<'py> = ModelVisibility::default(); let signatures: HookSignatures = HookSignatures::default(); for row in models { - let ModelPaths { + let ModelFileTexts { name, relative_path, file_path, @@ -342,8 +350,9 @@ impl NativeModelLoop { )?; let config: Bound<'py, PyAny> = configs.effective( &context, - record.header_value_dict(py), + &record.header_value_dict(py)?, (name, relative_path), + row.header.as_ref(), )?; let model: LoopModel<'_, 'py> = LoopModel { row: &row, @@ -381,7 +390,14 @@ impl NativeModelLoop { ) -> PyResult>> { let py: Python<'py> = slf.py(); let classes: LoopClasses<'py> = slf.get().loop_classes.bind(py).extract()?; - let models: Vec = prepared_rows(py, &classes, model_files)?; + let mut models: Vec = prepared_rows(py, &classes, model_files)?; + let headers: Vec> = models.iter().map(native_header).collect(); + let facts: Vec> = py + .compiler_detach(|| Ok(all_header_facts(&headers))) + .map_err(compiler_error)?; + for (row, facts) in models.iter_mut().zip(facts) { + row.header = facts; + } project .borrow_mut() .keep_attachment_loop(slf.clone().unbind(), host.clone().unbind()); @@ -531,7 +547,7 @@ impl NativeModelLoop { } = model; let ModelLoopRow { paths: - ModelPaths { + ModelFileTexts { name, posix_path, relative_path, @@ -542,6 +558,7 @@ impl NativeModelLoop { scan, interpolation, model_file, + header: header_facts, } = row; let values: Bound<'py, PyDict> = native_field(&config, VALUES_ATTRIBUTE)?.downcast_into()?; @@ -589,7 +606,7 @@ impl NativeModelLoop { relative_path, contents, authored_query, - authored: &interpolated == authored_query, + authored: interpolated.as_str() == &**authored_query, declaration_spans: &declarations.spliced.spans, }; Some(expanded_macros( @@ -634,7 +651,8 @@ impl NativeModelLoop { let scanned = match extract_sql_references(&query_sql, &self.syntax) { ReferenceExtraction::Extracted(scanned) => scanned, ReferenceExtraction::Failed(failure) => { - let authored: bool = &interpolated == authored_query && query_sql == macro_sql; + let authored: bool = + interpolated.as_str() == &**authored_query && query_sql == macro_sql; let origin: ScanFaultOrigin<'_> = ScanFaultOrigin { relative_path: posix_path, contents, @@ -696,6 +714,8 @@ impl NativeModelLoop { relative_path, visible: (&visible.declarations, &visible.consumer), schema: schema.as_ref(), + facts: header_facts.as_ref(), + model_file_handle: model_file, }; let header: AttachedHeader<'py> = header_schema(py, context, &source)?; let mut named_usages: Vec> = hook_usages; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs index 0a400c0a40..a5d1223038 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs @@ -16,6 +16,7 @@ use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::model_config_builder::{ BuiltConfig, ConfigBuild, NativeModelConfigBuilder, }; +use sqlbuild_discovery::model_files::models::HeaderFacts; const COLUMNS_KEY: &str = "columns"; const DESCRIPTION_KEY: &str = "description"; @@ -69,6 +70,7 @@ impl<'py> ModelConfigs<'py> { context: &LoopContext<'_, 'py>, header: &Bound<'py, PyDict>, model: (&str, &str), + facts: Option<&HeaderFacts>, ) -> PyResult> { let (name, relative_path) = model; let py: Python<'py> = header.py(); @@ -77,7 +79,7 @@ impl<'py> ModelConfigs<'py> { let matched: Option = builder .selected_path_default(py, relative_path)? .map_err(|error| config_error(error.bind(py).as_any(), None))?; - let metadata: Option> = reusable_metadata(header)?; + let metadata: Option> = reusable_metadata(header, facts)?; let reusable: bool = match metadata { Some(_) => self.reusable(context, &matched)?, None => false, @@ -90,7 +92,8 @@ impl<'py> ModelConfigs<'py> { values.update(metadata.as_mapping())?; return view(classes, &values, &cached.fields); } - let built: ConfigBuild<'py> = builder.built(py, header, matched.as_deref(), name)?; + let built: ConfigBuild<'py> = + builder.built(py, (header, facts), matched.as_deref(), name)?; let built: BuiltConfig<'py> = match built { ConfigBuild::Built(built) => built, ConfigBuild::Rejected(error) => { @@ -168,7 +171,10 @@ impl<'py> ModelConfigs<'py> { } /// `_reusable_metadata`: the header when only plain columns and description vary, else `None`. -fn reusable_metadata<'py>(header: &Bound<'py, PyDict>) -> PyResult>> { +fn reusable_metadata<'py>( + header: &Bound<'py, PyDict>, + facts: Option<&HeaderFacts>, +) -> PyResult>> { let py: Python<'py> = header.py(); if header.is_empty() { return Ok(Some(PyDict::new(py))); @@ -183,9 +189,14 @@ fn reusable_metadata<'py>(header: &Bound<'py, PyDict>) -> PyResult entry.blocks_reuse, + None => blocks_config_reuse(&PyNode(value)), + }; + if blocks { return Ok(None); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs index b82fe8f7bf..843076350c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs @@ -1,5 +1,7 @@ //! Header contract objects the model loop builds: factory audits, dynamic families, enum columns. +use std::sync::Arc; + use pyo3::BoundObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyTuple}; @@ -8,6 +10,7 @@ use sqlbuild_model_config::header_metadata::models::DynamicFamily; use crate::bindings::_helpers::compiled_project::model_loop::LoopContext; use crate::bindings::_helpers::compiled_project::model_loop_schema::DeclaredColumns; +use crate::bindings::_helpers::model_config::header_metadata::{HeaderColumn, HeaderColumns}; use crate::bindings::_helpers::records::fields::field as native_field; const CASES_ATTRIBUTE: &str = "cases"; @@ -44,15 +47,6 @@ const COPIED_CASE_FIELDS: [&str; 8] = [ "minimum_samples", "evidence_limit", ]; -/// `SchemaColumn` fields `dataclasses.replace` carries over unchanged. -const KEPT_COLUMN_FIELDS: [&str; 6] = [ - "name", - "nullable", - "description", - "meta", - "location", - "migrate_from", -]; const PIVOT_COLUMN_FIELD: &str = "pivot_column"; const VALUE_COLUMN_FIELD: &str = "value_column"; const AGGREGATE_FIELD: &str = "aggregate"; @@ -60,7 +54,7 @@ const NAME_PATTERN_FIELD: &str = "name_pattern"; /// A model's enum-typed columns resolved to scalar types, and the usages they generate. pub(crate) struct EnumColumns<'py> { - pub(crate) columns: Bound<'py, PyTuple>, + pub(crate) columns: Arc, pub(crate) facts: Vec, /// Column name to its enum declaration, as `CompileModelInput.enum_columns` holds them. pub(crate) declarations: Bound<'py, PyDict>, @@ -115,7 +109,7 @@ pub(crate) fn factory_audits<'py>( } /// `audit.name or audit.definition_name`. -pub(crate) fn audit_identity(audit: &Bound<'_, PyAny>) -> PyResult { +fn audit_identity(audit: &Bound<'_, PyAny>) -> PyResult { let name: Bound<'_, PyAny> = native_field(audit, NAME_FIELD)?; if name.is_truthy()? { return name.extract(); @@ -167,16 +161,24 @@ pub(crate) fn enum_columns<'py>( (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ), values: &Bound<'py, PyDict>, - columns: DeclaredColumns<'py>, + columns: DeclaredColumns, ) -> PyResult> { let py: Python<'py> = values.py(); let (enums, (declarations, consumer)) = visible; + if enums.is_empty() { + return Ok(EnumColumns { + columns: Arc::new(HeaderColumns::new(columns.specs)), + facts: columns.facts, + declarations: PyDict::new(py), + usages: Vec::new(), + }); + } let enforced: bool = match values.get_item(CONTRACT_KEY)? { Some(value) => value.eq(&context.classes.contract_enforced)?, None => false, }; let resolved: Bound<'py, PyDict> = PyDict::new(py); - let mut resolved_columns: Vec> = Vec::with_capacity(columns.specs.len()); + let mut resolved_columns: Vec = Vec::with_capacity(columns.specs.len()); let mut facts: Vec = Vec::with_capacity(columns.facts.len()); for (column, mut fact) in columns.specs.into_iter().zip(columns.facts) { let key: &str = fact.data_type.as_deref().unwrap_or_default(); @@ -186,7 +188,8 @@ pub(crate) fn enum_columns<'py>( continue; }; resolved.set_item(&fact.name, &declaration)?; - let mut audits: Vec> = native_field(&column, AUDITS_FIELD)? + let mut audits: Vec> = column + .field(py, AUDITS_FIELD)? .try_iter()? .collect::>()?; if enforced { @@ -206,19 +209,26 @@ pub(crate) fn enum_columns<'py>( } let scalar_type: Bound<'py, PyAny> = native_field(&declaration, SCALAR_TYPE_ATTRIBUTE)?; fact.data_type = scalar_type.extract()?; - let fields: Bound<'py, PyDict> = PyDict::new(py); - for field in KEPT_COLUMN_FIELDS { - fields.set_item(field, native_field(&column, field)?)?; - } - fields.set_item(TYPE_FIELD, scalar_type)?; - fields.set_item(AUDITS_FIELD, PyTuple::new(py, audits)?)?; - resolved_columns.push(context.contracts.schema_column.call((), Some(&fields))?); + resolved_columns.push(HeaderColumn::replaced( + &context.contracts.schema_column, + [ + column.field(py, "name")?, + scalar_type, + column.field(py, "nullable")?, + column.field(py, "description")?, + column.field(py, "meta")?, + PyTuple::new(py, audits)?.into_any(), + column.field(py, "location")?, + column.field(py, "migrate_from")?, + ], + [0, 2, 3, 4, 6, 7, 1, 5], + )?); facts.push(fact); } let usages: Vec> = generated_usages(context, declarations, consumer, &resolved)?; Ok(EnumColumns { - columns: PyTuple::new(py, resolved_columns)?, + columns: Arc::new(HeaderColumns::new(resolved_columns)), facts, declarations: resolved, usages, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs index 75fbcd9740..bc6456e4a9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs @@ -1,10 +1,13 @@ //! A model's MODEL header attached as its schema entry, by the native header parser. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use std::sync::Arc; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple, PyTupleMethods}; use sqlbuild_analysis::compiled_project::models::{ DynamicFamilyFact, ModelSchemaFact, SchemaColumnFact, }; +use sqlbuild_discovery::models::LineColumnSpan; use sqlbuild_model_config::header_metadata::main::audit_factory_references::audit_factory_references; use sqlbuild_model_config::header_metadata::main::merged_audit_identities::merged_audit_identities; use sqlbuild_model_config::header_metadata::main::parse_dynamic_columns::parse_dynamic_columns; @@ -16,7 +19,7 @@ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, compile_error, config_error, }; use crate::bindings::_helpers::compiled_project::model_loop_contracts::{ - EnumColumns, audit_identity, dynamic_family_objects, enum_columns, factory_audits, + EnumColumns, dynamic_family_objects, enum_columns, factory_audits, }; use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; use crate::bindings::_helpers::compiled_project::model_loop_schema::{ @@ -27,8 +30,12 @@ use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::column_migrations::column_migration_error; use crate::bindings::_helpers::model_config::config_errors::native_config_error; -use crate::bindings::_helpers::model_config::header_metadata::{audit_tuple, column_tuple}; +use crate::bindings::_helpers::model_config::header_metadata::{ + HeaderColumn, HeaderColumns, NativeColumnSpec, audit_instances, native_schema_column, + parsed_audit_identity, +}; use crate::bindings::_helpers::records::fields::field as native_field; +use sqlbuild_discovery::model_files::models::{HeaderFacts, NativeColumn}; const DESCRIPTION_KEY: &str = "description"; const COLUMNS_KEY: &str = "columns"; @@ -47,6 +54,9 @@ pub(crate) struct HeaderSource<'a, 'py> { /// The model's visible declarations and the identity its usages are recorded for. pub(crate) visible: (&'a Bound<'py, PyAny>, &'a Bound<'py, PyAny>), pub(crate) schema: Option<&'a ModelSchema<'py>>, + /// What the pre-pass read from the model's native header. + pub(crate) facts: Option<&'a HeaderFacts>, + pub(crate) model_file_handle: &'a Py, } /// The header's description and, when it declares any schema metadata, its entry. @@ -57,7 +67,7 @@ pub(crate) struct AttachedHeader<'py> { /// A header schema entry's Python contract objects, enum columns and retained fact. pub(crate) struct HeaderSchema<'py> { - pub(crate) columns: Bound<'py, PyTuple>, + pub(crate) columns: Arc, pub(crate) audits: Bound<'py, PyTuple>, pub(crate) dynamic_columns: Bound<'py, PyTuple>, pub(crate) type_enforcement: bool, @@ -96,19 +106,28 @@ pub(crate) fn header_schema<'py>( let none: Bound<'py, PyAny> = py.None().into_bound(py); let node = |value: Option>| PyNode(value.unwrap_or_else(|| none.clone())); let model: (&str, &str) = (source.name, source.relative_path); - let metadata = parse_header_metadata(&node(columns), &node(audits), source.relative_path); + let native: Option<&[NativeColumn]> = native_columns(context, source, columns.as_ref())?; + let python_columns: Option> = if native.is_some() { None } else { columns }; + let metadata = + parse_header_metadata(&node(python_columns), &node(audits), source.relative_path); let parsed_columns = metadata .columns .map_err(|error| raised_config_error(py, error))?; - let locations: Bound<'py, PyDict> = source.model_file.header_locations(py)?; - let model_path: &Bound<'py, PyAny> = source.model_file.paths(py).1; - let local_columns: DeclaredColumns<'py> = DeclaredColumns { - specs: column_tuple(py, context.contracts, &parsed_columns, &locations)? - .iter() - .collect(), - facts: local_column_facts(&parsed_columns)?, + let model_path: &Bound<'py, PyAny> = &source.model_file.path_objects(py)?.1; + let local_columns: DeclaredColumns = match native { + Some(native) => native_declared_columns(context, source, native)?, + None => { + let locations: Bound<'py, PyDict> = source.model_file.header_locations(py)?; + DeclaredColumns { + specs: parsed_columns + .iter() + .map(|column| native_schema_column(py, context.contracts, column, &locations)) + .collect::>()?, + facts: local_column_facts(&parsed_columns)?, + } + } }; - let columns: DeclaredColumns<'py> = match source.schema { + let columns: DeclaredColumns = match source.schema { Some(schema) => merged_columns( context, schema, @@ -121,14 +140,20 @@ pub(crate) fn header_schema<'py>( let parsed_audits = metadata .audits .map_err(|error| raised_config_error(py, error))?; - let available: Vec = context.classes.audit_factories.keys().extract()?; - let references: Vec = audit_factory_references(&node(factories), model, &available) - .map_err(|error| raised_config_error(py, error))?; - let direct_audits: Bound<'py, PyTuple> = audit_tuple(py, context.contracts, &parsed_audits)?; + let references: Vec = match factories { + Some(factories) => { + let available: Vec = context.classes.audit_factories.keys().extract()?; + audit_factory_references(&PyNode(factories), model, &available) + .map_err(|error| raised_config_error(py, error))? + } + None => Vec::new(), + }; + let mut direct_audits: Vec> = + audit_instances(py, context.contracts, &parsed_audits)?; let (generated, generated_identities) = factory_audits(py, context, &references)?; - let direct_identities: Vec = direct_audits + let direct_identities: Vec = parsed_audits .iter() - .map(|audit| audit_identity(&audit)) + .map(parsed_audit_identity) .collect::>()?; merged_audit_identities(&direct_identities, &generated_identities, model) .map_err(|error| raised_config_error(py, error))?; @@ -150,10 +175,8 @@ pub(crate) fn header_schema<'py>( .iter() .any(|column| column.data_type.is_some()); let family_facts: Vec = families.iter().map(family_fact).collect(); - let audits: Bound<'py, PyTuple> = PyTuple::new( - py, - direct_audits.iter().chain(generated).collect::>(), - )?; + direct_audits.extend(generated); + let audits: Bound<'py, PyTuple> = PyTuple::new(py, direct_audits)?; let families: Vec> = dynamic_family_objects(py, context, &families)?; let enums: Bound<'py, PyDict> = native_field(source.visible.0, ENUMS_FIELD)?.downcast_into()?; let enums: Bound<'py, PyDict> = match source.schema { @@ -188,6 +211,68 @@ pub(crate) fn header_schema<'py>( }) } +/// The natively parsed header columns, when the config's columns are the header's own value. +fn native_columns<'a>( + context: &LoopContext<'_, '_>, + source: &HeaderSource<'a, '_>, + columns: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + let (Some(facts), Some(columns)) = (source.facts, columns) else { + return Ok(None); + }; + if !context.contracts.uses_default_classes()? { + return Ok(None); + } + let Some(parsed) = &facts.columns else { + return Ok(None); + }; + let header: Bound<'_, PyDict> = source.model_file.header_value_dict(columns.py())?; + let own: bool = header + .get_item(COLUMNS_KEY)? + .is_some_and(|value| value.is(columns)); + Ok(own.then_some(parsed.as_slice())) +} + +/// Native header columns as declared columns, their records built only when Python reads them. +fn native_declared_columns( + context: &LoopContext<'_, '_>, + source: &HeaderSource<'_, '_>, + columns: &[NativeColumn], +) -> PyResult { + let py: Python<'_> = context.host.py(); + let rows: &[(String, LineColumnSpan)] = source.model_file.header_location_rows(); + let mut specs: Vec = Vec::with_capacity(columns.len()); + let mut facts: Vec = Vec::with_capacity(columns.len()); + for column in columns { + let location: Option = rows + .iter() + .rev() + .find(|(name, _)| *name == column.name) + .map(|(_, span)| *span); + facts.push(SchemaColumnFact { + name: column.name.clone(), + data_type: column.column_type.clone(), + nullable: column.nullable, + described: column.description.is_some(), + migrate_from: column.migrate_from.clone(), + audit_definitions: column + .audits + .iter() + .map(|audit| audit.definition_name.clone()) + .collect(), + location_path: None, + }); + specs.push(HeaderColumn::native( + &context.contracts.schema_column, + NativeColumnSpec { + column: column.clone(), + location: location.map(|span| (source.model_file_handle.clone_ref(py), span)), + }, + )?); + } + Ok(DeclaredColumns { specs, facts }) +} + /// The header's dynamic column families, each `name_pattern` compiled natively. fn dynamic_families( py: Python<'_>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 478af8e829..79036fdaeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -1,8 +1,10 @@ //! The Python host the loop hands finished models and user-code steps to. +use std::sync::Arc; + use pyo3::FromPyObject; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use sqlbuild_analysis::compiled_project::main::plain_model_facts::plain_model_facts; use sqlbuild_analysis::compiled_project::models::{ CompiledModelFacts, ModelConfigFacts, PlainModel, @@ -21,13 +23,14 @@ use crate::bindings::_helpers::compiled_project::project::{ use crate::bindings::_helpers::compiled_project::project::{ DeclarationPy, config_values, declarations, }; -use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; +use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::{ ExpandedDeclarations, ScannedReference, span_rows, }; use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::ExpansionOutput; +use sqlbuild_discovery::model_files::models::HeaderFacts; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; const LAYER_SCHEMA_ATTRIBUTE: &str = "layer_schema"; @@ -37,57 +40,41 @@ const CONSTANTS_ARGUMENT: &str = "constants"; const LOCAL_ENUMS_ATTRIBUTE: &str = "local_enums"; const LOCAL_CONSTANTS_ATTRIBUTE: &str = "local_constants"; const NAME_ATTRIBUTE: &str = "name"; -const STEM_ATTRIBUTE: &str = "stem"; -const AS_POSIX_METHOD: &str = "as_posix"; -/// What the host hands the loop for one model: its file, declaration scan and interpolation. -pub(crate) type ModelLoopInput = ( - Py, - Option, - InterpolationRow, -); /// A model's scanned declaration references and the scan's stop marker. pub(crate) type DeclarationScan = (Vec, Option); -/// The names a model's file is known by. -pub(crate) struct ModelPaths { - pub(crate) name: String, - pub(crate) posix_path: String, - pub(crate) relative_path: String, - pub(crate) file_path: String, -} - /// One model the loop builds: its discovered file and what the host prepared for it. pub(crate) struct ModelLoopRow { - pub(crate) paths: ModelPaths, - pub(crate) authored_query: String, - pub(crate) contents: String, + pub(crate) paths: ModelFileTexts, + pub(crate) authored_query: Arc, + pub(crate) contents: Arc, pub(crate) scan: Option, /// The `@@`-interpolated query, its template reads, its error, and its substitution spans. pub(crate) interpolation: InterpolationRow, /// The discovered model file, owned by the discovery inputs. pub(crate) model_file: Py, + /// What the pre-pass read from the model's native header, when it has one. + pub(crate) header: Option, } impl ModelLoopRow { - pub(crate) fn new(py: Python<'_>, input: ModelLoopInput) -> PyResult { - let (model_file, scan, interpolation) = input; + pub(crate) fn new( + file: (Py, ModelFileTexts), + scan: Option, + interpolation: InterpolationRow, + ) -> Self { + let (model_file, paths) = file; let record: &NativeModelFile = model_file.get(); - let (file_path, relative_path) = record.paths(py); - let (query_sql, contents) = record.texts(py); - Ok(Self { - paths: ModelPaths { - name: native_field(file_path, STEM_ATTRIBUTE)?.extract()?, - posix_path: relative_path.call_method0(AS_POSIX_METHOD)?.extract()?, - relative_path: relative_path.str()?.to_str()?.to_owned(), - file_path: file_path.str()?.to_str()?.to_owned(), - }, - authored_query: query_sql.to_str()?.to_owned(), - contents: contents.to_str()?.to_owned(), + Self { + paths, + authored_query: record.query_text().clone(), + contents: record.contents_text().clone(), scan, interpolation, model_file, - }) + header: None, + } } } /// `(interpolated query, template reads, error, substitution spans)`. @@ -138,7 +125,7 @@ pub(crate) fn finished_record<'py>( let py: Python<'py> = inputs.visible.py(); let ModelLoopRow { paths: - ModelPaths { + ModelFileTexts { name, posix_path: relative_path, .. @@ -218,27 +205,21 @@ pub(crate) fn finished_record<'py>( }; let schema: Option = finished.header.schema.as_ref().map(|schema| SchemaRecord { description: finished.header.description.clone(), - columns: schema.columns.clone().unbind(), + columns: schema.columns.clone(), audits: schema.audits.clone().unbind(), dynamic_columns: schema.dynamic_columns.clone().unbind(), type_enforcement: schema.type_enforcement, model_schema: schema.model_schema.clone(), }); - let discovered_query: Py = model_file.get().texts(py).0.clone().unbind(); - let authored: Py = shared_sql(py, &authored_query_sql, &[&discovered_query])?; - let macro_source: Py = - shared_sql(py, &finished.declarations.spliced.sql, &[&authored])?; - let expanded: Py = shared_sql(py, ¯o_sql, &[&authored, ¯o_source])?; - let query: Py = shared_sql( - py, - &finished.query_sql, - &[&authored, ¯o_source, &expanded], - )?; + let authored: Arc = authored_query_sql.clone(); + let macro_source: Arc = shared_sql(&finished.declarations.spliced.sql, &[&authored]); + let expanded: Arc = shared_sql(¯o_sql, &[&authored, ¯o_source]); + let query: Arc = shared_sql(&finished.query_sql, &[&authored, ¯o_source, &expanded]); let record: ModelInputRecord = ModelInputRecord { name: name.clone(), - model_file: model_file.clone_ref(py).into_any(), + model_file, config: config.unbind(), - query_sql: query, + query_sql: Arc::clone(&query), macro_source_sql: macro_source, references: finished.references.clone(), schema, @@ -256,7 +237,7 @@ pub(crate) fn finished_record<'py>( macro_usages: PyTuple::new(py, macro_usages)?.unbind(), declaration_usages: PyTuple::new(py, unique(py, declaration_usages)?)?.unbind(), expansion: ExpansionRecord { - authored: interpolated_sql == authored_query_sql, + authored: *interpolated_sql == *authored_query_sql, authored_sql: authored, expanded_sql: expanded, interpolation_spans, @@ -270,7 +251,7 @@ pub(crate) fn finished_record<'py>( let facts: CompiledModelFacts = plain_model_facts(PlainModel { name, relative_path, - query_sql: finished.query_sql, + query_sql: query, authored_query_sql, authored_sql, config: ModelConfigFacts { @@ -296,13 +277,13 @@ pub(crate) fn finished_record<'py>( } /// Reuse an identical immutable spelling already owned by this model. -fn shared_sql(py: Python<'_>, sql: &str, candidates: &[&Py]) -> PyResult> { +fn shared_sql(sql: &str, candidates: &[&Arc]) -> Arc { for candidate in candidates { - if candidate.bind(py).to_str()? == sql { - return Ok(candidate.clone_ref(py)); + if ***candidate == *sql { + return Arc::clone(candidate); } } - Ok(PyString::new(py, sql).unbind()) + Arc::from(sql) } /// A dict's values in insertion order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs new file mode 100644 index 0000000000..d5c9b515c0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_prepass.rs @@ -0,0 +1,69 @@ +//! Each model's native header facts, read before the ordered loop on a fixed pool off the GIL. + +use std::sync::{Arc, OnceLock}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods}; +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use rayon::{ThreadPool, ThreadPoolBuildError, ThreadPoolBuilder}; +use sqlbuild_discovery::model_files::main::header_facts::header_facts; +use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::bindings::_helpers::compiled_project::model_loop_host::ModelLoopRow; + +/// The compile pools' fixed worker count. +const PREPASS_WORKERS: usize = 4; +const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; + +static POOL: OnceLock> = OnceLock::new(); + +/// A model's native header values and the relative path parse messages name. +pub(crate) type NativeHeader = (Arc<[(String, AuthoredValue)]>, String); + +/// An entry's facts only when its value is the header's own, so replaced values read from Python. +pub(crate) fn own_entry<'a, 'py>( + facts: &'a HeaderFacts, + header: &Bound<'py, PyDict>, + (key, value): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + let Ok(name) = key.downcast::() else { + return Ok(None); + }; + let Ok(name) = name.to_str() else { + return Ok(None); + }; + let Some(entry) = facts.entry(name) else { + return Ok(None); + }; + Ok(header + .get_item(name)? + .filter(|own| own.is(value)) + .map(|_| entry)) +} + +/// Each model's header facts on the compile pool, or in turn when the pool cannot start. +pub(crate) fn all_header_facts(headers: &[Option]) -> Vec> { + let pool: &Result = POOL.get_or_init(|| { + ThreadPoolBuilder::new() + .num_threads(PREPASS_WORKERS) + .stack_size(WORKER_STACK_BYTES) + .thread_name(|index| format!("sqlbuild-model-prepass-{index}")) + .build() + }); + match pool { + Ok(pool) => pool.install(|| headers.par_iter().map(facts_of).collect()), + Err(_) => headers.iter().map(facts_of).collect(), + } +} + +fn facts_of(header: &Option) -> Option { + let (values, path) = header.as_ref()?; + Some(header_facts(values, path)) +} + +/// The row's native header values and relative path, unless Python constructed its file. +pub(crate) fn native_header(row: &ModelLoopRow) -> Option { + let values: &Arc<[(String, AuthoredValue)]> = row.model_file.get().native_header_values()?; + Some((Arc::clone(values), row.paths.relative_path.clone())) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs index 0bcfd48851..8931d52454 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs @@ -1,5 +1,6 @@ //! A model's `model_schema`: resolved, checked for visibility, and merged into its header columns. +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; @@ -7,6 +8,7 @@ use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compile_error}; use crate::bindings::_helpers::compiled_project::model_loop_inputs::present; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fact; +use crate::bindings::_helpers::model_config::header_metadata::HeaderColumn; use crate::bindings::_helpers::records::fields::field as native_field; const MODEL_SCHEMA_KEY: &str = "model_schema"; @@ -15,21 +17,11 @@ const TYPE_FIELD: &str = "type"; const NULLABLE_FIELD: &str = "nullable"; const DESCRIPTION_FIELD: &str = "description"; const AUDITS_FIELD: &str = "audits"; -const MIGRATE_FROM_FIELD: &str = "migrate_from"; const LOCATION_FIELD: &str = "location"; const PATH_FIELD: &str = "path"; const LINE_FIELD: &str = "line"; const COLUMNS_FIELD: &str = "columns"; const STRIP_METHOD: &str = "strip"; -/// `SchemaColumn` fields a named-schema column keeps when a header column augments it. -const KEPT_NAMED_FIELDS: [&str; 6] = [ - "name", - "type", - "nullable", - "description", - "meta", - "location", -]; /// The reusable schema a model names: its columns, name, description and enum column types. pub(crate) struct ModelSchema<'py> { @@ -101,8 +93,8 @@ pub(crate) fn schema_enums<'py>( } /// Schema columns as Python specs paired with their native facts, in declaration order. -pub(crate) struct DeclaredColumns<'py> { - pub(crate) specs: Vec>, +pub(crate) struct DeclaredColumns { + pub(crate) specs: Vec, pub(crate) facts: Vec, } @@ -110,11 +102,11 @@ pub(crate) struct DeclaredColumns<'py> { pub(crate) fn merged_columns<'py>( context: &LoopContext<'_, 'py>, schema: &ModelSchema<'py>, - local: DeclaredColumns<'py>, + local: DeclaredColumns, model: (&str, &str, &Bound<'py, PyAny>), -) -> PyResult> { +) -> PyResult { let (model_name, relative_path, model_path) = model; - let mut merged: DeclaredColumns<'py> = DeclaredColumns { + let mut merged: DeclaredColumns = DeclaredColumns { specs: Vec::new(), facts: Vec::new(), }; @@ -123,14 +115,14 @@ pub(crate) fn merged_columns<'py>( merged .facts .push(python_column_fact(&column, Some(model_path))?); - merged.specs.push(column); + merged.specs.push(HeaderColumn::from_python(column)?); } let keys: Vec = merged .facts .iter() .map(|fact| fact.name.to_lowercase()) .collect(); - let mut additional: DeclaredColumns<'py> = DeclaredColumns { + let mut additional: DeclaredColumns = DeclaredColumns { specs: Vec::new(), facts: Vec::new(), }; @@ -162,15 +154,17 @@ pub(crate) fn merged_columns<'py>( fn augmented<'py>( context: &LoopContext<'_, 'py>, - (named, named_fact): (&Bound<'py, PyAny>, &SchemaColumnFact), - (local, local_fact): (&Bound<'py, PyAny>, SchemaColumnFact), -) -> PyResult<(Bound<'py, PyAny>, SchemaColumnFact)> { - let py: Python<'py> = named.py(); - let mut audits: Vec> = native_field(named, AUDITS_FIELD)? + (named, named_fact): (&HeaderColumn, &SchemaColumnFact), + (local, local_fact): (&HeaderColumn, SchemaColumnFact), +) -> PyResult<(HeaderColumn, SchemaColumnFact)> { + let py: Python<'py> = context.contracts.schema_column.py(); + let mut audits: Vec> = named + .field(py, AUDITS_FIELD)? .try_iter()? .collect::>()?; let mut fact: SchemaColumnFact = named_fact.clone(); - for (audit, definition) in native_field(local, AUDITS_FIELD)? + for (audit, definition) in local + .field(py, AUDITS_FIELD)? .try_iter()? .zip(local_fact.audit_definitions) { @@ -187,15 +181,23 @@ fn augmented<'py>( fact.audit_definitions.push(definition); } } - let fields: Bound<'py, PyDict> = PyDict::new(py); - for field in KEPT_NAMED_FIELDS { - fields.set_item(field, native_field(named, field)?)?; - } - fields.set_item(AUDITS_FIELD, PyTuple::new(py, audits)?)?; - fields.set_item(MIGRATE_FROM_FIELD, &local_fact.migrate_from)?; + let migrate_from = local_fact.migrate_from.clone().into_bound_py_any(py)?; fact.migrate_from = local_fact.migrate_from; Ok(( - context.contracts.schema_column.call((), Some(&fields))?, + HeaderColumn::replaced( + &context.contracts.schema_column, + [ + named.field(py, NAME_FIELD)?, + named.field(py, TYPE_FIELD)?, + named.field(py, NULLABLE_FIELD)?, + named.field(py, DESCRIPTION_FIELD)?, + named.field(py, "meta")?, + PyTuple::new(py, audits)?.into_any(), + named.field(py, LOCATION_FIELD)?, + migrate_from, + ], + [0, 1, 2, 3, 4, 6, 5, 7], + )?, fact, )) } @@ -204,7 +206,7 @@ fn augmented<'py>( fn check_augmentation( context: &LoopContext<'_, '_>, local: &SchemaColumnFact, - named: &Bound<'_, PyAny>, + named: &HeaderColumn, model: (&str, &str), ) -> PyResult<()> { let (model_name, relative_path) = model; @@ -224,7 +226,7 @@ fn check_augmentation( "model '{model_name}' in {relative_path} cannot override {} for named-schema column \ '{name}' from {}; only audit augmentation is supported", overridden.join(", "), - origin(named)? + origin(&named.project(context.contracts.schema_column.py())?)? ); return Err(compile_error(context.classes, &message, None)); } @@ -233,7 +235,7 @@ fn check_augmentation( "model '{model_name}' in {relative_path} redeclares named-schema column '{name}' from \ {} without audits or migrate_from; only audit augmentation and column \ migrations are supported", - origin(named)? + origin(&named.project(context.contracts.schema_column.py())?)? ); return Err(compile_error(context.classes, &message, None)); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index c0422ad9ca..ef803053de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -131,6 +131,14 @@ impl NativeCompiledProject { &self.facts } + /// A project holding only `facts`, which analyses can be recorded into apart from the compile's. + pub(crate) fn with_facts(facts: CompiledProjectFacts) -> Self { + Self { + facts, + ..Self::default() + } + } + /// Retain facts a producer finished natively. pub(crate) fn retain(&mut self, facts: CompiledModelFacts) { record_model(&mut self.facts, facts); @@ -162,12 +170,10 @@ impl NativeCompiledProject { }; owners.push(model_owner( record.name.clone(), - native_field(record.model_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? - .str()? - .to_string(), + record.model_file.get().relative_text(py)?, ( schema.audits.bind(py), - (schema.columns.bind(py), &fact.columns), + (schema.columns.as_ref(), &fact.columns), ), )?); } @@ -443,9 +449,7 @@ impl NativeCompiledProject { .iter() .map(|record| { Ok(( - native_field(record.model_file.bind(py), RELATIVE_PATH_ATTRIBUTE)? - .str()? - .to_string(), + record.model_file.get().relative_text(py)?, record.references.clone(), )) }) @@ -626,9 +630,9 @@ pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult = authored_expansion(&classes, record)?; if !expansion.is_none(py) { - let path: Bound<'_, PyAny> = - native_field(record.model_file.bind(py), FILE_PATH_ATTRIBUTE)?; + let path: Bound<'_, PyAny> = record.model_file.get().path_objects(py)?.0; expansions.set_item(path, expansion)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index 08671c4219..503d7026a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -3,7 +3,9 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple}; +use pyo3::types::{ + PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple, PyType, +}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::CompiledModelFacts; @@ -16,6 +18,7 @@ use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView use crate::bindings::_helpers::compiled_project::schema_facts::entry_schema_fact; use crate::bindings::_helpers::records::construct::{enum_member, new_record, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::records::fields::record_spec; use crate::bindings::_helpers::records::ids::RecordId; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -44,12 +47,8 @@ fn record<'py>(value: &Bound<'py, PyAny>, class: RecordId) -> PyResult = PyDict::new(py); - for field in imported(py, DATACLASSES_MODULE, "fields")? - .call1((&class,))? - .try_iter()? - { - let name: Bound<'py, PyAny> = native_field(&field?, "name")?; - fields.set_item(&name, native_field(value, &name.extract::()?)?)?; + for field in record_spec(class.cast::()?)?.fields { + fields.set_item(field.name, native_field(value, field.name)?)?; } class.call((), Some(&fields)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index 8d0aeef213..efd61f55c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -22,6 +22,7 @@ use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::records::fields::field as native_field; const MISSING_DESCRIPTION_CODE: &str = "P010"; @@ -201,6 +202,9 @@ pub(crate) fn resource_diagnostics<'py>( fn orphaned_factories<'py>(diagnostics: &Diagnostics<'_, 'py>) -> PyResult>> { let py: Python<'py> = diagnostics.py; + if diagnostics.settings.audit_factories.is_empty() { + return Ok(Vec::new()); + } let mut referenced: HashSet = HashSet::new(); for model_file in &diagnostics.settings.model_files { let header: Bound<'py, PyDict> = @@ -249,12 +253,10 @@ fn missing_descriptions<'py>( let py: Python<'py> = diagnostics.py; let mut found: Vec>> = Vec::new(); for record in project.inputs() { - let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); + let model_file: &NativeModelFile = record.model_file.get(); let name: String = record.name.clone(); - let path: String = native_field(model_file, "relative_path")? - .str()? - .extract()?; - let contents: String = native_field(model_file, "contents")?.extract()?; + let path: String = model_file.relative_text(py)?; + let contents: &str = model_file.contents_text(); let entry: String = format!("description \"What one row of {name} represents\""); let help: String = header_help(&HeaderHelp { kind: "model", @@ -271,7 +273,7 @@ fn missing_descriptions<'py>( found.push(diagnostics.described( ("model", &name, Some("model")), description, - || diagnostics.header_location(&path, &contents), + || diagnostics.header_location(&path, contents), &help, )?); } @@ -487,7 +489,7 @@ fn cursor_without_inputs<'py>( let materialized: Option = config_text(&values, "materialized")?; let cursor: Option = config_text(&values, CURSOR_KEY)?; let strategy: Option = config_text(&values, "microbatch_strategy")?; - let model_file: &Bound<'py, PyAny> = record.model_file.bind(py); + let model_file: &NativeModelFile = record.model_file.get(); let name: String = record.name.clone(); let Some((message, help)) = cursor_lacks_inputs( &name, @@ -503,13 +505,11 @@ fn cursor_without_inputs<'py>( ) else { continue; }; - let path: String = native_field(model_file, "relative_path")? - .str()? - .extract()?; - let contents: String = native_field(model_file, "contents")?.extract()?; + let path: String = model_file.relative_text(py)?; + let contents: &str = model_file.contents_text(); let span: Option<(usize, usize)> = - model_header_entry_span(&contents, CURSOR_KEY).map_err(PyValueError::new_err)?; - let start: TextPosition = text_position(&contents, span.map_or(0, |span| span.0)); + model_header_entry_span(contents, CURSOR_KEY).map_err(PyValueError::new_err)?; + let start: TextPosition = text_position(contents, span.map_or(0, |span| span.0)); let location: Bound<'py, PyAny> = diagnostics.location( &path, (start.line, start.column), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs index 1d558d435e..8c3c3aa0d3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs @@ -110,7 +110,7 @@ fn authored<'py>( py: Python<'py>, values: &[(String, AuthoredValue)], ) -> PyResult> { - Ok(map_to_python(py, values.to_vec())? + Ok(map_to_python(py, values)? .into_bound(py) .downcast_into::()?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs index 8106673ec7..deaf9dbb15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs @@ -1,13 +1,20 @@ //! A discovered SQL model file as a native record whose Python attributes are built on demand. +use std::path::PathBuf; +use std::sync::Arc; + use crate::bindings::_helpers::records::construct::new_record; +use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::ids::RecordId; -use pyo3::exceptions::PyTypeError; +use crate::bindings::_helpers::sqltext::authored_values::map_to_python; +use pyo3::exceptions::{PyRuntimeError, PyTypeError}; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::sync::PyOnceLock; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; -use pyo3::{FromPyObject, IntoPyObjectExt, pyclass, pymethods}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::{FromPyObject, IntoPyObject, IntoPyObjectExt, pyclass, pymethods}; +use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_discovery::models::LineColumnSpan; +use sqlbuild_sqltext::compiler::models::AuthoredValue; const FIELD_NAMES: [&str; 10] = [ "file_path", @@ -21,6 +28,7 @@ const FIELD_NAMES: [&str; 10] = [ "constant_declarations", "extract_implicit_alias_columns", ]; +const STEM_ATTRIBUTE: &str = "stem"; /// Authored locations: native rows until Python reads them, then a cached dict. struct Locations { @@ -58,7 +66,8 @@ impl Locations { } /// Python's `{name: SourceLocation(...)}`: a repeated name keeps its first position and last value. - fn dict<'py>(&self, py: Python<'py>, path: &Py) -> PyResult> { + fn dict<'py>(&self, path: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = path.py(); Ok(self .projected .get_or_try_init(py, || { @@ -67,13 +76,7 @@ impl Locations { let location: Bound<'py, PyAny> = new_record( py, RecordId::SourceLocation, - ( - path.bind(py), - span.line, - span.column, - span.end_line, - span.end_column, - ), + (path, span.line, span.column, span.end_line, span.end_column), None, )?; dict.set_item(name, location)?; @@ -85,34 +88,276 @@ impl Locations { } } +/// The names a model file is known by, as Python's `pathlib` spells them. +#[derive(Clone)] +pub(crate) struct ModelFileTexts { + /// `file_path.stem`, the model's name. + pub(crate) name: String, + /// `relative_path.as_posix()`. + pub(crate) posix_path: String, + /// `str(relative_path)`. + pub(crate) relative_path: String, + /// `str(file_path)`. + pub(crate) file_path: String, +} + +/// A model file's paths: native text, with the Python path objects built on first read. +struct ModelFilePaths { + native_file_path: PathBuf, + /// `None` when a path is not text Python can encode, so readers raise as Python does. + texts: Option, + /// `(file_path, relative_path)` as Python path objects. + objects: PyOnceLock<(Py, Py)>, + /// The relative path the objects are built from, unless they were given. + relative: Option, +} + +impl ModelFilePaths { + /// Paths given as Python objects; their text is read once, here. + fn given(py: Python<'_>, file_path: Py, relative_path: Py) -> PyResult { + let native_file_path: PathBuf = file_path.bind(py).extract()?; + let relative: PathBuf = relative_path.bind(py).extract()?; + let texts: Option = + native_file_path + .to_str() + .zip(relative.to_str()) + .map(|(file_text, relative_text)| ModelFileTexts { + name: model_file_stem(&posix(file_text)).to_owned(), + posix_path: posix(relative_text), + relative_path: relative_text.to_owned(), + file_path: file_text.to_owned(), + }); + let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); + let _ = objects.set(py, (file_path, relative_path)); + Ok(Self { + native_file_path, + texts, + objects, + relative: None, + }) + } + + /// Paths native discovery named; their Python path objects are built only when read. + fn discovered(relative_path: &str, file_path: String) -> Self { + let name: String = model_file_stem(relative_path).to_owned(); + Self { + native_file_path: PathBuf::from(&file_path), + texts: Some(ModelFileTexts { + name, + posix_path: relative_path.to_owned(), + relative_path: relative_path.replace('/', std::path::MAIN_SEPARATOR_STR), + file_path, + }), + objects: PyOnceLock::new(), + relative: Some(PathBuf::from(relative_path)), + } + } + + fn shared(&self, py: Python<'_>) -> Self { + let objects: PyOnceLock<(Py, Py)> = PyOnceLock::new(); + if let Some((file_path, relative_path)) = self.objects.get(py) { + let _ = objects.set(py, (file_path.clone_ref(py), relative_path.clone_ref(py))); + } + Self { + native_file_path: self.native_file_path.clone(), + texts: self.texts.clone(), + objects, + relative: self.relative.clone(), + } + } + + /// `(file_path, relative_path)` as the `pathlib` objects discovery names the file by. + fn objects<'py>(&self, py: Python<'py>) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let (file_path, relative_path) = self.objects.get_or_try_init(py, || { + let Some(relative) = &self.relative else { + return Err(PyRuntimeError::new_err("a model file has no paths")); + }; + Ok::<_, pyo3::PyErr>(( + self.native_file_path.as_path().into_pyobject(py)?.unbind(), + relative.as_path().into_pyobject(py)?.unbind(), + )) + })?; + Ok((file_path.bind(py).clone(), relative_path.bind(py).clone())) + } + + /// The file's names, or the error Python raises reading a name that is not UTF-8 text. + fn texts(&self, py: Python<'_>) -> PyResult<&ModelFileTexts> { + if let Some(texts) = &self.texts { + return Ok(texts); + } + let (file_path, relative_path) = self.objects(py)?; + for path in [file_path, relative_path] { + path.str()?.to_str()?; + } + Err(PyRuntimeError::new_err("a model file path is not text")) + } +} + +/// `text` with the platform's path separator spelled `/`, as `PurePath.as_posix` spells it. +fn posix(text: &str) -> String { + text.replace(std::path::MAIN_SEPARATOR, "/") +} + +/// `text` as the Python string `cache` shares with every reader, built on first read. +pub(crate) fn projected_text<'py>( + py: Python<'py>, + cache: &PyOnceLock>, + text: &str, +) -> Bound<'py, PyString> { + cache + .get_or_init(py, || PyString::new(py, text).unbind()) + .bind(py) + .clone() +} + +/// A model header's authored values: native entries, with the Python dict built on first read. +struct HeaderValues { + native: Option>, + projected: PyOnceLock>, +} + +impl HeaderValues { + fn native(values: Vec<(String, AuthoredValue)>) -> Self { + Self { + native: Some(values.into()), + projected: PyOnceLock::new(), + } + } + + fn given(py: Python<'_>, values: Py) -> Self { + let projected: PyOnceLock> = PyOnceLock::new(); + let _ = projected.set(py, values); + Self { + native: None, + projected, + } + } + + fn shared(&self, py: Python<'_>) -> Self { + let projected: PyOnceLock> = PyOnceLock::new(); + if let Some(dict) = self.projected.get(py) { + let _ = projected.set(py, dict.clone_ref(py)); + } + Self { + native: self.native.clone(), + projected, + } + } + + fn dict<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(self + .projected + .get_or_try_init(py, || { + let values: &[(String, AuthoredValue)] = self.native.as_deref().unwrap_or_default(); + map_to_python(py, values)? + .into_bound(py) + .downcast_into::() + .map(Bound::unbind) + .map_err(pyo3::PyErr::from) + })? + .bind(py) + .clone()) + } +} + /// A discovered SQL model file and its raw contents. #[pyclass(module = "sqlbuild._native", name = "DiscoveredSqlModelFile", frozen)] pub(crate) struct NativeModelFile { - file_path: Py, - relative_path: Py, - contents: Py, - header_values: Py, + paths: ModelFilePaths, + contents: Arc, + /// `contents` and `query_sql` as the Python strings readers share, built on first read. + projected_texts: (PyOnceLock>, PyOnceLock>), + header_values: HeaderValues, header_column_locations: Locations, output_column_locations: Locations, - query_sql: Py, + query_sql: Arc, enum_declarations: Py, constant_declarations: Py, extract_implicit_alias_columns: bool, } -/// What native discovery read for one model file, before its locations are projected. +/// What native discovery read for one model file. pub(crate) struct ModelFileParts { - pub(crate) file_path: Py, - pub(crate) relative_path: Py, - pub(crate) contents: Py, - pub(crate) header_values: Py, + pub(crate) contents: String, + pub(crate) header_values: Vec<(String, AuthoredValue)>, pub(crate) header_column_locations: Vec<(String, LineColumnSpan)>, pub(crate) output_column_locations: Vec<(String, LineColumnSpan)>, - pub(crate) query_sql: Py, - pub(crate) declarations: (Py, Py), + pub(crate) query_sql: String, pub(crate) extract_implicit_alias_columns: bool, } +/// How discovery names a model file: natively, or by Python paths for names Python escapes. +pub(crate) enum DiscoveredPaths { + /// The `/`-separated relative path and `str(project_dir / relative_path)`. + Native(String, String), + /// `(file_path, relative_path)` built by Python. + Python(Py, Py), +} + +/// A model file before its header declarations are attached. +pub(crate) struct PendingModelFile { + paths: ModelFilePaths, + parts: ModelFileParts, +} + +impl PendingModelFile { + pub(crate) fn new( + py: Python<'_>, + paths: DiscoveredPaths, + parts: ModelFileParts, + ) -> PyResult { + let paths: ModelFilePaths = match paths { + DiscoveredPaths::Native(relative_path, file_path) => { + ModelFilePaths::discovered(&relative_path, file_path) + } + DiscoveredPaths::Python(file_path, relative_path) => { + ModelFilePaths::given(py, file_path, relative_path)? + } + }; + Ok(Self { paths, parts }) + } + + /// The model's name, its file stem. + pub(crate) fn name(&self, py: Python<'_>) -> PyResult { + match &self.paths.texts { + Some(texts) => Ok(texts.name.clone()), + None => field(&self.paths.objects(py)?.0, STEM_ATTRIBUTE)?.extract(), + } + } + + /// The relative path as the Python path object the record keeps. + pub(crate) fn relative_path<'py>(&self, py: Python<'py>) -> PyResult> { + Ok(self.paths.objects(py)?.1) + } + + /// The authored header value of `key`. + pub(crate) fn header_value(&self, key: &str) -> Option<&AuthoredValue> { + self.parts + .header_values + .iter() + .rev() + .find(|(name, _)| name == key) + .map(|(_, value)| value) + } + + pub(crate) fn finished(self, declarations: (Py, Py)) -> NativeModelFile { + let (enum_declarations, constant_declarations) = declarations; + let parts: ModelFileParts = self.parts; + NativeModelFile { + paths: self.paths, + contents: parts.contents.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), + header_values: HeaderValues::native(parts.header_values), + header_column_locations: Locations::from_rows(parts.header_column_locations), + output_column_locations: Locations::from_rows(parts.output_column_locations), + query_sql: parts.query_sql.into(), + enum_declarations, + constant_declarations, + extract_implicit_alias_columns: parts.extract_implicit_alias_columns, + } + } +} + /// The keyword fields Python constructs a model file record with. #[derive(FromPyObject)] #[pyo3(from_item_all)] @@ -133,28 +378,24 @@ struct ModelFileFields { } impl NativeModelFile { - pub(crate) fn discovered(parts: ModelFileParts) -> Self { - let (enum_declarations, constant_declarations) = parts.declarations; - Self { - file_path: parts.file_path, - relative_path: parts.relative_path, - contents: parts.contents, - header_values: parts.header_values, - header_column_locations: Locations::from_rows(parts.header_column_locations), - output_column_locations: Locations::from_rows(parts.output_column_locations), - query_sql: parts.query_sql, - enum_declarations, - constant_declarations, - extract_implicit_alias_columns: parts.extract_implicit_alias_columns, - } + pub(crate) fn header_locations<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_column_locations + .dict(&self.paths.objects(py)?.1) } - pub(crate) fn header_locations<'py>(&self, py: Python<'py>) -> PyResult> { - self.header_column_locations.dict(py, &self.relative_path) + /// The header's authored values as the Python dict config builds read. + pub(crate) fn header_value_dict<'py>(&self, py: Python<'py>) -> PyResult> { + self.header_values.dict(py) + } + + /// The header columns' authored locations, unless Python constructed this record. + pub(crate) fn header_location_rows(&self) -> &[(String, LineColumnSpan)] { + &self.header_column_locations.rows } - pub(crate) fn header_value_dict<'py>(&self, py: Python<'py>) -> &Bound<'py, PyDict> { - self.header_values.bind(py) + /// The header's native authored values, unless Python constructed this record. + pub(crate) fn native_header_values(&self) -> Option<&Arc<[(String, AuthoredValue)]>> { + self.header_values.native.as_ref() } pub(crate) fn declaration_tuples<'py>( @@ -167,28 +408,68 @@ impl NativeModelFile { ) } - pub(crate) fn paths<'py>(&self, py: Python<'py>) -> (&Bound<'py, PyAny>, &Bound<'py, PyAny>) { - (self.file_path.bind(py), self.relative_path.bind(py)) + /// The file path as a native path. + pub(crate) fn file_path_buf(&self) -> PathBuf { + self.paths.native_file_path.clone() + } + + /// The file's names, or the error Python raises reading a name that is not UTF-8 text. + pub(crate) fn path_texts(&self, py: Python<'_>) -> PyResult { + self.paths.texts(py).cloned() } - pub(crate) fn texts<'py>( + /// `relative_path.as_posix()`. + pub(crate) fn posix_path(&self, py: Python<'_>) -> PyResult { + Ok(self.paths.texts(py)?.posix_path.clone()) + } + + /// `str(relative_path)`. + pub(crate) fn relative_text(&self, py: Python<'_>) -> PyResult { + Ok(self.paths.texts(py)?.relative_path.clone()) + } + + /// `str(relative_path.parent)`, the folder project graphs group the model by. + pub(crate) fn folder(&self, py: Python<'_>) -> PyResult { + Ok(self + .paths + .texts(py)? + .relative_path + .rsplit_once(std::path::MAIN_SEPARATOR) + .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned())) + } + + /// `(file_path, relative_path)` as Python path objects. + pub(crate) fn path_objects<'py>( &self, py: Python<'py>, - ) -> (&Bound<'py, PyString>, &Bound<'py, PyString>) { - (self.query_sql.bind(py), self.contents.bind(py)) + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + self.paths.objects(py) + } + + /// The authored query below the header. + pub(crate) fn query_text(&self) -> &Arc { + &self.query_sql + } + + /// The whole authored file. + pub(crate) fn contents_text(&self) -> &Arc { + &self.contents } fn fields<'py>(&self, py: Python<'py>) -> PyResult>> { + let (file_path, relative_path) = self.paths.objects(py)?; Ok(vec![ - self.file_path.bind(py).clone(), - self.relative_path.bind(py).clone(), - self.contents.bind(py).clone().into_any(), - self.header_values.bind(py).clone().into_any(), - self.header_locations(py)?.into_any(), + file_path, + relative_path.clone(), + self.contents(py).into_any(), + self.header_values.dict(py)?.into_any(), + self.header_column_locations + .dict(&relative_path)? + .into_any(), self.output_column_locations - .dict(py, &self.relative_path)? + .dict(&relative_path)? .into_any(), - self.query_sql.bind(py).clone().into_any(), + self.query_sql(py).into_any(), self.enum_declarations.bind(py).clone().into_any(), self.constant_declarations.bind(py).clone().into_any(), pyo3::types::PyBool::new(py, self.extract_implicit_alias_columns) @@ -218,13 +499,13 @@ impl NativeModelFile { } let fields: ModelFileFields = fields.extract()?; Ok(Self { - file_path: fields.file_path, - relative_path: fields.relative_path, - contents: fields.contents, - header_values: fields.header_values, + paths: ModelFilePaths::given(py, fields.file_path, fields.relative_path)?, + contents: fields.contents.bind(py).to_str()?.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), + header_values: HeaderValues::given(py, fields.header_values), header_column_locations: Locations::from_dict(py, fields.header_column_locations), output_column_locations: Locations::from_dict(py, fields.output_column_locations), - query_sql: fields.query_sql, + query_sql: fields.query_sql.bind(py).to_str()?.into(), enum_declarations: fields .enum_declarations .unwrap_or_else(|| PyTuple::empty(py).unbind()), @@ -241,39 +522,39 @@ impl NativeModelFile { py: Python<'_>, contents: Py, query_sql: Py, - ) -> Self { - Self { - file_path: self.file_path.clone_ref(py), - relative_path: self.relative_path.clone_ref(py), - contents, - header_values: self.header_values.clone_ref(py), + ) -> PyResult { + Ok(Self { + paths: self.paths.shared(py), + contents: contents.bind(py).to_str()?.into(), + projected_texts: (PyOnceLock::new(), PyOnceLock::new()), + header_values: self.header_values.shared(py), header_column_locations: self.header_column_locations.shared(py), output_column_locations: self.output_column_locations.shared(py), - query_sql, + query_sql: query_sql.bind(py).to_str()?.into(), enum_declarations: self.enum_declarations.clone_ref(py), constant_declarations: self.constant_declarations.clone_ref(py), extract_implicit_alias_columns: self.extract_implicit_alias_columns, - } + }) } #[getter] - pub(crate) fn file_path(&self, py: Python<'_>) -> Py { - self.file_path.clone_ref(py) + pub(crate) fn file_path(&self, py: Python<'_>) -> PyResult> { + Ok(self.paths.objects(py)?.0.unbind()) } #[getter] - pub(crate) fn relative_path(&self, py: Python<'_>) -> Py { - self.relative_path.clone_ref(py) + pub(crate) fn relative_path(&self, py: Python<'_>) -> PyResult> { + Ok(self.paths.objects(py)?.1.unbind()) } #[getter] - pub(crate) fn contents(&self, py: Python<'_>) -> Py { - self.contents.clone_ref(py) + pub(crate) fn contents<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { + projected_text(py, &self.projected_texts.0, &self.contents) } #[getter] - pub(crate) fn header_values(&self, py: Python<'_>) -> Py { - self.header_values.clone_ref(py) + pub(crate) fn header_values(&self, py: Python<'_>) -> PyResult> { + Ok(self.header_values.dict(py)?.unbind()) } #[getter] @@ -289,12 +570,13 @@ impl NativeModelFile { &self, py: Python<'py>, ) -> PyResult> { - self.output_column_locations.dict(py, &self.relative_path) + self.output_column_locations + .dict(&self.paths.objects(py)?.1) } #[getter] - pub(crate) fn query_sql(&self, py: Python<'_>) -> Py { - self.query_sql.clone_ref(py) + pub(crate) fn query_sql<'py>(&self, py: Python<'py>) -> Bound<'py, PyString> { + projected_text(py, &self.projected_texts.1, &self.query_sql) } #[getter] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs index 3b11b2612d..1515583bcb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs @@ -1,15 +1,18 @@ //! SQL model files discovered natively and built as Python discovery records. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyTuple}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_discovery::model_files::main::discover_model_files::discover_model_files as discover_models; +use sqlbuild_discovery::model_files::main::model_file_stem::model_file_stem; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use sqlbuild_discovery::models::{DiscoveredFile, FileOutcome, StageFailure}; +use std::path::Path; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; use crate::bindings::_helpers::discovery::constants; use crate::bindings::_helpers::discovery::faults::{FileFaults, record_tuple}; +use crate::bindings::_helpers::discovery::model_file_record::DiscoveredPaths; use crate::bindings::_helpers::discovery::project_files::{model_record, path_object}; use crate::bindings::_helpers::discovery::raised::{failure_error, read_error, stage_error}; use crate::bindings::_helpers::discovery::snapshot::{in_snapshot, project_root, project_tree}; @@ -20,8 +23,31 @@ use crate::bindings::models::NativeProjectTree; const PATHLIB_MODULE: &str = "pathlib"; const STEM_ATTRIBUTE: &str = "stem"; -/// One discovered model file: its record, or the error building it raises. -type ModelItem<'py> = (Bound<'py, PyAny>, PyResult>); +/// One discovered model file: its relative path and record, or the error building it raises. +type ModelItem<'py> = (RelativePath<'py>, PyResult>); + +/// A discovered file's relative path: native text, or the Python path already built for it. +enum RelativePath<'py> { + Native(String), + Python(Bound<'py, PyAny>), +} + +impl<'py> RelativePath<'py> { + fn object(&self, py: Python<'py>) -> PyResult> { + match self { + Self::Native(relative) => Path::new(relative).into_pyobject(py), + Self::Python(relative) => Ok(relative.clone()), + } + } + + /// The model's name, its file stem. + fn name(&self, py: Python<'py>) -> PyResult> { + match self { + Self::Native(relative) => Ok(PyString::new(py, model_file_stem(relative)).into_any()), + Self::Python(relative) => native_field(relative, STEM_ATTRIBUTE), + } + } +} /// Discover every model file natively, each as its record or the error it raises when built. fn model_items<'py>( @@ -41,6 +67,15 @@ fn model_items<'py>( let path_class: Bound<'py, PyAny> = imported(py, PATHLIB_MODULE, "Path")?; let mut items: Vec> = Vec::with_capacity(files.len()); for file in files { + let raw: bool = tree.get().inner.raw_relative(&file.relative_path).is_some(); + if !raw && let FileOutcome::Parsed(model) = file.outcome { + let file_path: String = root.display_path(&file.relative_path); + let paths = DiscoveredPaths::Native(file.relative_path.clone(), file_path); + let built = model_record(py, model, paths, options.extract_implicit_alias_columns)? + .map(|record| record.into_bound(py)); + items.push((RelativePath::Native(file.relative_path), built)); + continue; + } let relative_path: Bound<'py, PyAny> = path_class.call1((path_object(py, &tree.get().inner, file.relative_path)?,))?; let file_path: Bound<'py, PyAny> = project_dir.div(&relative_path)?; @@ -48,14 +83,14 @@ fn model_items<'py>( FileOutcome::Parsed(model) => model_record( py, model, - (file_path, relative_path.clone()), + DiscoveredPaths::Python(file_path.unbind(), relative_path.clone().unbind()), options.extract_implicit_alias_columns, )? .map(|record| record.into_bound(py)), FileOutcome::Unreadable(failure) => Err(read_error(failure, &file_path)), FileOutcome::Failed(failure) => Err(failure_error(py, failure)), }; - items.push((relative_path, built)); + items.push((RelativePath::Python(relative_path), built)); } Ok(items) }) @@ -92,11 +127,16 @@ pub(crate) fn discovered_model_files<'py>( let mut records: Vec> = Vec::with_capacity(items.len()); for (relative_path, built) in items { if let Some(selected) = &selected - && !selected.contains(native_field(&relative_path, STEM_ATTRIBUTE)?)? + && !selected.contains(relative_path.name(py)?)? { continue; } - records.extend(faults.kept(&relative_path, built)?); + match built { + Ok(record) => records.push(record), + Err(error) => { + records.extend(faults.kept(&relative_path.object(py)?, Err(error))?); + } + } } record_tuple(py, records) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs index 2042228633..c7d5e780bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_files.rs @@ -4,7 +4,7 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::types::PyTuple; use pyo3::{IntoPyObjectExt, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_discovery::model_files::main::parse_model_text::parse_model_text; use sqlbuild_discovery::model_files::models::{DiscoveredModelFile, ModelFileOptions}; @@ -12,19 +12,19 @@ use sqlbuild_discovery::tree::models::ProjectTree; use std::path::Path; use crate::bindings::_helpers::discovery::declaration_records::source_locations; -use crate::bindings::_helpers::discovery::header_declarations::{ - HeaderDeclarations, model_header_declarations, +use crate::bindings::_helpers::discovery::header_declarations::model_header_declarations; +use crate::bindings::_helpers::discovery::model_file_record::{ + DiscoveredPaths, ModelFileParts, NativeModelFile, PendingModelFile, }; -use crate::bindings::_helpers::discovery::model_file_record::{ModelFileParts, NativeModelFile}; use crate::bindings::_helpers::discovery::raised::failure_error; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::sqltext::authored_values::map_to_python; +use crate::bindings::_helpers::sqltext::authored_values::{ + map_to_python, optional_authored_value_to_python, +}; use crate::bindings::models::{ModelTextRequest, NativeProjectTree}; use crate::bindings::types::CompilerDetach; pub(crate) type PyObject = Py; -const STEM_ATTRIBUTE: &str = "stem"; const ENUMS_KEY: &str = "enums"; const CONSTANTS_KEY: &str = "constants"; @@ -51,43 +51,46 @@ impl NativeProjectTree { } /// One parsed model file's record, or its header declaration error, raised only if it is built. -pub(crate) fn model_record<'py>( - py: Python<'py>, +pub(crate) fn model_record( + py: Python<'_>, model: DiscoveredModelFile, - paths: (Bound<'py, PyAny>, Bound<'py, PyAny>), + paths: DiscoveredPaths, extract_implicit_alias_columns: bool, ) -> PyResult> { - let (file_path, relative_path) = paths; - let header_values: Bound<'py, PyDict> = map_to_python(py, model.header_values)? - .into_bound(py) - .downcast_into::()?; - let model_name: String = native_field(&file_path, STEM_ATTRIBUTE)?.extract()?; - let declarations: HeaderDeclarations<'py> = match model_header_declarations( + let pending: PendingModelFile = PendingModelFile::new( py, - ( - header_values.get_item(ENUMS_KEY)?, - header_values.get_item(CONSTANTS_KEY)?, - ), - model_name, - relative_path.clone(), - ) { - Ok(declarations) => declarations, - Err(error) if error.is_instance_of::(py) => return Ok(Err(error)), - Err(error) => return Err(error), + paths, + ModelFileParts { + contents: model.contents, + header_values: model.header_values, + header_column_locations: model.header_column_locations, + output_column_locations: model.output_column_locations, + query_sql: model.query_sql, + extract_implicit_alias_columns, + }, + )?; + let authored = |key: &str| -> PyResult>> { + Ok( + optional_authored_value_to_python(py, pending.header_value(key))? + .map(|value| value.into_bound(py)), + ) + }; + let (enums, constants) = (authored(ENUMS_KEY)?, authored(CONSTANTS_KEY)?); + let declarations: (Py, Py) = if enums.is_none() && constants.is_none() { + (PyTuple::empty(py).unbind(), PyTuple::empty(py).unbind()) + } else { + match model_header_declarations( + py, + (enums, constants), + pending.name(py)?, + pending.relative_path(py)?, + ) { + Ok((enums, constants)) => (enums.unbind(), constants.unbind()), + Err(error) if error.is_instance_of::(py) => return Ok(Err(error)), + Err(error) => return Err(error), + } }; - let (enum_declarations, constant_declarations) = declarations; - let record: NativeModelFile = NativeModelFile::discovered(ModelFileParts { - file_path: file_path.unbind(), - relative_path: relative_path.unbind(), - contents: PyString::new(py, &model.contents).unbind(), - header_values: header_values.unbind(), - header_column_locations: model.header_column_locations, - output_column_locations: model.output_column_locations, - query_sql: PyString::new(py, &model.query_sql).unbind(), - declarations: (enum_declarations.unbind(), constant_declarations.unbind()), - extract_implicit_alias_columns, - }); - Ok(Ok(record.into_py_any(py)?)) + Ok(Ok(pending.finished(declarations).into_py_any(py)?)) } /// Parse model contents as discovery does: `(header_values, query_sql, column locations)`. @@ -112,7 +115,7 @@ fn parse_model_contents( .getattr("Path")? .call1((request.file_path.as_str(),))?; Ok(( - map_to_python(py, model.header_values)?, + map_to_python(py, &model.header_values)?, model.query_sql, source_locations(py, &path, &model.header_column_locations)? .into_any() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs index 7fcbca43fa..4ae7d8acca 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs @@ -98,7 +98,7 @@ fn test_blocks<'py>( let mut native = Vec::with_capacity(blocks.len()); for block in blocks { native.push(( - map_to_python(py, block.header_values)? + map_to_python(py, &block.header_values)? .into_bound(py) .downcast_into::()?, block.sql_body, @@ -154,7 +154,7 @@ fn scenario_file<'py>( ) -> PyResult> { let (file_path, relative_path) = paths; let py: Python<'py> = file_path.py(); - let header_values: Bound<'py, PyDict> = map_to_python(py, file.header_values)? + let header_values: Bound<'py, PyDict> = map_to_python(py, &file.header_values)? .into_bound(py) .downcast_into::()?; let shown: Bound<'py, PyString> = file_path.str()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs index 2d91542cb0..b99f80b25d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs @@ -1,13 +1,324 @@ //! Parsed MODEL header columns and audits built into the Python contract objects. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use std::sync::OnceLock; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyTuple}; -use pyo3::{FromPyObject, IntoPyObject}; +use pyo3::{FromPyObject, IntoPyObject, IntoPyObjectExt}; +use sqlbuild_discovery::model_files::models::{NativeAudit, NativeColumn}; +use sqlbuild_discovery::models::LineColumnSpan; use sqlbuild_model_config::header_metadata::models::{ ParsedAudit, ParsedColumn, ParsedThresholds, ThresholdBound, }; +use sqlbuild_sqltext::compiler::models::AuthoredValue; +use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; +use crate::bindings::_helpers::model_config::contract_classes::contract_classes; +use crate::bindings::_helpers::records::construct::{new_record, record_class}; +use crate::bindings::_helpers::records::fields::all_specs; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::sqltext::authored_values::authored_value_to_python; + +/// `SchemaColumn` fields in declaration order, the order its objects are built positionally in. +const SCHEMA_COLUMN_FIELDS: [&str; 8] = [ + "name", + "type", + "nullable", + "description", + "meta", + "audits", + "location", + "migrate_from", +]; +/// `SchemaAuditInstance` fields in declaration order. +const AUDIT_INSTANCE_FIELDS: [&str; 11] = [ + "definition_name", + "arguments", + "name", + "description", + "severity", + "run_scope", + "always_run", + "thresholds", + "minimum_samples", + "evidence_limit", + "location", +]; +/// `MeasurementThresholds` fields in declaration order. +const THRESHOLDS_FIELDS: [&str; 2] = ["warn", "error"]; +/// `MeasurementThresholdBound` fields in declaration order. +const THRESHOLD_BOUND_FIELDS: [&str; 4] = ["operator", "limit", "lower", "upper"]; +/// Each positionally built record with the field order its values are given in. +const POSITIONAL_RECORDS: [(&str, &[&str]); 4] = [ + ("SchemaColumn", &SCHEMA_COLUMN_FIELDS), + ("SchemaAuditInstance", &AUDIT_INSTANCE_FIELDS), + ("MeasurementThresholds", &THRESHOLDS_FIELDS), + ("MeasurementThresholdBound", &THRESHOLD_BOUND_FIELDS), +]; + +/// Native column fields; a Python record is constructed only for public consumers. +pub(crate) struct HeaderColumn { + fields: ColumnFields, + class: Py, + projection: PyOnceLock>, +} + +/// What a column's Python record is built from. +enum ColumnFields { + /// The record's field values, in declaration order. + Python([Py; 8]), + /// A header column read natively, with what its location and audits are built with. + Native(Box), + /// Only the record itself, built by Python. + Projected, +} + +/// A natively parsed header column and what its Python record is built with. +pub(crate) struct NativeColumnSpec { + pub(crate) column: NativeColumn, + /// The model file whose relative path the column's location names, and the location. + pub(crate) location: Option<(Py, LineColumnSpan)>, +} + +impl HeaderColumn { + pub(crate) fn replaced<'py>( + class: &Bound<'py, PyAny>, + values: [Bound<'py, PyAny>; 8], + keyword_order: [usize; 8], + ) -> PyResult { + if class.is(&record_class(class.py(), RecordId::SchemaColumn)?) { + return Self::new(class, values); + } + let fields = PyDict::new(class.py()); + for index in keyword_order { + fields.set_item(SCHEMA_COLUMN_FIELDS[index], &values[index])?; + } + Self::from_python(class.call((), Some(&fields))?) + } + + pub(crate) fn field<'py>(&self, py: Python<'py>, name: &str) -> PyResult> { + let index = || { + SCHEMA_COLUMN_FIELDS + .iter() + .position(|field| *field == name) + .ok_or_else(|| PyRuntimeError::new_err("unknown column field")) + }; + if let Some(projected) = self.projection.get(py) { + return field(projected.bind(py), name); + } + match &self.fields { + ColumnFields::Python(values) => Ok(values[index()?].bind(py).clone()), + ColumnFields::Native(spec) => native_value(py, spec, index()?), + ColumnFields::Projected => field(&self.project(py)?, name), + } + } + + pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.projection + .get_or_try_init(py, || { + let values: [Bound<'py, PyAny>; 8] = match &self.fields { + ColumnFields::Python(values) => { + values.each_ref().map(|value| value.bind(py).clone()) + } + ColumnFields::Native(spec) => native_values(py, spec)?, + ColumnFields::Projected => { + return Err(PyRuntimeError::new_err("missing column projection")); + } + }; + contract_object(self.class.bind(py), values).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } + + pub(crate) fn from_python(value: Bound<'_, PyAny>) -> PyResult { + let py = value.py(); + let projection = PyOnceLock::new(); + let class = value.get_type().into_any().unbind(); + let _ = projection.set(py, value.unbind()); + Ok(Self { + fields: ColumnFields::Projected, + class, + projection, + }) + } + + pub(crate) fn new<'py>( + class: &Bound<'py, PyAny>, + values: [Bound<'py, PyAny>; 8], + ) -> PyResult { + if !class.is(&record_class(class.py(), RecordId::SchemaColumn)?) { + return Self::from_python(contract_object(class, values)?); + } + checked_field_orders()?; + Ok(Self { + fields: ColumnFields::Python(values.map(Bound::unbind)), + class: class.clone().unbind(), + projection: PyOnceLock::new(), + }) + } + + /// A natively parsed column, whose Python record is built only when Python reads it. + pub(crate) fn native(class: &Bound<'_, PyAny>, spec: NativeColumnSpec) -> PyResult { + let py: Python<'_> = class.py(); + if !class.is(&record_class(py, RecordId::SchemaColumn)?) { + return Self::from_python(contract_object(class, native_values(py, &spec)?)?); + } + checked_field_orders()?; + Ok(Self { + fields: ColumnFields::Native(Box::new(spec)), + class: class.clone().unbind(), + projection: PyOnceLock::new(), + }) + } +} + +/// A native column's record fields, in declaration order. +fn native_values<'py>( + py: Python<'py>, + spec: &NativeColumnSpec, +) -> PyResult<[Bound<'py, PyAny>; 8]> { + let location: Bound<'py, PyAny> = native_location(py, spec)?; + let audits: Bound<'py, PyAny> = native_audits(py, spec, &location)?; + let column: &NativeColumn = &spec.column; + Ok([ + column.name.as_str().into_bound_py_any(py)?, + column.column_type.as_deref().into_bound_py_any(py)?, + column.nullable.into_bound_py_any(py)?, + column.description.as_deref().into_bound_py_any(py)?, + PyDict::new(py).into_any(), + audits, + location, + column.migrate_from.as_deref().into_bound_py_any(py)?, + ]) +} + +/// The native column's record field at `index` of `SCHEMA_COLUMN_FIELDS`. +fn native_value<'py>( + py: Python<'py>, + spec: &NativeColumnSpec, + index: usize, +) -> PyResult> { + let column: &NativeColumn = &spec.column; + match SCHEMA_COLUMN_FIELDS[index] { + "name" => column.name.as_str().into_bound_py_any(py), + "type" => column.column_type.as_deref().into_bound_py_any(py), + "nullable" => column.nullable.into_bound_py_any(py), + "description" => column.description.as_deref().into_bound_py_any(py), + "meta" => Ok(PyDict::new(py).into_any()), + "audits" => native_audits(py, spec, &native_location(py, spec)?), + "location" => native_location(py, spec), + _ => column.migrate_from.as_deref().into_bound_py_any(py), + } +} + +/// The column's `SourceLocation`, or `None`. +fn native_location<'py>(py: Python<'py>, spec: &NativeColumnSpec) -> PyResult> { + match &spec.location { + Some((model_file, span)) => new_record( + py, + RecordId::SourceLocation, + ( + model_file.get().path_objects(py)?.1, + span.line, + span.column, + span.end_line, + span.end_column, + ), + None, + ), + None => Ok(py.None().into_bound(py)), + } +} + +/// The column's audits as a tuple of `SchemaAuditInstance`s at `location`. +fn native_audits<'py>( + py: Python<'py>, + spec: &NativeColumnSpec, + location: &Bound<'py, PyAny>, +) -> PyResult> { + let classes: ContractClasses<'py> = contract_classes(py)?.extract()?; + let audits: Vec> = spec + .column + .audits + .iter() + .map(|audit| native_audit_instance(py, &classes, audit, location)) + .collect::>()?; + Ok(PyTuple::new(py, audits)?.into_any()) +} + +/// A native audit as the `SchemaAuditInstance` the Python parse builds. +fn native_audit_instance<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + audit: &NativeAudit, + location: &Bound<'py, PyAny>, +) -> PyResult> { + let arguments = PyDict::new(py); + for (key, value) in &audit.arguments { + arguments.set_item(key, authored_value_to_python(py, value)?)?; + } + let severity = match &audit.severity { + Some(severity) => classes + .severities + .get_item(severity)? + .unwrap_or_else(|| py.None().into_bound(py)), + None => py.None().into_bound(py), + }; + let count = |value: Option<&AuthoredValue>| -> PyResult> { + match value { + Some(value) => Ok(authored_value_to_python(py, value)?.into_bound(py)), + None => Ok(py.None().into_bound(py)), + } + }; + contract_object( + &classes.schema_audit_instance, + [ + audit.definition_name.as_str().into_bound_py_any(py)?, + arguments.into_any(), + audit.name.as_deref().into_bound_py_any(py)?, + audit.description.as_deref().into_bound_py_any(py)?, + severity, + audit.run_scope.as_deref().into_bound_py_any(py)?, + audit.always_run.unwrap_or(false).into_bound_py_any(py)?, + thresholds(py, classes, audit.thresholds.as_ref())?, + count(audit.minimum_samples.as_ref())?, + count(audit.evidence_limit.as_ref())?, + location.clone(), + ], + ) +} + +/// Retained native columns and their shared, lazy public tuple. +pub(crate) struct HeaderColumns { + pub(crate) specs: Vec, + projection: PyOnceLock>, +} + +impl HeaderColumns { + pub(crate) fn new(specs: Vec) -> Self { + Self { + specs, + projection: PyOnceLock::new(), + } + } + + pub(crate) fn project<'py>(&self, py: Python<'py>) -> PyResult> { + self.projection + .get_or_try_init(py, || { + let columns = self + .specs + .iter() + .map(|column| column.project(py)) + .collect::>>()?; + PyTuple::new(py, columns).map(Bound::unbind) + }) + .map(|value| value.bind(py).clone()) + } +} /// The Python classes the parsed metadata becomes, read from a Python mapping. #[derive(FromPyObject)] @@ -23,17 +334,48 @@ pub(crate) struct ContractClasses<'py> { threshold_operators: Bound<'py, PyDict>, } +impl ContractClasses<'_> { + /// Whether these are the classes native header parsing builds by default. + pub(crate) fn uses_default_classes(&self) -> PyResult { + let py: Python<'_> = self.schema_column.py(); + let default: ContractClasses<'_> = contract_classes(py)?.extract()?; + Ok(self.schema_column.is(&default.schema_column) + && self + .schema_audit_instance + .is(&default.schema_audit_instance) + && self.source_location.is(&default.source_location) + && self + .measurement_thresholds + .is(&default.measurement_thresholds) + && self + .measurement_threshold_bound + .is(&default.measurement_threshold_bound) + && self.severities.is(&default.severities) + && self.threshold_operators.is(&default.threshold_operators)) + } +} + pub(crate) fn column_tuple<'py>( py: Python<'py>, classes: &ContractClasses<'py>, parsed: &[ParsedColumn>], locations: &Bound<'py, PyDict>, ) -> PyResult> { + PyTuple::new(py, schema_columns(py, classes, parsed, locations)?) +} + +/// The parsed columns as `SchemaColumn` objects, in declaration order. +pub(crate) fn schema_columns<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + parsed: &[ParsedColumn>], + locations: &Bound<'py, PyDict>, +) -> PyResult>> { let mut columns: Vec> = Vec::with_capacity(parsed.len()); for column in parsed { columns.push(schema_column(py, classes, column, locations)?); } - PyTuple::new(py, columns) + Ok(columns) } pub(crate) fn audit_tuple<'py>( @@ -41,24 +383,73 @@ pub(crate) fn audit_tuple<'py>( classes: &ContractClasses<'py>, parsed: &[ParsedAudit>], ) -> PyResult> { + PyTuple::new(py, audit_instances(py, classes, parsed)?) +} + +/// The parsed audits as `SchemaAuditInstance` objects without a location, in order. +pub(crate) fn audit_instances<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + parsed: &[ParsedAudit>], +) -> PyResult>> { let no_location: Bound<'py, PyAny> = py.None().into_bound(py); let mut audits: Vec> = Vec::with_capacity(parsed.len()); for audit in parsed { audits.push(audit_instance(py, classes, audit, &no_location)?); } - PyTuple::new(py, audits) + Ok(audits) +} + +/// `audit.name or audit.definition_name` of the instance `audit` builds. +pub(crate) fn parsed_audit_identity(audit: &ParsedAudit>) -> PyResult { + if let Some(name) = &audit.name + && name.0.is_truthy()? + { + return name.0.extract(); + } + audit.definition_name.0.extract() } -/// `class(**fields)`. -fn contract_object<'py>( +/// `class(*values)`: the same record as keywords bind, given in `POSITIONAL_RECORDS` order. +fn contract_object<'py, const N: usize>( class: &Bound<'py, PyAny>, - fields: &[(&str, Bound<'py, PyAny>)], + values: [Bound<'py, PyAny>; N], ) -> PyResult> { - let keywords: Bound<'py, PyDict> = PyDict::new(class.py()); - for (name, value) in fields { - keywords.set_item(name, value)?; + checked_field_orders()?; + class.call1(PyTuple::new(class.py(), values)?) +} + +/// Fail when a positionally built record no longer declares its fields in the expected order. +fn checked_field_orders() -> PyResult<()> { + static MISMATCH: OnceLock> = OnceLock::new(); + match MISMATCH.get_or_init(mismatched_record) { + Some(record) => Err(PyRuntimeError::new_err(format!( + "native record {record} does not declare the fields header metadata builds it with" + ))), + None => Ok(()), + } +} + +/// The first positionally built record whose registered fields differ from its expected order. +fn mismatched_record() -> Option<&'static str> { + for (record, fields) in POSITIONAL_RECORDS { + let mut declared: bool = false; + for spec in all_specs() { + if spec.name == record { + declared = spec.fields.len() == fields.len() + && spec + .fields + .iter() + .zip(fields) + .all(|(field, expected)| field.name == *expected); + break; + } + } + if !declared { + return Some(record); + } } - class.call((), Some(&keywords)) + None } fn optional<'py>(py: Python<'py>, value: Option<&PyNode<'py>>) -> Bound<'py, PyAny> { @@ -71,6 +462,15 @@ fn schema_column<'py>( column: &ParsedColumn>, locations: &Bound<'py, PyDict>, ) -> PyResult> { + native_schema_column(py, classes, column, locations)?.project(py) +} + +pub(crate) fn native_schema_column<'py>( + py: Python<'py>, + classes: &ContractClasses<'py>, + column: &ParsedColumn>, + locations: &Bound<'py, PyDict>, +) -> PyResult { let location = locations .get_item(&column.name.0)? .unwrap_or_else(|| py.None().into_bound(py)); @@ -79,17 +479,17 @@ fn schema_column<'py>( .iter() .map(|audit| audit_instance(py, classes, audit, &location)) .collect::>>()?; - contract_object( + HeaderColumn::new( &classes.schema_column, - &[ - ("name", column.name.0.clone()), - ("type", optional(py, column.column_type.as_ref())), - ("nullable", optional(py, column.nullable.as_ref())), - ("description", optional(py, column.description.as_ref())), - ("meta", PyDict::new(py).into_any()), - ("audits", PyTuple::new(py, audits)?.into_any()), - ("location", location), - ("migrate_from", optional(py, column.migrate_from.as_ref())), + [ + column.name.0.clone(), + optional(py, column.column_type.as_ref()), + optional(py, column.nullable.as_ref()), + optional(py, column.description.as_ref()), + PyDict::new(py).into_any(), + PyTuple::new(py, audits)?.into_any(), + location, + optional(py, column.migrate_from.as_ref()), ], ) } @@ -117,27 +517,18 @@ fn audit_instance<'py>( ); contract_object( &classes.schema_audit_instance, - &[ - ("definition_name", audit.definition_name.0.clone()), - ("arguments", arguments.into_any()), - ("name", optional(py, audit.name.as_ref())), - ("description", optional(py, audit.description.as_ref())), - ("severity", severity), - ("run_scope", optional(py, audit.run_scope.as_ref())), - ("always_run", always_run), - ( - "thresholds", - thresholds(py, classes, audit.thresholds.as_ref())?, - ), - ( - "minimum_samples", - optional(py, audit.minimum_samples.as_ref()), - ), - ( - "evidence_limit", - optional(py, audit.evidence_limit.as_ref()), - ), - ("location", location.clone()), + [ + audit.definition_name.0.clone(), + arguments.into_any(), + optional(py, audit.name.as_ref()), + optional(py, audit.description.as_ref()), + severity, + optional(py, audit.run_scope.as_ref()), + always_run, + thresholds(py, classes, audit.thresholds.as_ref())?, + optional(py, audit.minimum_samples.as_ref()), + optional(py, audit.evidence_limit.as_ref()), + location.clone(), ], ) } @@ -153,9 +544,9 @@ pub(crate) fn thresholds<'py>( }; contract_object( &classes.measurement_thresholds, - &[ - ("warn", threshold_bound(py, classes, parsed.warn)?), - ("error", threshold_bound(py, classes, parsed.error)?), + [ + threshold_bound(py, classes, parsed.warn)?, + threshold_bound(py, classes, parsed.error)?, ], ) } @@ -179,11 +570,11 @@ fn threshold_bound<'py>( .unwrap_or_else(|| py.None().into_bound(py)); contract_object( &classes.measurement_threshold_bound, - &[ - ("operator", operator), - ("limit", limit.into_pyobject(py)?.into_any()), - ("lower", lower.into_pyobject(py)?.into_any()), - ("upper", upper.into_pyobject(py)?.into_any()), + [ + operator, + limit.into_pyobject(py)?.into_any(), + lower.into_pyobject(py)?.into_any(), + upper.into_pyobject(py)?.into_any(), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs index ab7dc904ee..cd1b534582 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs @@ -8,7 +8,7 @@ use pyo3::types::{ }; use pyo3::{PyErr, pyclass, pymethods}; use sqlbuild_model_config::config_presence::main::contains_template::contains_template; -use sqlbuild_model_config::config_presence::main::first_macro_path::first_macro_path; +use sqlbuild_model_config::config_presence::main::entry_macro_path::entry_macro_path; use sqlbuild_model_config::config_presence::models::MacroPath; use sqlbuild_model_config::errors::ConfigError; use sqlbuild_model_config::model_validation::main::retention_override::retention_override; @@ -21,6 +21,7 @@ use sqlbuild_model_config::types::AuthoredNode; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, built_config}; +use crate::bindings::_helpers::compiled_project::model_loop_prepass::own_entry; use crate::bindings::_helpers::model_config::authored_nodes::PyNode; use crate::bindings::_helpers::model_config::config_errors::config_exception; use crate::bindings::_helpers::model_config::config_errors::native_config_error; @@ -29,6 +30,7 @@ use crate::bindings::_helpers::model_config::config_templates::{ }; use crate::bindings::_helpers::model_config::errors::NativeConfigError; use crate::bindings::_helpers::model_config::project_defaults::project_defaults_mapping; +use sqlbuild_discovery::model_files::models::{EntryFacts, HeaderFacts}; const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; /// Hook keys in the sorted order `validate_model_hook_config` checks them. @@ -164,14 +166,14 @@ impl NativeModelConfigBuilder { ) -> PyResult> { let (matched_path_default, model_name) = model; let matched_path_default: Option<&str> = matched_path_default.as_deref(); - compiler_guard( - || match self.built(py, &header, matched_path_default, &model_name)? { + compiler_guard(|| { + match self.built(py, (&header, None), matched_path_default, &model_name)? { ConfigBuild::Built(built) => { Ok(built_config(&classes, &built, matched_path_default)?.unbind()) } ConfigBuild::Rejected(error) => Ok(error.into_any()), - }, - ) + } + }) } } @@ -192,11 +194,11 @@ impl NativeModelConfigBuilder { pub(crate) fn built<'py>( &self, py: Python<'py>, - header: &Bound<'py, PyDict>, + (header, facts): (&Bound<'py, PyDict>, Option<&HeaderFacts>), matched_path_default: Option<&str>, model_name: &str, ) -> PyResult> { - match self.build_config(py, header, matched_path_default, model_name) { + match self.build_config(py, (header, facts), matched_path_default, model_name) { Ok(built) => Ok(ConfigBuild::Built(built)), Err(Halt::Config(error)) => Ok(ConfigBuild::Rejected(native_config_error(py, error)?)), Err(Halt::Error(error)) => Err(error), @@ -290,10 +292,11 @@ impl NativeModelConfigBuilder { fn build_config<'py>( &self, py: Python<'py>, - header: &Bound<'py, PyDict>, + input: HeaderInput<'_, 'py>, matched_path_default: Option<&str>, model_name: &str, ) -> Built> { + let header: &Bound<'py, PyDict> = input.0; check_header_tags(header, model_name)?; let path_values = matched_path_default .map(|key| self.path_default_values(py, key)) @@ -311,7 +314,7 @@ impl NativeModelConfigBuilder { layered.del_item(key)?; } } - let has_templates = contains_template(&PyNode(layered.clone().into_any())); + let has_templates = templated(input, &layered)?; let mut expansion = Expansion { builder: self, py, @@ -380,7 +383,7 @@ impl NativeModelConfigBuilder { header.get_item(TABLE_TYPE_KEY)?.map(PyNode).as_ref(), model_name, )?; - check_no_config_macros(&values)?; + check_no_config_macros(input, &values)?; Ok(BuiltConfig { values, header_keys: sorted_keys(header)?, @@ -765,15 +768,58 @@ fn storage_free_values<'py>( Ok(values) } +/// A model's authored header and, when it was read natively, the header's facts. +type HeaderInput<'a, 'py> = (&'a Bound<'py, PyDict>, Option<&'a HeaderFacts>); + +/// The facts of a config entry whose value is the header's own. +fn header_entry<'a, 'py>( + (header, facts): HeaderInput<'a, 'py>, + entry: (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult> { + match facts { + Some(facts) => own_entry(facts, header, entry), + None => Ok(None), + } +} + +/// `contains_template_data(values)`, reading the header's own entries from their facts. +fn templated<'py>(input: HeaderInput<'_, 'py>, values: &Bound<'py, PyDict>) -> Built { + for (key, value) in values.iter() { + let found: bool = match header_entry(input, (&key, &value))? { + Some(facts) => facts.template, + None => contains_template(&PyNode(value)), + }; + if found { + return Ok(true); + } + } + Ok(false) +} + /// Raise for the first config field outside hooks holding a macro call, as Python's walk does. -fn check_no_config_macros(values: &Bound<'_, PyDict>) -> Built<()> { - match first_macro_path(&PyNode(values.clone().into_any()), &HOOK_KEYS) { - MacroPath::Absent => Ok(()), - MacroPath::Found(path) => Err(Halt::Config(ConfigError::compile(format!( - "model config field '{}' does not allow macros", - path.join(".") - )))), +fn check_no_config_macros<'py>( + input: HeaderInput<'_, 'py>, + values: &Bound<'py, PyDict>, +) -> Built<()> { + for (key, value) in values.iter() { + if HOOK_KEYS + .iter() + .any(|name| PyNode(key.clone()).is_text(name)) + { + continue; + } + let found: MacroPath = match header_entry(input, (&key, &value))? { + Some(facts) => facts.macro_path.clone(), + None => entry_macro_path(&PyNode(key), &PyNode(value)), + }; + if let MacroPath::Found(path) = found { + return Err(Halt::Config(ConfigError::compile(format!( + "model config field '{}' does not allow macros", + path.join(".") + )))); + } } + Ok(()) } fn sorted_keys(header: &Bound<'_, PyDict>) -> Built> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs index 3539a0e634..a3001a18c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/fields.rs @@ -5,8 +5,9 @@ use std::sync::{Mutex, OnceLock}; use pyo3::IntoPyObjectExt; use pyo3::exceptions::{PyAttributeError, PyRuntimeError, PyTypeError}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; use pyo3::pyclass::CompareOp; +use pyo3::sync::PyOnceLock; use pyo3::types::{ PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyStringMethods, PyTuple, PyTupleMethods, PyType, PyTypeMethods, @@ -17,17 +18,26 @@ use sqlbuild_rules::project_tree::main::pure_path_posix::pure_path_posix; use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::constants::{ - CACHED_PROPERTIES, CLASS_PROPERTIES, FROZEN_INSTANCE_ERROR, NATIVE_MODULE, ORDERED_RECORDS, - RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, + CACHED_PROPERTIES, CLASS_PROPERTIES, ENUMS, FROZEN_INSTANCE_ERROR, NATIVE_MODULE, + ORDERED_RECORDS, RECORD_FINISHERS, RECORD_SPEC_ATTRIBUTE, RECORDS, }; use crate::bindings::_helpers::records::construct::{enum_member, new_record}; use crate::bindings::_helpers::records::getters::getter_field; -use crate::bindings::_helpers::records::models::{FieldDefault, NativeRecord, RecordSpec}; +use crate::bindings::_helpers::records::models::{ + EnumSpec, FieldDefault, NativeRecord, RecordSpec, +}; use crate::bindings::_helpers::records::types::ClassProperty; /// How a test run treats `field` falling back to `getattr` on a native class: unset outside tests. pub(crate) const NATIVE_FIELD_STRICT_ENV_VAR: &str = "SQLBUILD_NATIVE_FIELD_STRICT"; +/// The string enum member attributes answered natively. +const ENUM_VALUE_ATTRIBUTE: &str = "value"; +const ENUM_NAME_ATTRIBUTE: &str = "name"; + +/// Native string enums by class object, for `.value` and `.name` reads answered from their members. +static ENUM_CLASSES: PyOnceLock> = PyOnceLock::new(); + /// Every registered record spec, in registration order. pub(crate) fn all_specs() -> impl Iterator { RECORDS.iter().flat_map(|group| group.iter()) @@ -67,6 +77,8 @@ pub(crate) fn field<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult( + value: &Bound<'py, PyAny>, + name: &str, +) -> Option>> { + let wants_value: bool = match name { + ENUM_VALUE_ATTRIBUTE => true, + ENUM_NAME_ATTRIBUTE => false, + _ => return None, + }; + let py: Python<'py> = value.py(); + let classes: &HashMap = + match ENUM_CLASSES.get_or_try_init(py, || enum_classes(py)) { + Ok(classes) => classes, + Err(error) => return Some(Err(error)), + }; + let spec: &EnumSpec = classes.get(&(value.get_type().as_ptr() as usize))?; + let Ok(member) = value.cast::() else { + return None; + }; + let text: String = match member.to_str() { + Ok(text) => text.to_owned(), + Err(error) => return Some(Err(error)), + }; + let (member_name, member_value) = spec + .members + .iter() + .find(|(_, declared)| *declared == text)?; + let answer: &str = if wants_value { + member_value + } else { + member_name + }; + Some(Ok(PyString::new(py, answer).into_any())) +} + +fn enum_classes(py: Python<'_>) -> PyResult> { + let module: Bound<'_, PyModule> = py.import(NATIVE_MODULE)?; + let mut classes: HashMap = HashMap::new(); + for spec in ENUMS.iter().flat_map(|group| group.iter()) { + let class: Bound<'_, PyAny> = module.getattr(spec.name)?; + let _ = classes.insert(class.as_ptr() as usize, spec); + } + Ok(classes) +} + /// `mapping.get(key)`: a dict is read through the C API. pub(crate) fn mapping_get<'py, K: pyo3::IntoPyObject<'py>>( mapping: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/getter_readers.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/getter_readers.rs index c2316a7d5e..8b2d3f11e8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/getter_readers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/getter_readers.rs @@ -1097,10 +1097,16 @@ pub(crate) fn read_native_model_file<'py>( return None; }; Some(match name { - "file_path" => this.file_path(py).into_bound_py_any(py), - "relative_path" => this.relative_path(py).into_bound_py_any(py), + "file_path" => this + .file_path(py) + .and_then(|value| value.into_bound_py_any(py)), + "relative_path" => this + .relative_path(py) + .and_then(|value| value.into_bound_py_any(py)), "contents" => this.contents(py).into_bound_py_any(py), - "header_values" => this.header_values(py).into_bound_py_any(py), + "header_values" => this + .header_values(py) + .and_then(|value| value.into_bound_py_any(py)), "header_column_locations" => this .header_column_locations(py) .and_then(|value| value.into_bound_py_any(py)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs new file mode 100644 index 0000000000..ad7205070d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_builtin_rules.rs @@ -0,0 +1,269 @@ +//! Built-in rule evaluation started speculatively once a compile's model types are recovered. + +use std::cell::RefCell; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_rules::engine::main::answer_speculative_rows::answer_speculative_rows; +use sqlbuild_rules::engine::main::discard_speculative_rows::discard_speculative_rows; +use sqlbuild_rules::engine::main::start_speculative_rows::start_speculative_rows; +use sqlbuild_rules::errors::RowsError; +use sqlbuild_rules::models::{RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::compile_timings::{ + add_compile_timing, record_compile_metric, record_compile_timing, +}; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::records::{ + compiled_project_facts, record_model_analyses, +}; +use crate::bindings::_helpers::records::fields::field; +use crate::bindings::_helpers::rules::catalogue::references_custom_rules; +use crate::bindings::_helpers::rules::rule_dispatch::speculative_rules_request; +use crate::bindings::_helpers::rules::rule_plan::rule_run_plan; +use crate::bindings::_helpers::rules::rules_config::load_rules_config; +use crate::bindings::types::CompilerDetach; + +const CANCELLED: &str = "native rules evaluation was cancelled"; +const BUILD_PHASE: &str = "early_rules_build_ms"; +const WAIT_PHASE: &str = "early_rules_wait_ms"; +const HIT_METRIC: &str = "early_rules_hits"; +const MISS_METRIC: &str = "early_rules_misses"; + +/// What one compile's early built-in rules start from. +pub(crate) struct EarlyRulesSettings { + pub(crate) project_dir: PathBuf, + pub(crate) dialect: String, + pub(crate) no_sql_analysis: bool, +} + +/// One compile thread's early built-in rules, from arming to the rules phase taking them. +enum Slot { + Armed(EarlyRulesSettings), + Started(EarlyBuiltinRules), + Spent, +} + +thread_local! { + static SLOT: RefCell> = const { RefCell::new(None) }; +} + +/// Whether the rules phase used the early evaluation, and how long it waited for it. +#[derive(Default)] +pub(crate) struct EarlyRulesReport { + used: AtomicBool, + consulted: AtomicBool, + wait_ns: std::sync::atomic::AtomicU64, +} + +impl EarlyRulesReport { + /// Record the outcome in the compile timings of the calling thread. + pub(crate) fn record(&self) { + if !self.consulted.load(Ordering::Acquire) { + return; + } + add_compile_timing(WAIT_PHASE, self.wait_ns.load(Ordering::Acquire)); + let used: bool = self.used.load(Ordering::Acquire); + record_compile_metric(HIT_METRIC, i64::from(used)); + record_compile_metric(MISS_METRIC, i64::from(!used)); + } +} + +/// Arms early built-in rules for one compile analysis on this thread; dropping it stops them. +pub(crate) struct EarlyRulesScope { + owner: bool, +} + +impl EarlyRulesScope { + /// Claim this thread's slot unarmed; a nested compile analysis on this thread claims nothing. + pub(crate) fn new() -> Self { + let owner: bool = SLOT.with(|slot| { + let mut slot = slot.borrow_mut(); + if slot.is_some() { + return false; + } + *slot = Some(Slot::Spent); + true + }); + Self { owner } + } + + /// Start early rules with `settings` once the compile recovers its model types. + pub(crate) fn arm(&self, settings: EarlyRulesSettings) { + if !self.owner { + return; + } + SLOT.with(|slot| { + if let Some(current @ Slot::Spent) = slot.borrow_mut().as_mut() { + *current = Slot::Armed(settings); + } + }); + } + + /// Cancel and join early rules this scope armed that started and were not taken. + pub(crate) fn stop(&self, py: Python<'_>) { + if self.owner + && let Some(rules) = take_started() + { + rules.discard(py); + } + } +} + +impl Drop for EarlyRulesScope { + fn drop(&mut self) { + if !self.owner { + return; + } + let slot: Option = SLOT.with(|slot| slot.borrow_mut().take()); + drop(slot); + } +} + +/// The started early rules, which only the rules phase of the arming compile takes. +pub(crate) fn take_early_rules() -> Option { + take_started() +} + +fn take_started() -> Option { + SLOT.with(|slot| { + let mut slot = slot.borrow_mut(); + match slot.as_mut() { + Some(current) => match std::mem::replace(current, Slot::Spent) { + Slot::Started(rules) => Some(rules), + other => { + *current = other; + None + } + }, + None => None, + } + }) +} + +/// Start early built-in rules over `project` when armed; only non-`Exception` errors propagate. +pub(crate) fn start_early_rules(project: &Bound<'_, PyAny>) -> PyResult<()> { + let settings: Option = SLOT.with(|slot| { + let mut slot = slot.borrow_mut(); + match slot.as_mut() { + Some(current @ Slot::Armed(_)) => match std::mem::replace(current, Slot::Spent) { + Slot::Armed(settings) => Some(settings), + _ => None, + }, + _ => None, + } + }); + let Some(settings) = settings else { + return Ok(()); + }; + let py: Python<'_> = project.py(); + let started: Option = + match record_compile_timing(BUILD_PHASE, || early_request(project, &settings)) { + Ok(Some(rows)) => EarlyBuiltinRules::spawn(rows), + Ok(None) => None, + Err(error) if error.is_instance_of::(py) => None, + Err(error) => return Err(error), + }; + if let Some(started) = started { + SLOT.with(|slot| { + if let Some(current) = slot.borrow_mut().as_mut() { + *current = Slot::Started(started); + } + }); + } + Ok(()) +} + +/// The built-in rules request the rules phase is expected to build, when it evaluates no custom rules. +fn early_request( + project: &Bound<'_, PyAny>, + settings: &EarlyRulesSettings, +) -> PyResult> { + let py: Python<'_> = project.py(); + let config: Bound<'_, PyAny> = + load_rules_config(py, settings.project_dir.clone())?.into_bound(py); + if references_custom_rules(&config)? { + return Ok(None); + } + let project_dir: Bound<'_, PyAny> = settings.project_dir.clone().into_pyobject(py)?.into_any(); + let plan: Bound<'_, PyAny> = rule_run_plan( + (project, &config, &project_dir), + None, + settings.no_sql_analysis, + )?; + if field(&plan, "selected")?.len()? == 0 || field(&plan, "custom_payloads")?.is_truthy()? { + return Ok(None); + } + let facts: Bound<'_, NativeCompiledProject> = Bound::new( + py, + NativeCompiledProject::with_facts( + compiled_project_facts(project)?.borrow().facts().clone(), + ), + )?; + let _ = record_model_analyses(&facts, &field(project, "models")?)?; + speculative_rules_request(&plan, &settings.dialect, &facts).map(Some) +} + +/// One speculative built-in rules evaluation and the report of how the rules phase used it. +pub(crate) struct EarlyBuiltinRules { + speculative: SpeculativeRows, + report: Arc, +} + +impl EarlyBuiltinRules { + /// The started speculation, or `None` when its thread cannot start. + fn spawn(rows: RowsRequest) -> Option { + match start_speculative_rows(rows) { + Ok(speculative) => Some(Self { + speculative, + report: Arc::new(EarlyRulesReport::default()), + }), + Err(_) => None, + } + } + + /// The outcome report the rules phase records once it has used or discarded these rules. + pub(crate) fn report(&self) -> Arc { + Arc::clone(&self.report) + } + + /// Evaluate `rows`, from this speculation only when identical; `interrupted` stops the wait. + pub(crate) fn answer( + self, + py: Python<'_>, + rows: RowsRequest, + interrupted: &AtomicBool, + ) -> PyResult> { + let Self { + speculative, + report, + } = self; + report.consulted.store(true, Ordering::Release); + let waited: Instant = Instant::now(); + let answered: SpeculativeAnswer = py + .compiler_detach(|| Ok(answer_speculative_rows(speculative, rows, interrupted))) + .map_err(value_error)?; + report.wait_ns.store( + u64::try_from(waited.elapsed().as_nanos()).unwrap_or(u64::MAX), + Ordering::Release, + ); + match answered { + SpeculativeAnswer::Answered { evaluation, used } => { + report.used.store(used, Ordering::Release); + Ok(evaluation) + } + SpeculativeAnswer::Stopped => Ok(Err(RowsError::Rules(CANCELLED.to_owned()))), + } + } + + /// Cancel the speculation and wait for its thread without holding the interpreter. + pub(crate) fn discard(self, py: Python<'_>) { + let speculative: SpeculativeRows = self.speculative; + py.detach(|| discard_speculative_rows(speculative)); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index dc31b364be..03d70f111e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -501,7 +501,7 @@ fn replace<'py>( /// A discovered model file's contents, read in place from a native model file. fn model_contents<'py>(model: &Bound<'py, PyAny>) -> PyResult> { match model.cast::() { - Ok(native) => Ok(native.get().texts(model.py()).1.clone().into_any()), + Ok(native) => Ok(native.get().contents(model.py()).into_any()), Err(_) => native_field(model, "contents"), } } @@ -509,7 +509,7 @@ fn model_contents<'py>(model: &Bound<'py, PyAny>) -> PyResult> /// A discovered model file's path, read in place from a native model file. fn model_file_path<'py>(model: &Bound<'py, PyAny>) -> PyResult> { match model.cast::() { - Ok(native) => Ok(native.get().paths(model.py()).0.clone()), + Ok(native) => Ok(native.get().path_objects(model.py())?.0), Err(_) => native_field(model, "file_path"), } } @@ -534,9 +534,8 @@ fn edited_model<'py>( return model.call_method1("with_text", (text, query)); }; let native: &NativeModelFile = native.get(); - let (query_sql, contents) = native.texts(py); - let contents: String = contents.to_str()?.to_owned(); - let Some(offset) = contents.find(query_sql.to_str()?) else { + let (query_sql, contents): (&str, &str) = (native.query_text(), native.contents_text()); + let Some(offset) = contents.find(query_sql) else { return Err(pyo3::exceptions::PyValueError::new_err( "substring not found", )); @@ -549,7 +548,7 @@ fn edited_model<'py>( .nth(start) .map_or("", |(index, _)| &edited[index..]); let query: Bound<'py, PyString> = PyString::new(py, python_strip(tail)); - Ok(Bound::new(py, native.with_text(py, text.unbind(), query.unbind()))?.into_any()) + Ok(Bound::new(py, native.with_text(py, text.unbind(), query.unbind())?)?.into_any()) } fn updated_inputs<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs index 1c60070c08..9d21f0f735 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fixture_schemas.rs @@ -41,11 +41,11 @@ fn model_file_view<'py>(model: &Bound<'py, PyAny>) -> PyResult() { let native: &NativeModelFile = native.get(); - let (file_path, relative_path) = native.paths(py); + let (file_path, relative_path) = native.path_objects(py)?; return Ok(ModelFileView { - file_path: file_path.clone(), - relative_path: relative_path.clone(), - header_values: native.header_value_dict(py).clone(), + file_path, + relative_path, + header_values: native.header_value_dict(py)?, header_column_locations: native.header_locations(py)?, }); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 0252f0f7e6..7e6a2607ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::path::PathBuf; +use std::sync::Arc; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -33,23 +34,18 @@ pub(crate) fn model_key_row(model: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - record - .model_file - .bind(py) - .getattr(FILE_PATH_ATTRIBUTE)? - .extract() +fn record_path(record: &ModelInputRecord) -> PathBuf { + record.model_file.get().file_path_buf() } /// The record's shared texts with the given passes, read only when their file is linted. fn expansion( expansion: &ExpansionRecord, passes: &[&Vec<(usize, usize, usize, usize)>], - py: Python<'_>, ) -> CompiledSource { CompiledSource::Shared { - authored_sql: expansion.authored_sql.clone_ref(py), - expanded_sql: expansion.expanded_sql.clone_ref(py), + authored_sql: Arc::clone(&expansion.authored_sql), + expanded_sql: Arc::clone(&expansion.expanded_sql), passes: passes.iter().map(|rows| span_pass(rows)).collect(), } } @@ -60,10 +56,10 @@ pub(crate) fn model_lint_expansion( ) -> PyResult> { let py: Python<'_> = model.py(); if let Ok(input) = model.downcast::() { - return input.get().read(py, |record, py| { + return input.get().read(py, |record, _| { let whole: &ExpansionRecord = &record.expansion; Ok(Some(( - record_path(record, py)?, + record_path(record), expansion( whole, &[ @@ -71,7 +67,6 @@ pub(crate) fn model_lint_expansion( &whole.declaration_spans, &whole.macro_spans, ], - py, ), ))) }); @@ -104,11 +99,10 @@ pub(crate) fn project_sql_expansions( let authored: &ExpansionRecord = &record.expansion; if authored.authored { expansions.insert( - record_path(record, py)?, + record_path(record), expansion( authored, &[&authored.declaration_spans, &authored.macro_spans], - py, ), ); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index d9986069f2..fe8990db6d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -2,11 +2,12 @@ use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use pyo3::exceptions::{PyException, PyValueError}; -use pyo3::prelude::{Bound, IntoPyObject, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use sqlbuild_core::text::main::python_isspace::python_isspace; use sqlbuild_core::text::main::python_str_repr::python_str_repr; @@ -176,8 +177,8 @@ impl CompiledExpansion { pub(crate) enum CompiledSource { Owned(CompiledExpansion), Shared { - authored_sql: Py, - expanded_sql: Py, + authored_sql: Arc, + expanded_sql: Arc, passes: Vec>, }, } @@ -192,18 +193,18 @@ impl CompiledSource { } /// The shared texts as an expansion of their own, built only while one file is linted. - fn shared(&self, py: Python<'_>) -> PyResult> { + fn shared(&self) -> Option { match self { - Self::Owned(_) => Ok(None), + Self::Owned(_) => None, Self::Shared { authored_sql, expanded_sql, passes, - } => Ok(Some(CompiledExpansion { - authored_sql: authored_sql.bind(py).to_str()?.to_owned(), - expanded_sql: expanded_sql.bind(py).to_str()?.to_owned(), + } => Some(CompiledExpansion { + authored_sql: authored_sql.to_string(), + expanded_sql: expanded_sql.to_string(), passes: passes.clone(), - })), + }), } } } @@ -592,7 +593,7 @@ pub(crate) fn lint( } }; let file: &LintFile = &files[plan.file]; - let (traits, bodies) = match shared_plan(py, request, file)? { + let (traits, bodies) = match shared_plan(request, file) { Some(Ok(planned)) => planned, Some(Err(failure)) => { scan_failure = Some(failure.into_pyerr(py)); @@ -643,29 +644,20 @@ pub(crate) fn lint( } /// A file's bodies planned again against its shared compile expansion, read only for this file. -fn shared_plan( - py: Python<'_>, - request: &LintRequest<'_>, - file: &LintFile, -) -> PyResult> { - let Some(source) = request.compiled_expansions.get(&file.path) else { - return Ok(None); - }; - let Some(compiled) = source.shared(py)? else { - return Ok(None); - }; +fn shared_plan(request: &LintRequest<'_>, file: &LintFile) -> Option { + let compiled: CompiledExpansion = request.compiled_expansions.get(&file.path)?.shared()?; let Ok(relative) = file.path.strip_prefix(request.project_dir) else { - return Ok(None); + return None; }; let parts: Vec = path_parts(relative); let text: Vec = file.contents.chars().collect(); let headers: Vec = scan_headers(&text, first_header_only(&parts)); - Ok(Some(plan_bodies( + Some(plan_bodies( request, file, (&text, &headers), (&parts, Some(&compiled)), - ))) + )) } /// Each file's headers, header violations and bodies in path order, up to the first failure. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index f39c6ee92c..4b2670b1b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -8,6 +8,7 @@ pub(crate) mod custom_hosts; pub(crate) mod custom_rule_cache; pub(crate) mod custom_rule_evidence; pub(crate) mod dsl_headers; +pub(crate) mod early_builtin_rules; pub(crate) mod early_sql_lint; pub(crate) mod evaluation; pub(crate) mod fact_encoder; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 7ab29e953d..cef85ca350 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use std::path::PathBuf; use std::sync::Mutex; +use std::sync::atomic::AtomicBool; use pyo3::exceptions::{PyOSError, PyTypeError, PyValueError}; use pyo3::prelude::{ @@ -38,11 +39,12 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::config_json::asdict_json; +use crate::bindings::_helpers::rules::early_builtin_rules::EarlyBuiltinRules; use crate::bindings::_helpers::rules::fact_replay::{project_attribute, project_models}; use crate::bindings::_helpers::rules::findings::{FindingClasses, finding_fault}; use crate::bindings::_helpers::rules::selection::rules_error; use crate::bindings::_helpers::rules::sql_test_objects::{scenario_fact, sql_test_fact}; -use crate::bindings::_helpers::scopes::scope_reports::metadata_projection; +use crate::bindings::_helpers::scopes::scope_reports::metadata_projection_value; use crate::bindings::_helpers::type_system::equality::types_equal; use crate::bindings::types::CompilerDetach; @@ -107,29 +109,108 @@ pub(crate) fn build_rules_request( ) -> PyResult { let request = Request::new("build_rules_request", options)?; let py: Python<'_> = request.py(); - as_rules_error(py, built_request(&request)) + as_rules_error( + py, + built_request(&request, None).map(|request| NativeRulesRequest { + request: Mutex::new(Some(request)), + }), + ) } -fn built_request(options: &Request<'_, '_>) -> PyResult { - let py: Python<'_> = options.py(); - let project: Bound<'_, PyAny> = options.get("project")?; - let config: Bound<'_, PyAny> = options.get("config")?; - let project_dir: Bound<'_, PyAny> = options.get("project_dir")?; - let dialect: Bound<'_, PyAny> = options.get("dialect")?; +/// `build_rules_request`'s request, reading model facts from `facts` instead of the project. +pub(crate) fn rules_request_over( + options: &Bound<'_, PyDict>, + facts: &Bound<'_, NativeCompiledProject>, +) -> PyResult { + built_request( + &Request::new("build_rules_request", Some(options))?, + Some(facts), + ) +} + +/// The built request `request` holds, which only one evaluation may take. +pub(crate) fn taken_rows(request: &Bound<'_, NativeRulesRequest>) -> PyResult { + request + .borrow() + .request + .lock() + .map_err(|_| value_error("rules request lock is poisoned"))? + .take() + .ok_or_else(|| value_error("rules request was already evaluated")) +} + +/// The model-free request of a run selecting no rules, which only validates its configuration. +pub(crate) fn selectionless_rules_request( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + dialect: &str, + custom_payloads: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = config.py(); + let header: PyResult = dialect + .into_bound_py_any(py) + .and_then(|dialect| request_header(config, project_dir, &dialect, (None, custom_payloads))); + as_rules_error( + py, + header.and_then(|header| { + compiler_guard(|| { + let request: EvaluateRequest = + serde_json::from_value(header).map_err(value_error)?; + Ok(NativeRulesRequest { + request: Mutex::new(Some(RowsRequest { + request, + memo: None, + })), + }) + }) + }), + ) +} + +/// The request's version, paths, dialect, config, custom rules and, when given, scope index. +fn request_header( + config: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, + (project, custom_payloads): (Option<&Bound<'_, PyAny>>, &Bound<'_, PyAny>), +) -> PyResult { + let py: Python<'_> = config.py(); let resolved: Bound<'_, PyAny> = project_dir.call_method0("resolve")?; let resolved_path: PathBuf = resolved.extract()?; - let header: Value = json!({ + let mut header: Value = json!({ "version": API_VERSION, "project_dir": resolved.str()?.to_str()?, "dialect": dialect.extract::()?, - "config": config_value(&config)?, - "scope_index": plain_value( - metadata_projection(py, &project_attribute(&project, "scope_index")?)?.bind(py), - )?, - "defer_suppressions": true, - "custom_rules": plain_value(&options.get("custom_payloads")?)?, - "rules_cache_path": path_text(py, rules_bulk_cache_path(&resolved_path, NATIVE_RULES_CACHE_FILE))?, + "config": config_value(config)?, }); + if let Some(project) = project { + header["scope_index"] = + metadata_projection_value(&project_attribute(project, "scope_index")?)?; + } + header["defer_suppressions"] = Value::Bool(true); + header["custom_rules"] = plain_value(custom_payloads)?; + header["rules_cache_path"] = Value::String(path_text( + py, + rules_bulk_cache_path(&resolved_path, NATIVE_RULES_CACHE_FILE), + )?); + Ok(header) +} + +fn built_request( + options: &Request<'_, '_>, + facts: Option<&Bound<'_, NativeCompiledProject>>, +) -> PyResult { + let py: Python<'_> = options.py(); + let project: Bound<'_, PyAny> = options.get("project")?; + let config: Bound<'_, PyAny> = options.get("config")?; + let project_dir: Bound<'_, PyAny> = options.get("project_dir")?; + let dialect: Bound<'_, PyAny> = options.get("dialect")?; + let header: Value = request_header( + &config, + &project_dir, + &dialect, + (Some(&project), &options.get("custom_payloads")?), + )?; let (public_enums, public_constants) = declaration_rows( PyTuple::new( py, @@ -178,7 +259,10 @@ fn built_request(options: &Request<'_, '_>) -> PyResult { .collect::>()?, dialect, }; - let facts: Bound<'_, NativeCompiledProject> = compiled_project_facts(&project)?; + let facts: Bound<'_, NativeCompiledProject> = match facts { + Some(facts) => facts.clone(), + None => compiled_project_facts(&project)?, + }; let names: Vec = project_models(&project)? .into_iter() .map(|model| model.name) @@ -213,9 +297,7 @@ fn built_request(options: &Request<'_, '_>) -> PyResult { .memo .take() .map(|(identity, path)| (identity, PathBuf::from(path))); - Ok(NativeRulesRequest { - request: Mutex::new(Some(RowsRequest { request, memo })), - }) + Ok(RowsRequest { request, memo }) }) } @@ -225,27 +307,35 @@ pub(crate) fn evaluate_rules_request<'py>( py: Python<'py>, request: &Bound<'py, NativeRulesRequest>, ) -> PyResult> { - as_rules_error(py, evaluated(py, request)) + as_rules_error(py, evaluated(py, request, None)) +} + +/// `evaluate_rules_request`, answered by `early` when identical; `interrupted` stops the wait. +pub(crate) fn evaluate_rules_request_with<'py>( + py: Python<'py>, + request: &Bound<'py, NativeRulesRequest>, + early: EarlyBuiltinRules, + interrupted: &AtomicBool, +) -> PyResult> { + as_rules_error(py, evaluated(py, request, Some((early, interrupted)))) } fn evaluated<'py>( py: Python<'py>, request: &Bound<'py, NativeRulesRequest>, + early: Option<(EarlyBuiltinRules, &AtomicBool)>, ) -> PyResult> { - let request: RowsRequest = request - .borrow() - .request - .lock() - .map_err(|_| value_error("rules request lock is poisoned"))? - .take() - .ok_or_else(|| value_error("rules request was already evaluated"))?; - let evaluation: RowsEvaluation = py - .compiler_detach(|| Ok(evaluate_rows(request))) - .map_err(value_error)? - .map_err(|error| match error { - RowsError::Rules(message) => value_error(message), - RowsError::Io(error) => PyOSError::new_err(error.to_string()), - })?; + let request: RowsRequest = taken_rows(request)?; + let evaluation: Result = match early { + None => py + .compiler_detach(|| Ok(evaluate_rows(request))) + .map_err(value_error)?, + Some((early, interrupted)) => early.answer(py, request, interrupted)?, + }; + let evaluation: RowsEvaluation = evaluation.map_err(|error| match error { + RowsError::Rules(message) => value_error(message), + RowsError::Io(error) => PyOSError::new_err(error.to_string()), + })?; let classes = FindingClasses::new(py)?; let findings: Vec> = evaluation .faults diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index ac9dcc3ecf..7334eb244e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::rules::findings::FindingClasses; use sqlbuild_rules::rule_cache::main::lint_key::lint_key; use sqlbuild_rules::rule_cache::main::lookup_rule_cache::lookup_rule_cache; -use sqlbuild_rules::rule_cache::main::model_rule_key::model_rule_key; +use sqlbuild_rules::rule_cache::main::model_rule_keys::model_rule_keys as parallel_model_rule_keys; use sqlbuild_rules::rule_cache::main::parse_rule_cache::parse_rule_cache; use sqlbuild_rules::rule_cache::main::rule_cache_text::rule_cache_text; use sqlbuild_rules::rule_cache::models::{CacheLookup, CachedEntry, LintKeyInput, ModelKeyInput}; @@ -56,29 +56,34 @@ fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { #[pyfunction] #[pyo3(signature = (*, lint_key, codes, models))] fn sql_model_rule_keys( + py: Python<'_>, lint_key: &str, codes: Vec, models: Vec<(String, String, String, Option)>, ) -> BTreeMap { - model_rule_keys(lint_key, &codes, &models) + model_rule_keys(py, lint_key, &codes, &models) } -/// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. +/// Each model path's SQL rule cache key, hashed in parallel with the GIL released. pub(crate) fn model_rule_keys( + py: Python<'_>, lint_key: &str, codes: &[String], models: &[(String, String, String, Option)], ) -> BTreeMap { - models + let inputs: Vec> = models .iter() - .map(|(path, authored_sql, query_sql, contract)| { - let model = ModelKeyInput { - authored_sql, - query_sql, - contract: contract.as_deref(), - }; - (path.clone(), model_rule_key(lint_key, codes, &model)) + .map(|(_, authored_sql, query_sql, contract)| ModelKeyInput { + authored_sql, + query_sql, + contract: contract.as_deref(), }) + .collect(); + let keys: Vec = py.detach(|| parallel_model_rule_keys(lint_key, codes, &inputs)); + models + .iter() + .map(|(path, ..)| path.clone()) + .zip(keys) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs index aa7b557440..3d30794f32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, keywords, }; use crate::bindings::_helpers::compile_driver::stage_work::count_stage; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::records::construct::new_record_with; @@ -34,14 +35,20 @@ use crate::bindings::_helpers::rules::custom_evaluation::{ }; use crate::bindings::_helpers::rules::custom_hosts::HostCancel; use crate::bindings::_helpers::rules::custom_rule_evidence::verify_custom_rules; +use crate::bindings::_helpers::rules::early_builtin_rules::{ + EarlyBuiltinRules, EarlyRulesReport, take_early_rules, +}; use crate::bindings::_helpers::rules::rows::finalize_rule_findings_rows; use crate::bindings::_helpers::rules::rows::{ - NativeRulesRequest, build_rules_request, evaluate_rules_request, + NativeRulesRequest, build_rules_request, evaluate_rules_request, evaluate_rules_request_with, + rules_request_over, selectionless_rules_request, }; use crate::bindings::_helpers::rules::rule_plan::plan_rule_run; use crate::bindings::_helpers::rules::rule_run::finish_rule_run; use crate::bindings::_helpers::rules::selection::{rule_selected, selected_rule_codes}; use crate::bindings::_helpers::rules::sql_rules::run_planned_sql_rules; +use sqlbuild_rules::models::RowsRequest; +use std::sync::Arc; const CUSTOM_THREAD: &str = "sqlbuild-custom-rules"; const GENERIC_DIALECT: &str = "generic"; @@ -159,7 +166,7 @@ fn evaluate_project_rules<'py>( initial_findings: None, cancel: HostCancel::default(), }; - evaluate_native(&run, (false, None), CustomSource::Deferred) + evaluate_native(&run, (false, None), CustomSource::Deferred, None) } /// Evaluate built-in rules natively with the catalogue's selected custom rules beside them. @@ -183,7 +190,12 @@ fn evaluate_native_rules<'py>( initial_findings: (!initial_findings.is_none()).then_some(initial_findings), cancel: HostCancel::default(), }; - evaluate_native(&run, (defer_suppressions, None), CustomSource::Deferred) + evaluate_native( + &run, + (defer_suppressions, None), + CustomSource::Deferred, + None, + ) } /// Evaluate custom rules through their incremental cache. @@ -268,27 +280,38 @@ pub(crate) fn evaluate_rules<'py>( ]; let native_dialect: String = run.dialect.clone(); let native_cancel: HostCancel = run.cancel.clone(); - let native_thread: PhaseThread = PhaseThread::spawn(RULES_THREAD, move || { - Python::attach(|py| { - let [project, config, project_dir, catalogue, payloads] = native_run; - let run: NativeRun<'_> = NativeRun { - project: project.into_bound(py), - config: config.into_bound(py), - project_dir: project_dir.into_bound(py), - catalogue: catalogue.into_bound(py), - dialect: native_dialect, - verify_determinism: false, - initial_findings: None, - cancel: native_cancel, - }; - evaluate_native( - &run, - (true, Some(payloads.into_bound(py))), - CustomSource::Started(custom), - ) - .map(Bound::unbind) - }) - })?; + let selectionless: bool = custom.is_none() && field(&plan, "selected")?.len()? == 0; + let early: Option = take_early_rules(); + let early_report: Option> = early.as_ref().map(EarlyBuiltinRules::report); + let native_thread: Option = if selectionless { + if let Some(early) = early { + early.discard(py); + } + None + } else { + Some(PhaseThread::spawn(RULES_THREAD, move || { + Python::attach(|py| { + let [project, config, project_dir, catalogue, payloads] = native_run; + let run: NativeRun<'_> = NativeRun { + project: project.into_bound(py), + config: config.into_bound(py), + project_dir: project_dir.into_bound(py), + catalogue: catalogue.into_bound(py), + dialect: native_dialect, + verify_determinism: false, + initial_findings: None, + cancel: native_cancel, + }; + evaluate_native( + &run, + (true, Some(payloads.into_bound(py))), + CustomSource::Started(custom), + early, + ) + .map(Bound::unbind) + }) + })?) + }; let sql_result: PyResult> = run_planned_sql_rules(Some(&keywords( py, &[ @@ -300,17 +323,24 @@ pub(crate) fn evaluate_rules<'py>( ], )?)); cancel_on_interrupt(py, &sql_result, &run.cancel); - let result: PyResult> = joined(py, native_thread, &run.cancel); + let result: PyResult> = match native_thread { + Some(native_thread) => joined(py, native_thread, &run.cancel), + None => selectionless_result(&run, &payloads), + }; + if let Some(report) = early_report { + report.record(); + } let sql_result: Bound<'py, PyAny> = sql_result?; let result: Bound<'py, PyAny> = result?; finish_rule_run(&plan, &sql_result, &result) } -/// Evaluate built-in rules natively while custom rules run beside them. +/// Evaluate built-in rules natively, or from `early` when identical, beside the custom rules. fn evaluate_native<'py>( run: &NativeRun<'py>, (defer_suppressions, custom_payloads): (bool, Option>), custom: CustomSource, + early: Option, ) -> PyResult> { let py: Python<'py> = run.project.py(); let prepared: PyResult<(Bound<'py, PyAny>, Bound<'py, NativeRulesRequest>)> = @@ -326,13 +356,54 @@ fn evaluate_native<'py>( (CustomSource::Deferred, Ok((payloads, _))) => start_custom_rules(run, payloads)?, (CustomSource::Deferred, Err(_)) => None, }; - let (_, request): (Bound<'py, PyAny>, Bound<'py, NativeRulesRequest>) = prepared?; - let evaluated: PyResult> = evaluate_rules_request(py, &request); + let (_, request): (Bound<'py, PyAny>, Bound<'py, NativeRulesRequest>) = match prepared { + Ok(prepared) => prepared, + Err(error) => { + if let Some(early) = early { + early.discard(py); + } + return Err(error); + } + }; + let evaluated: PyResult> = match early { + Some(early) => evaluate_rules_request_with(py, &request, early, &run.cancel), + None => evaluate_rules_request(py, &request), + }; cancel_on_interrupt(py, &evaluated, &run.cancel); let custom: PyResult> = match custom { Some(handle) => joined(py, handle, &run.cancel), None => outcome(py, (Vec::new(), 0, 0), (0, 0)), }; + rules_result(run, defer_suppressions, evaluated, custom) +} + +/// The built-in rules result of a run that selects no rules, without building its models. +fn selectionless_result<'py>( + run: &NativeRun<'py>, + payloads: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = run.project.py(); + let request: Bound<'py, NativeRulesRequest> = Bound::new( + py, + selectionless_rules_request(&run.config, &run.project_dir, &run.dialect, payloads)?, + )?; + let evaluated: PyResult> = evaluate_rules_request(py, &request); + rules_result( + run, + true, + evaluated, + outcome(py, (Vec::new(), 0, 0), (0, 0)), + ) +} + +/// Combine one native evaluation with its custom-rule outcome into a `RulesResult`. +fn rules_result<'py>( + run: &NativeRun<'py>, + defer_suppressions: bool, + evaluated: PyResult>, + custom: PyResult>, +) -> PyResult> { + let py: Python<'py> = run.project.py(); let native_result: Bound<'py, PyAny> = evaluated?; let custom: Bound<'py, PyAny> = custom?; let combined: Vec> = field(&native_result, "findings")? @@ -410,34 +481,65 @@ fn native_request<'py>( ], )?, }; + let request: Bound<'py, NativeRulesRequest> = Bound::new( + py, + build_rules_request(Some(&request_options(run, &payloads)?))?, + )?; + Ok((payloads, request)) +} + +/// The `build_rules_request` options of one native evaluation with its custom-rule payloads. +fn request_options<'py>( + run: &NativeRun<'py>, + payloads: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = run.project.py(); let mut include_type_proof: bool = false; for code in TYPE_PROOF_RULE_CODES { include_type_proof = include_type_proof || rule_selected(&run.config, code)?; } - let request: Bound<'py, NativeRulesRequest> = Bound::new( + keywords( py, - build_rules_request(Some(&keywords( - py, - &[ - ("project", run.project.clone()), - ("config", run.config.clone()), - ("project_dir", run.project_dir.clone()), - ("dialect", run.dialect.clone().into_bound_py_any(py)?), - ( - "initial_findings", - run.initial_findings - .clone() - .unwrap_or_else(|| PyTuple::empty(py).into_any()), - ), - ("custom_payloads", payloads.clone()), - ( - "include_type_proof", - include_type_proof.into_bound_py_any(py)?, - ), - ], - )?))?, - )?; - Ok((payloads, request)) + &[ + ("project", run.project.clone()), + ("config", run.config.clone()), + ("project_dir", run.project_dir.clone()), + ("dialect", run.dialect.clone().into_bound_py_any(py)?), + ( + "initial_findings", + run.initial_findings + .clone() + .unwrap_or_else(|| PyTuple::empty(py).into_any()), + ), + ("custom_payloads", payloads.clone()), + ( + "include_type_proof", + include_type_proof.into_bound_py_any(py)?, + ), + ], + ) +} + +/// The request `evaluate_rules` would build for `plan`, reading model facts from `facts`. +pub(crate) fn speculative_rules_request<'py>( + plan: &Bound<'py, PyAny>, + dialect: &str, + facts: &Bound<'py, NativeCompiledProject>, +) -> PyResult { + let run: NativeRun<'py> = NativeRun { + project: field(plan, "project")?, + config: field(plan, "config")?, + project_dir: field(plan, "project_dir")?, + catalogue: field(plan, "catalogue")?, + dialect: dialect.to_owned(), + verify_determinism: false, + initial_findings: None, + cancel: HostCancel::default(), + }; + rules_request_over( + &request_options(&run, &field(plan, "custom_payloads")?)?, + facts, + ) } /// Start the selected custom rules on their thread after digesting installed code here. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs index 60127a6820..5d2745fb88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_plan.rs @@ -36,9 +36,27 @@ pub(crate) fn plan_rule_run<'py>( let graph: Bound<'py, PyAny> = options.get("graph")?; let project: Bound<'py, PyAny> = field(&graph, "project")?; freed_memory_released(py)?; - let effective_config: Bound<'py, PyAny> = - resolved_ignore_selectors(&options.get("config")?, &project)?; - let project_dir: Bound<'py, PyAny> = options.get("project_dir")?.call_method0("resolve")?; + let selected_keys: Bound<'py, PyAny> = options.get("selected_keys")?; + rule_run_plan( + ( + &project, + &options.get("config")?, + &options.get("project_dir")?, + ), + Some((&graph, &selected_keys)), + options.get("no_sql_analysis")?.is_truthy()?, + ) +} + +/// `plan_rule_run`'s plan of `project`, selected by `(graph, selected_keys)` when given. +pub(crate) fn rule_run_plan<'py>( + (project, config, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &Bound<'py, PyAny>), + selection: Option<(&Bound<'py, PyAny>, &Bound<'py, PyAny>)>, + no_sql_analysis: bool, +) -> PyResult> { + let py: Python<'py> = project.py(); + let effective_config: Bound<'py, PyAny> = resolved_ignore_selectors(config, project)?; + let project_dir: Bound<'py, PyAny> = project_dir.call_method0("resolve")?; let include_custom: bool = references_custom_rules(&effective_config)?; let catalogue: Vec> = rule_catalogue(&effective_config, &project_dir, include_custom)?; @@ -57,8 +75,8 @@ pub(crate) fn plan_rule_run<'py>( Some(custom_payloads.clone()), )?; let mut skipped_type_proof_rules: Vec> = Vec::new(); - if options.get("no_sql_analysis")?.is_truthy()? - || !field(&project_attribute(&project, "settings")?, "sql_analysis")?.is_truthy()? + if no_sql_analysis + || !field(&project_attribute(project, "settings")?, "sql_analysis")?.is_truthy()? { let mut kept: Vec> = Vec::with_capacity(selected.len()); for rule in selected { @@ -80,11 +98,13 @@ pub(crate) fn plan_rule_run<'py>( if !custom_rules.is_empty() { verify_custom_rules(&custom_rules, &project_dir)?; } - let model_paths: Option> = - selected_model_paths(&graph, &options.get("selected_keys")?)?; + let model_paths: Option> = match selection { + Some((graph, selected_keys)) => selected_model_paths(graph, selected_keys)?, + None => None, + }; let selected_project: Bound<'py, PyAny> = match &model_paths { None => project.clone(), - Some(paths) => selected_models_project(&project, paths)?, + Some(paths) => selected_models_project(project, paths)?, }; let mut codes: Vec> = Vec::with_capacity(selected.len()); for rule in &selected { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs index 5f53eaf5ca..56f766520f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_run.rs @@ -88,8 +88,10 @@ pub(crate) fn finish_rule_run<'py>( .map(|finding| finding.path.clone()) .collect(); let mut unevaluated_models: i64 = 0; - for model in project_models(&record_field(plan, "project")?)? { - unevaluated_models += i64::from(unevaluated.contains(&model.posix)); + if !unevaluated.is_empty() { + for model in project_models(&record_field(plan, "project")?)? { + unevaluated_models += i64::from(unevaluated.contains(&model.posix)); + } } let findings: Vec> = grouped .into_iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index 370ad14585..d3124c629b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -1,5 +1,6 @@ //! Run the planned SQL rules through native lint, reusing cached and early-lint results. +use std::collections::hash_map::Entry; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::time::Instant; @@ -213,14 +214,17 @@ fn sql_rule_models<'py>( if !field(&project_attribute(project, "settings")?, "sql_analysis")?.is_truthy()? { return Ok(models); } + let mut positions: HashMap = HashMap::new(); for model in project_attribute(project, "models")?.try_iter()? { let Some(entry) = sql_rule_model(model?, keyed)? else { continue; }; - if let Some(index) = models.iter().position(|known| known.path == entry.path) { - models[index] = entry; - } else { - models.push(entry); + match positions.entry(entry.path.clone()) { + Entry::Occupied(known) => models[*known.get()] = entry, + Entry::Vacant(slot) => { + slot.insert(models.len()); + models.push(entry); + } } } Ok(models) @@ -292,7 +296,8 @@ fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult = model_rule_keys(lint_key, run.codes, &models); + let keys: BTreeMap = + model_rule_keys(run.project.py(), lint_key, run.codes, &models); Ok(models .into_iter() .map(|(path, ..)| { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs index 68014b23f0..6f50a0d376 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_report_objects.rs @@ -24,6 +24,8 @@ use crate::bindings::_helpers::scopes::scope_objects::{ type PyObject = Py; const PATHLIB_MODULE: &str = "pathlib"; +/// The error `plain_value` raises for an integer outside 64 bits. +const WIDE_INTEGER: &str = "Integer exceeds 64-bit range"; fn texts(item: &Bound<'_, PyAny>, name: &str) -> PyResult> { native_field(item, name)? @@ -130,6 +132,47 @@ pub(crate) fn json_to_python(py: Python<'_>, value: JsonValue, tuples: bool) -> } } +/// `plain_value(int(integer))`: a 64-bit signed or unsigned number, else the wide-integer error. +fn plain_integer(integer: &JsonInteger) -> PyResult { + let Some(number) = integer.to_i128() else { + return Err(PyValueError::new_err(WIDE_INTEGER)); + }; + if let Ok(signed) = i64::try_from(number) { + return Ok(serde_json::Value::from(signed)); + } + match u64::try_from(number) { + Ok(unsigned) => Ok(serde_json::Value::from(unsigned)), + Err(_) => Err(PyValueError::new_err(WIDE_INTEGER)), + } +} + +/// `plain_value(json_to_python(value, False))` computed without building the Python value. +pub(crate) fn json_to_plain(value: JsonValue) -> PyResult { + Ok(match value { + JsonValue::Null => serde_json::Value::Null, + JsonValue::Bool(flag) => serde_json::Value::Bool(flag), + JsonValue::Integer(integer) => plain_integer(&integer)?, + JsonValue::Float(number) => serde_json::Number::from_f64(number) + .map_or(serde_json::Value::Null, serde_json::Value::Number), + JsonValue::String(text) => serde_json::Value::String(text), + JsonValue::Utf16(units) => serde_json::Value::String(String::from_utf16_lossy(&units)), + JsonValue::Array(items) => serde_json::Value::Array( + items + .into_iter() + .map(json_to_plain) + .collect::>()?, + ), + JsonValue::Object(entries) => { + let mut object: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for (key, item) in entries { + let _replaced = object.insert(key, json_to_plain(item)?); + } + serde_json::Value::Object(object.into_iter().collect()) + } + }) +} + /// `_value` of one metadata value; an unsupported type is the error Python's serializer raises. pub(crate) fn python_to_json(value: &Bound<'_, PyAny>) -> PyResult { if value.is_none() { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index 94fdc75d68..9583d93bff 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -32,9 +32,9 @@ use crate::bindings::_helpers::scopes::scope_objects::{ read_declaration, read_index, }; use crate::bindings::_helpers::scopes::scope_report_objects::{ - browse_object, diagnostics_object, explanation_object, json_to_python, list_object, - move_object, read_declaration_target, read_filters, read_optional_target, read_result, - read_target, report_object, + browse_object, diagnostics_object, explanation_object, json_to_plain, json_to_python, + list_object, move_object, read_declaration_target, read_filters, read_optional_target, + read_result, read_target, report_object, }; type PyObject = Py; @@ -318,9 +318,17 @@ pub(crate) fn serialize_scope_report( #[pyfunction] #[pyo3(name = "scope_metadata_projection", signature = (*, index))] pub(crate) fn metadata_projection(py: Python<'_>, index: &Bound<'_, PyAny>) -> PyResult { + json_to_python(py, projection_json(index)?, false) +} + +/// `plain_value` of `scope_metadata_projection(index=index)`, without a Python round trip. +pub(crate) fn metadata_projection_value(index: &Bound<'_, PyAny>) -> PyResult { + json_to_plain(projection_json(index)?) +} + +fn projection_json(index: &Bound<'_, PyAny>) -> PyResult { let index: ProjectionIndex = projection_index(read_index(index)?); - let projection: JsonValue = scope_metadata_projection(&index); - json_to_python(py, projection, false) + Ok(scope_metadata_projection(&index)) } fn optional<'py>( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 02aeae8369..4be1bbc2fa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -30,6 +30,8 @@ use crate::bindings::_helpers::semantic_checks::type_recovery::items; use crate::bindings::_helpers::semantic_shapes::shapes::project_shapes; use crate::bindings::types::CompilerDetach; +/// `SqlReferenceKind.UDF` and `SqlReferenceKind.TABLE_FUNCTION` values. +const FUNCTION_REFERENCE_KINDS: [&str; 2] = ["udf", "table_fn"]; const METADATA_CONFIG_KEYS: [&str; 4] = [ "unique_key", "cursor", @@ -444,18 +446,32 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes upstreams.push((upstream.str()?.extract()?, column_names(&columns)?)); } } - let mut calls_functions: bool = false; - for reference in native_field(model, "references")?.try_iter()? { - let kind: Bound<'_, PyAny> = native_field(&reference?, "ref_kind")?; - if kind.eq(&classes.udf_kind)? || kind.eq(&classes.table_function_kind)? { - calls_functions = true; - break; + let (name, query_sql, authored_sql, calls_functions) = match model.cast::() + { + Ok(native) => native + .get() + .metadata_texts(model.py(), &FUNCTION_REFERENCE_KINDS)?, + Err(_) => { + let mut calls_functions: bool = false; + for reference in native_field(model, "references")?.try_iter()? { + let kind: Bound<'_, PyAny> = native_field(&reference?, "ref_kind")?; + if kind.eq(&classes.udf_kind)? || kind.eq(&classes.table_function_kind)? { + calls_functions = true; + break; + } + } + ( + native_field(model, "name")?.extract()?, + native_field(model, "query_sql")?.extract()?, + native_field(model, "authored_sql")?.extract()?, + calls_functions, + ) } - } + }; Ok(MetadataModel { - name: native_field(model, "name")?.extract()?, - query_sql: native_field(model, "query_sql")?.extract()?, - authored_sql: native_field(model, "authored_sql")?.extract()?, + name, + query_sql, + authored_sql, checked: checked(classes, model)?, calls_functions, references, @@ -480,8 +496,7 @@ fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { attached.push((None, audit)); } for (index, name) in audited { - let column: Bound<'_, PyAny> = columns.bind(py).get_item(index)?; - for audit in native_field(&column, "audits")?.try_iter()? { + for audit in columns.specs[index].field(py, "audits")?.try_iter()? { attached.push((Some(name.clone()), audit?)); } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 60c4367e72..42644f97bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -15,6 +15,7 @@ use sqlbuild_analysis::semantic_checks::models::{ use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::compiled_project::audit_views::audit_reference_rows; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::record_class; @@ -227,10 +228,19 @@ fn model_input( }); } } + let (name, path, authored_sql): (String, String, Arc) = + match model.cast::() { + Ok(native) => native.get().file_texts(model.py())?, + Err(_) => ( + native_field(model, "name")?.extract()?, + native_field(model, "relative_path")?.str()?.extract()?, + Arc::from(native_field(model, "authored_sql")?.extract::()?), + ), + }; Ok(HookModelInput { - name: native_field(model, "name")?.extract()?, - path: native_field(model, "relative_path")?.str()?.extract()?, - authored_sql: Arc::from(native_field(model, "authored_sql")?.extract::()?), + name, + path, + authored_sql, analysis_disabled: opts_out(&values)?, hooks, }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs index b89ec1db02..14e1670531 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -11,6 +11,7 @@ use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSes use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::rules::early_builtin_rules::start_early_rules; use crate::bindings::_helpers::semantic_checks::completion::completed_diagnostics; use crate::bindings::_helpers::semantic_checks::metadata::metadata_diagnostics; use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; @@ -71,6 +72,7 @@ pub(crate) fn completed_semantic_project<'py>( }; let recovered: Bound<'py, PyAny> = recovered_types(&classes, &project, &binding_results, &shared)?; + start_early_rules(&recovered)?; let metadata: Vec> = metadata_diagnostics( &classes, (&recovered, &profile), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs index 1be9e9d02e..7d824578c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/authored_values.rs @@ -5,10 +5,10 @@ use crate::bindings::_helpers::records::ids::RecordId; use pyo3::prelude::{IntoPyObject, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use sqlbuild_sqltext::compiler::main::header_word_value::header_word_value; -use sqlbuild_sqltext::compiler::models::HeaderWord; +use sqlbuild_sqltext::compiler::models::{AuthoredValue, HeaderWord}; -fn word_to_python(py: Python<'_>, word: String) -> PyResult> { - match header_word_value(&word) { +fn word_to_python(py: Python<'_>, word: &str) -> PyResult> { + match header_word_value(word) { HeaderWord::Bool(value) => Ok(value.into_pyobject(py)?.to_owned().unbind().into_any()), HeaderWord::Null => Ok(py.None()), HeaderWord::Integer(digits) => match digits.parse::() { @@ -32,23 +32,17 @@ fn instance( Ok(new_record(py, class, (), Some(arguments))?.unbind()) } -fn values_tuple( - py: Python<'_>, - values: Vec, -) -> PyResult> { +fn values_tuple(py: Python<'_>, values: &[AuthoredValue]) -> PyResult> { let projected = values - .into_iter() + .iter() .map(|item| authored_value_to_python(py, item)) .collect::>>()?; Ok(PyTuple::new(py, projected)?.unbind().into_any()) } -fn constant_call( - py: Python<'_>, - values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, -) -> PyResult> { +fn constant_call(py: Python<'_>, values: &[(String, AuthoredValue)]) -> PyResult> { let arguments = values - .into_iter() + .iter() .map(|(key, value)| { let value = authored_value_to_python(py, value)?; PyTuple::new(py, [key.into_pyobject(py)?.unbind().into_any(), value]) @@ -59,12 +53,11 @@ fn constant_call( instance(py, RecordId::AuthoredSqlValueCall, &keywords) } -fn authored_value_to_python( +/// One authored value as the Python header object discovery hands to config builds. +pub(crate) fn authored_value_to_python( py: Python<'_>, - value: sqlbuild_sqltext::compiler::models::AuthoredValue, + value: &AuthoredValue, ) -> PyResult> { - use sqlbuild_sqltext::compiler::models::AuthoredValue; - match value { AuthoredValue::Null => Ok(py.None()), AuthoredValue::Boolean(value) => { @@ -74,7 +67,7 @@ fn authored_value_to_python( AuthoredValue::String(value) => Ok(value.into_pyobject(py)?.unbind().into_any()), AuthoredValue::List(values) => { let projected = values - .into_iter() + .iter() .map(|item| authored_value_to_python(py, item)) .collect::>>()?; Ok(PyList::new(py, projected)?.unbind().into_any()) @@ -103,7 +96,7 @@ fn authored_value_to_python( pub(crate) fn map_to_python( py: Python<'_>, - values: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, + values: &[(String, AuthoredValue)], ) -> PyResult> { let result = PyDict::new(py); for (key, value) in values { @@ -115,8 +108,8 @@ pub(crate) fn map_to_python( fn hook( py: Python<'_>, class: RecordId, - name: String, - kwargs: Vec<(String, sqlbuild_sqltext::compiler::models::AuthoredValue)>, + name: &str, + kwargs: &[(String, AuthoredValue)], ) -> PyResult> { let keywords = PyDict::new(py); keywords.set_item("name", name)?; @@ -126,7 +119,7 @@ fn hook( pub(crate) fn optional_authored_value_to_python( py: Python<'_>, - value: Option, + value: Option<&AuthoredValue>, ) -> PyResult>> { value .map(|item| authored_value_to_python(py, item)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index c66b211658..8d2aab2a9f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -27,7 +27,7 @@ pub(crate) fn parse_model_headers( .into_iter() .map(|(value, offsets, error)| { Ok(( - optional_authored_value_to_python(py, value)?, + optional_authored_value_to_python(py, value.as_ref())?, offsets, error, )) diff --git a/crates/sqlbuild-rules/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml index 0679b5241b..41b5e84013 100644 --- a/crates/sqlbuild-rules/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +blake3.workspace = true fensu-policy.workspace = true globset.workspace = true polyglot-sql.workspace = true diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index c8e55bb51f..dd728e9a56 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -30,7 +30,7 @@ pub(crate) const RULE_CODE_NUMBER_LENGTH: usize = 3; pub(crate) const EMPTY_FIXTURE_QUERY_TOKEN_COUNT: usize = 6; pub(crate) const ENCLOSING_CTE_TOKEN_COUNT: usize = 3; pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; -pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v1"; +pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v2"; pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; /// Threshold key setting the fewest test cases a custom rule must ship. pub const MIN_CUSTOM_RULE_TEST_CASES_KEY: &str = "min_custom_rule_test_cases"; diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs index 720e15600a..75c4553175 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/_helpers/digest.rs @@ -1,21 +1,19 @@ -//! SHA-256 over length-prefixed fields, so no two field lists share an identity. - -use sha2::{Digest, Sha256}; +//! BLAKE3 over length-prefixed fields, so no two field lists share an identity. use crate::constants::NATIVE_BUILD_IDENTITY; use crate::custom_rule_cache::constants::CACHE_VERSION; /// The hex digest of the cache version, the native build and each field in order. pub(crate) fn field_digest(kind: &str, fields: &[&str]) -> String { - let mut digest = Sha256::new(); + let mut digest = blake3::Hasher::new(); for field in [CACHE_VERSION, NATIVE_BUILD_IDENTITY, kind] .iter() .chain(fields.iter()) { - digest.update((field.len() as u64).to_le_bytes()); + digest.update(&(field.len() as u64).to_le_bytes()); digest.update(field.as_bytes()); } - format!("{:x}", digest.finalize()) + digest.finalize().to_hex().to_string() } /// An optional field as a presence flag and its text. diff --git a/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs b/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs index 18d9b415af..5b0fdb3329 100644 --- a/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs +++ b/crates/sqlbuild-rules/src/custom_rule_cache/constants.rs @@ -1,7 +1,7 @@ //! The custom-rule result cache format and the inputs its identities separate. /// Bumped whenever identity derivation or the stored entry shape changes. -pub(crate) const CACHE_VERSION: &str = "custom-rules-native-v1"; +pub(crate) const CACHE_VERSION: &str = "custom-rules-native-v2"; pub(crate) const RULE_KIND: &str = "rule"; pub(crate) const PROJECT_SUBJECT_KIND: &str = "project"; pub(crate) const MODEL_SUBJECT_KIND: &str = "model"; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs index 80efa9b7dc..6430d93cea 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs @@ -43,6 +43,34 @@ impl Cache { ) -> Result<(), String> { write_bucket(&self.native_rules_path, ruleset_identity, bucket) } + + /// The write `put_native_rules_bucket` would make, held until it is published. + pub(crate) fn pending_native_rules_bucket( + &self, + ruleset_identity: &str, + bucket: RuleCacheBucket, + ) -> PendingBucketWrite { + PendingBucketWrite { + path: self.native_rules_path.clone(), + fingerprint: ruleset_identity.to_owned(), + bucket, + } + } +} + +/// One bulk rules cache write that has not been made yet. +#[derive(Debug)] +pub(crate) struct PendingBucketWrite { + path: PathBuf, + fingerprint: String, + bucket: RuleCacheBucket, +} + +impl PendingBucketWrite { + /// Make the write exactly as `Cache::put_native_rules_bucket` makes it. + pub(crate) fn publish(&self) -> Result<(), String> { + write_bucket(&self.path, &self.fingerprint, &self.bucket) + } } fn read_bucket(path: &Path, fingerprint: &str) -> Result { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 86c9fa99f2..e99d55b938 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -1,6 +1,6 @@ use crate::configuration::main::validate as config; use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; -use crate::engine::_helpers::cache::{Cache, RuleCacheEntry}; +use crate::engine::_helpers::cache::{Cache, PendingBucketWrite, RuleCacheEntry}; use crate::models::{ EvaluateRequest, EvaluateResponse, Fault, Model, RuleMetadata, RuleScope, RulesCodeGrammar, RulesConfig, @@ -20,14 +20,39 @@ use fensu_policy::lifecycle::models::{ #[cfg(test)] use rayon::iter::IndexedParallelIterator; use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; -use sha2::{Digest, Sha256}; +use std::cell::RefCell; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; const NATIVE_RULE_WORKERS: usize = 4; const NATIVE_RULE_STACK_BYTES: usize = 16 * 1024 * 1024; const MODEL_RULES_IDENTITY_SEED: &[u8] = b"sqlbuild-rules-model-v2"; +const CANCELLED: &str = "native rules evaluation was cancelled"; + +/// How one evaluation publishes its bulk rules cache bucket and whether it may stop early. +pub(crate) struct EvaluationControl<'a> { + /// `None` writes the bucket at once; `Some` holds the write for the caller to publish. + pub(crate) deferred_bucket: Option<&'a RefCell>>, + /// Stops model evaluation between models once set. + pub(crate) cancel: Option<&'a AtomicBool>, +} + +impl EvaluationControl<'_> { + /// Write the cache at once and never stop early, as an ordinary evaluation does. + pub(crate) fn immediate() -> Self { + Self { + deferred_bucket: None, + cancel: None, + } + } + + fn cancelled(&self) -> bool { + self.cancel + .is_some_and(|cancel| cancel.load(Ordering::Acquire)) + } +} struct PendingModelRule<'a> { index: usize, @@ -45,6 +70,7 @@ struct ModelRuleBatchContext<'a> { request: &'a EvaluateRequest, selected: &'a BTreeMap, threshold_overrides: &'a [ResolvedThresholdOverride], + cancel: Option<&'a AtomicBool>, } struct NativeModelRuleRequest<'a> { @@ -62,6 +88,7 @@ struct NativeModelEvaluation { fn evaluate_native_models( input: NativeModelRuleRequest<'_>, + control: &EvaluationControl<'_>, ) -> Result { let NativeModelRuleRequest { context, @@ -146,7 +173,13 @@ fn evaluate_native_models( bucket .entries .retain(|path, _| current.contains(path.as_str()) || project_dir.join(path).is_file()); - cache.put_native_rules_bucket(model_rules_fingerprint, &bucket)?; + match control.deferred_bucket { + None => cache.put_native_rules_bucket(model_rules_fingerprint, &bucket)?, + Some(deferred) => { + *deferred.borrow_mut() = + Some(cache.pending_native_rules_bucket(model_rules_fingerprint, bucket)); + } + } } Ok(result) } @@ -158,14 +191,21 @@ fn evaluate_pending_models( let evaluate_pending = |pending: PendingModelRule<'_>| CompletedModelRule { index: pending.index, identity: pending.identity, - faults: rules::evaluate_model(ModelEvaluationRequest { - model: pending.model, - config: &context.request.config, - selected: context.selected, - request: context.request, - threshold_overrides: context.threshold_overrides, - }) - .and_then(|faults| scoped_faults(context.selected, faults, RuleScope::evaluates_models)), + faults: if context + .cancel + .is_some_and(|cancel| cancel.load(Ordering::Acquire)) + { + Err(CANCELLED.to_owned()) + } else { + rules::evaluate_model(ModelEvaluationRequest { + model: pending.model, + config: &context.request.config, + selected: context.selected, + request: context.request, + threshold_overrides: context.threshold_overrides, + }) + .and_then(|faults| scoped_faults(context.selected, faults, RuleScope::evaluates_models)) + }, }; if pending_models.len() <= 1 { return Ok(pending_models.into_iter().map(evaluate_pending).collect()); @@ -272,7 +312,16 @@ fn model_rules_fingerprint( fingerprint::fingerprint(selected, &model_config, dialect) } +#[cfg(test)] pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result { + evaluate_request_with(request, &EvaluationControl::immediate()) +} + +/// Evaluate `request`, publishing its cache bucket and stopping early as `control` says. +pub(crate) fn evaluate_request_with( + request: EvaluateRequest, + control: &EvaluationControl<'_>, +) -> Result { if request.version != API_VERSION { return Err(format!( "unsupported rules native API version {}; expected {API_VERSION}", @@ -335,15 +384,23 @@ pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result = control.cancel; + let native = evaluate_native_models( + NativeModelRuleRequest { + context: ModelRuleBatchContext { + request: &request, + selected: &selected_by_code, + threshold_overrides: &threshold_overrides, + cancel, + }, + cache: cache.as_ref(), + model_rules_fingerprint: &model_rules_fingerprint, }, - cache: cache.as_ref(), - model_rules_fingerprint: &model_rules_fingerprint, - })?; + control, + )?; let cache_hits = native.hits; let cache_misses = native.misses; raw_faults.extend(native.faults); @@ -382,13 +439,13 @@ fn model_cache_identities( /// Key one model's findings by payload, model-rules fingerprint, and derived test counts. fn model_cache_identity(model: &crate::models::Model, model_rules: &str) -> Result { - let mut digest = Sha256::new(); + let mut digest = blake3::Hasher::new(); digest.update(MODEL_RULES_IDENTITY_SEED); - digest.update([0]); + digest.update(&[0]); digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); - digest.update([0]); + digest.update(&[0]); digest.update(model_rules.as_bytes()); - digest.update([0]); + digest.update(&[0]); match &model.payload_digest { Some(payload) => { digest.update(b"payload:"); @@ -396,12 +453,12 @@ fn model_cache_identity(model: &crate::models::Model, model_rules: &str) -> Resu } None => { digest.update(b"model:"); - digest.update(serde_json::to_vec(model).map_err(|error| error.to_string())?); + serde_json::to_writer(&mut digest, model).map_err(|error| error.to_string())?; } } - digest.update([0]); - digest.update(model.empty_input_only_test_count.to_le_bytes()); - Ok(format!("{:x}", digest.finalize())) + digest.update(&[0]); + digest.update(&model.empty_input_only_test_count.to_le_bytes()); + Ok(digest.finalize().to_hex().to_string()) } fn validate_suppression_codes( diff --git a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs index 52219e9666..b49f3ff516 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/memo.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/memo.rs @@ -1,34 +1,76 @@ //! Evaluate a natively built rules request, reusing the last identical request's response. +use std::cell::RefCell; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; use serde_json::Value; -use sha2::{Digest, Sha256}; use crate::constants::{API_VERSION, NATIVE_ROWS_MEMO_VERSION}; -use crate::engine::_helpers::evaluation::evaluate_request; +use crate::engine::_helpers::cache::PendingBucketWrite; +use crate::engine::_helpers::evaluation::{EvaluationControl, evaluate_request_with}; use crate::errors::RowsError; use crate::models::{EvaluateRequest, Model, RowsEvaluation, RowsRequest}; +/// A rows request whose models are digested, ready to evaluate once. +#[derive(Debug)] +pub(crate) struct PreparedRows { + request: EvaluateRequest, + /// The request's memo identity and memo path, when the rules cache is enabled. + memo: Option<(String, PathBuf)>, + /// The digest of every evaluation input the request carries, when it was asked for. + pub(crate) key: Option, +} + +/// What a speculative evaluation found, with every write it would have made still pending. +#[derive(Debug)] +pub(crate) enum SpeculativeOutcome { + /// The memo answered the request, so the evaluation would have written nothing. + Reused(String), + /// The request was evaluated; its bucket write precedes its memo write. + Evaluated { + bucket: Option, + response: Result, + memo: Option<(String, PathBuf)>, + }, +} + pub(crate) fn evaluate_rows(rows: RowsRequest) -> Result { + evaluate_prepared(prepare(rows, false)?) +} + +/// Digest `rows`' models and derive its memo identity and, when asked, its speculation key. +pub(crate) fn prepare(rows: RowsRequest, keyed: bool) -> Result { let RowsRequest { mut request, memo } = rows; let models: Vec = std::mem::take(&mut request.models) .into_iter() .map(digested) .collect::>()?; - let identity: Option<(String, PathBuf)> = match memo { + let memo: Option<(String, PathBuf)> = match memo { Some((code_identity, path)) => Some((identity(&request, &models, &code_identity)?, path)), None => None, }; - let reused: Option = identity + let key: Option = if keyed { + Some(speculation_key(&request, &models, memo.as_ref())?) + } else { + None + }; + request.models = models; + Ok(PreparedRows { request, memo, key }) +} + +/// Answer a prepared request from the memo or evaluate it, writing the cache and memo at once. +pub(crate) fn evaluate_prepared(prepared: PreparedRows) -> Result { + let PreparedRows { request, memo, .. } = prepared; + let reused: Option = memo .as_ref() .and_then(|(identity, path)| read_response(path, identity)); let (response, was_reused): (String, bool) = match reused { Some(response) => (response, true), None => { - request.models = models; - let response: String = evaluate_request(request).map_err(RowsError::Rules)?; - if let Some((identity, path)) = &identity { + let response: String = evaluate_request_with(request, &EvaluationControl::immediate()) + .map_err(RowsError::Rules)?; + if let Some((identity, path)) = &memo { write_response(path, identity, &response).map_err(RowsError::Io)?; } (response, false) @@ -37,10 +79,92 @@ pub(crate) fn evaluate_rows(rows: RowsRequest) -> Result SpeculativeOutcome { + let PreparedRows { request, memo, .. } = prepared; + if let Some(response) = memo + .as_ref() + .and_then(|(identity, path)| read_response(path, identity)) + { + return SpeculativeOutcome::Reused(response); + } + let bucket: RefCell> = RefCell::new(None); + let response: Result = evaluate_request_with( + request, + &EvaluationControl { + deferred_bucket: Some(&bucket), + cancel: Some(cancel), + }, + ); + let bucket: Option = bucket.into_inner(); + SpeculativeOutcome::Evaluated { + bucket, + response, + memo, + } +} + +/// Make a speculative evaluation's pending writes in order and answer as `evaluate_prepared` would. +impl SpeculativeOutcome { + /// Whether the speculation produced a response, so a failure is retried by the real request. + pub(crate) fn evaluated_cleanly(&self) -> bool { + !matches!( + self, + Self::Evaluated { + response: Err(_), + .. + } + ) + } +} + +pub(crate) fn commit(speculative: SpeculativeOutcome) -> Result { + match speculative { + SpeculativeOutcome::Reused(response) => decode(&response, true), + SpeculativeOutcome::Evaluated { + bucket, + response, + memo, + } => { + if let Some(bucket) = &bucket { + bucket.publish().map_err(RowsError::Rules)?; + } + let response: String = response.map_err(RowsError::Rules)?; + if let Some((identity, path)) = &memo { + write_response(path, identity, &response).map_err(RowsError::Io)?; + } + decode(&response, false) + } + } +} + +/// Every evaluation input a request carries: its memo identity inputs and memo location. +fn speculation_key( + request: &EvaluateRequest, + models: &[Model], + memo: Option<&(String, PathBuf)>, +) -> Result { + let mut hasher: blake3::Hasher = blake3::Hasher::new(); + match memo { + Some((identity, path)) => { + hasher.update(b"memo\0"); + hasher.update(identity.as_bytes()); + hasher.update(b"\0"); + hasher.update(path.as_os_str().as_encoded_bytes()); + } + None => { + hasher.update(b"unmemoized\0"); + hasher.update(identity(request, models, "")?.as_bytes()); + } + } + Ok(hasher.finalize().to_hex().to_string()) +} + +/// `model` with the BLAKE3 digest of its exact facts, which keys its cached findings. fn digested(mut model: Model) -> Result { - let bytes: Vec = serde_json::to_vec(&model).map_err(rules_error)?; - model.payload_digest = Some(format!("{:x}", Sha256::digest(&bytes))); + let mut hasher: blake3::Hasher = blake3::Hasher::new(); + serde_json::to_writer(&mut hasher, &model).map_err(rules_error)?; + model.payload_digest = Some(hasher.finalize().to_hex().to_string()); Ok(model) } @@ -50,14 +174,14 @@ fn identity( models: &[Model], code_identity: &str, ) -> Result { - let mut hasher: Sha256 = Sha256::new(); + let mut hasher: blake3::Hasher = blake3::Hasher::new(); for part in [NATIVE_ROWS_MEMO_VERSION, code_identity] { hasher.update(part.as_bytes()); hasher.update(b"\0"); } - hasher.update(serde_json::to_vec(request).map_err(rules_error)?); + serde_json::to_writer(&mut hasher, request).map_err(rules_error)?; hasher.update(b"\0"); - hasher.update((models.len() as u64).to_le_bytes()); + hasher.update(&(models.len() as u64).to_le_bytes()); for model in models { hasher.update( model @@ -68,7 +192,7 @@ fn identity( ); hasher.update(b"\0"); } - Ok(format!("{:x}", hasher.finalize())) + Ok(hasher.finalize().to_hex().to_string()) } /// The stored response for exactly `identity`; an unreadable or foreign memo answers nothing. diff --git a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs index b82023ac5c..ec8f8aa27d 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/mod.rs @@ -4,3 +4,4 @@ pub mod evaluation; pub(crate) mod memo; pub(crate) mod project_rows; pub(crate) mod rows; +pub(crate) mod speculation; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index ece8718b45..9fedfc925e 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -18,11 +18,11 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result Ok(ModelRow { name: facts.name.clone(), relative_path: facts.relative_path.clone(), - query_sql: facts.query_sql.clone(), + query_sql: facts.query_sql.to_string(), authored_sql: if facts.authored_query_sql.is_empty() { - facts.authored_sql.clone() + facts.authored_sql.to_string() } else { - facts.authored_query_sql.clone() + facts.authored_query_sql.to_string() }, config: facts.config.values.clone(), authored_config_keys: facts.config.header_keys.clone(), diff --git a/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs b/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs new file mode 100644 index 0000000000..4df1f3b38b --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/_helpers/speculation.rs @@ -0,0 +1,137 @@ +//! Evaluate a built rules request speculatively and answer the real request from it when identical. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender, channel}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; + +use crate::engine::_helpers::memo::{ + PreparedRows, SpeculativeOutcome, commit, evaluate_prepared, prepare, speculate, +}; +use crate::models::{RowsRequest, SpeculativeAnswer, SpeculativeRows}; + +const SPECULATION_THREAD: &str = "sqlbuild-early-rules"; +/// How often a wait for the speculative thread checks its stop flag. +const WAIT_POLL: Duration = Duration::from_millis(5); + +/// What the speculative thread finished with: its held outcome, or `None` when preparing failed. +type Speculated = Result, String>; + +pub(crate) fn start(rows: RowsRequest) -> std::io::Result { + let cancel: Arc = Arc::new(AtomicBool::new(false)); + let (sender, key): (Sender>, Receiver>) = channel(); + let stop: Arc = Arc::clone(&cancel); + let thread = std::thread::Builder::new() + .name(SPECULATION_THREAD.to_owned()) + .spawn(move || -> Speculated { + catch_compiler_panic(|| { + let Ok(prepared) = prepare(rows, true) else { + let _ = sender.send(None); + return Ok(None); + }; + let _ = sender.send(prepared.key.clone()); + Ok(Some(speculate(prepared, &stop))) + }) + })?; + Ok(SpeculativeRows { + key: Mutex::new(key), + thread: Some(thread), + cancel, + }) +} + +pub(crate) fn answer( + mut speculative: SpeculativeRows, + rows: RowsRequest, + stop: &AtomicBool, +) -> SpeculativeAnswer { + let prepared: PreparedRows = match prepare(rows, true) { + Ok(prepared) => prepared, + Err(error) => { + drop(speculative); + return SpeculativeAnswer::Answered { + evaluation: Err(error), + used: false, + }; + } + }; + let Some(speculated) = speculated_key(&speculative, stop) else { + drop(speculative); + return SpeculativeAnswer::Stopped; + }; + let outcome: Option = + if prepared.key.is_some() && speculated == prepared.key { + if !finished(&speculative, stop) { + drop(speculative); + return SpeculativeAnswer::Stopped; + } + match speculative.thread.take().map(std::thread::JoinHandle::join) { + Some(Ok(Ok(outcome))) => outcome.filter(SpeculativeOutcome::evaluated_cleanly), + _ => None, + } + } else { + drop(speculative); + None + }; + match outcome { + Some(outcome) => SpeculativeAnswer::Answered { + evaluation: commit(outcome), + used: true, + }, + None => SpeculativeAnswer::Answered { + evaluation: evaluate_prepared(prepared), + used: false, + }, + } +} + +/// The speculation's key once known (`Some(None)` when it has none), or `None` once `stop` is set. +fn speculated_key(speculative: &SpeculativeRows, stop: &AtomicBool) -> Option> { + let receiver = speculative + .key + .lock() + .unwrap_or_else(PoisonError::into_inner); + loop { + if stop.load(Ordering::Acquire) { + return None; + } + match receiver.recv_timeout(WAIT_POLL) { + Ok(key) => return Some(key), + Err(RecvTimeoutError::Disconnected) => return Some(None), + Err(RecvTimeoutError::Timeout) => {} + } + } +} + +/// Whether the speculative thread finished before `stop` was set. +fn finished(speculative: &SpeculativeRows, stop: &AtomicBool) -> bool { + loop { + if stop.load(Ordering::Acquire) { + return false; + } + if speculative + .thread + .as_ref() + .is_none_or(std::thread::JoinHandle::is_finished) + { + return true; + } + std::thread::sleep(WAIT_POLL); + } +} + +pub(crate) fn discard(speculative: SpeculativeRows) { + drop(speculative); +} + +impl Drop for SpeculativeRows { + /// Cancel the speculation and wait for its thread. + fn drop(&mut self) { + self.cancel.store(true, Ordering::Release); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} diff --git a/crates/sqlbuild-rules/src/engine/main/answer_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/main/answer_speculative_rows.rs new file mode 100644 index 0000000000..35a0a87af9 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/answer_speculative_rows.rs @@ -0,0 +1,12 @@ +use std::sync::atomic::AtomicBool; + +use crate::models::{RowsRequest, SpeculativeAnswer, SpeculativeRows}; + +/// Evaluate `request` as `evaluate_rows` would, from `speculative` only when its key matches. +pub fn answer_speculative_rows( + speculative: SpeculativeRows, + request: RowsRequest, + stop: &AtomicBool, +) -> SpeculativeAnswer { + crate::engine::_helpers::speculation::answer(speculative, request, stop) +} diff --git a/crates/sqlbuild-rules/src/engine/main/discard_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/main/discard_speculative_rows.rs new file mode 100644 index 0000000000..7b2f5ce3aa --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/discard_speculative_rows.rs @@ -0,0 +1,6 @@ +use crate::models::SpeculativeRows; + +/// Cancel a speculative evaluation and wait for its thread; nothing it held is written. +pub fn discard_speculative_rows(speculative: SpeculativeRows) { + crate::engine::_helpers::speculation::discard(speculative); +} diff --git a/crates/sqlbuild-rules/src/engine/main/mod.rs b/crates/sqlbuild-rules/src/engine/main/mod.rs index 8cae1cda6c..a20ecdbdd4 100644 --- a/crates/sqlbuild-rules/src/engine/main/mod.rs +++ b/crates/sqlbuild-rules/src/engine/main/mod.rs @@ -1,5 +1,8 @@ +pub mod answer_speculative_rows; pub mod build_models; pub mod custom_host; +pub mod discard_speculative_rows; pub mod evaluate_rows; pub mod finalize_rows; pub mod project_model_rows; +pub mod start_speculative_rows; diff --git a/crates/sqlbuild-rules/src/engine/main/start_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/main/start_speculative_rows.rs new file mode 100644 index 0000000000..ca090e0aee --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/main/start_speculative_rows.rs @@ -0,0 +1,6 @@ +use crate::models::{RowsRequest, SpeculativeRows}; + +/// Evaluate a built request on its own thread, holding every cache and memo write it would make. +pub fn start_speculative_rows(request: RowsRequest) -> std::io::Result { + crate::engine::_helpers::speculation::start(request) +} diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index 1dd70e7f7c..3190903ee8 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -10,6 +10,8 @@ mod helpers; mod incremental_rules; #[path = "tests/test_scope_facts.rs"] mod scope_facts; +#[path = "tests/test_speculative_rows.rs"] +mod speculative_rows; #[path = "tests/test_sql_scanning.rs"] mod sql_scanning; #[path = "tests/test_sql_test_rules.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index dd22e28f03..1300b5ab36 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -1,7 +1,17 @@ +use std::path::{Path, PathBuf}; + use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use tempfile::TempDir; +use std::sync::atomic::AtomicBool; + +use crate::engine::main::answer_speculative_rows::answer_speculative_rows; +use crate::engine::main::discard_speculative_rows::discard_speculative_rows; +use crate::models::{ + EvaluateRequest, Fault, RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows, +}; + use crate::engine::_helpers::evaluation::evaluate_json; use crate::engine::_helpers::evaluation::evaluate_request as evaluate_parsed; use crate::engine::_helpers::evaluation::parse_parts; @@ -480,3 +490,101 @@ pub(crate) fn cached_paths(project_dir: &TempDir) -> Result, String> .map(|entries| entries.keys().cloned().collect()) .unwrap_or_default()) } + +const SPECULATIVE_CODE_IDENTITY: &str = "speculative-rows-test-build"; +pub(crate) const SPECULATIVE_CACHE_FILE: &str = "target/rules-cache/bulk/native.json"; +const SPECULATIVE_MEMO_FILE: &str = "target/rules-cache/bulk/native-response.json"; + +pub(crate) fn speculative_rows_request( + project_dir: &Path, + cache_enabled: bool, + models: &[Value], +) -> Result { + let mut request: Value = speculative_request_json(project_dir, cache_enabled); + request["models"] = Value::Array(models.to_vec()); + let request: EvaluateRequest = + serde_json::from_value(request).map_err(|error| error.to_string())?; + Ok(RowsRequest { + request, + memo: cache_enabled.then(|| { + ( + SPECULATIVE_CODE_IDENTITY.to_owned(), + project_dir.join(SPECULATIVE_MEMO_FILE), + ) + }), + }) +} + +fn speculative_request_json(project_dir: &Path, cache_enabled: bool) -> Value { + json!({ + "version": 1, + "project_dir": project_dir, + "dialect": "duckdb", + "config": {"select": ["SQBR"], "domains": ["commerce"], "cache": {"enabled": cache_enabled}}, + "models": [], + "defer_suppressions": true + }) +} + +pub(crate) fn changed_orders_models(query: Option<&str>) -> Vec { + let mut models: Vec = orders_models(); + let query: Value = query.map_or_else(|| models[0]["query_sql"].clone(), |query| json!(query)); + models[0]["query_sql"] = query.clone(); + models[0]["authored_sql"] = query; + models +} + +pub(crate) fn evaluation_summary( + evaluation: &RowsEvaluation, +) -> (Vec, Vec, u64, u64, u64, bool) { + ( + evaluation.faults.clone(), + evaluation.selected_codes.clone(), + evaluation.evaluated_models, + evaluation.cache_hits, + evaluation.cache_misses, + evaluation.reused, + ) +} + +pub(crate) fn written_rules_files(project_dir: &Path) -> Vec { + [SPECULATIVE_CACHE_FILE, SPECULATIVE_MEMO_FILE] + .iter() + .map(|file| project_dir.join(file)) + .filter(|path| path.is_file()) + .collect() +} + +/// Discard a speculation after it ran a while; it never reports stopping. +pub(crate) fn discard_after_running( + speculative: SpeculativeRows, + _project_dir: &Path, +) -> Result { + std::thread::sleep(std::time::Duration::from_millis(200)); + discard_speculative_rows(speculative); + Ok(false) +} + +/// Answer beside a speculation once the rules run was cancelled; whether the answer stopped. +pub(crate) fn answer_after_stop( + speculative: SpeculativeRows, + project_dir: &Path, +) -> Result { + Ok(matches!( + answer_speculative_rows( + speculative, + speculative_rows_request(project_dir, true, &orders_models())?, + &AtomicBool::new(true), + ), + SpeculativeAnswer::Stopped + )) +} + +/// Drop a speculation at once; it never reports stopping. +pub(crate) fn drop_at_once( + speculative: SpeculativeRows, + _project_dir: &Path, +) -> Result { + drop(speculative); + Ok(false) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs index 959acb40da..7e0025814d 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs @@ -7,6 +7,106 @@ use crate::engine::tests::helpers; use crate::engine::tests::test_types; use crate::rules::_helpers::evaluation::normalize_rules_sql; +#[test] +fn given_dependency_tokens_when_classifying_imports_then_preserves_dialect_and_original_sql() +-> Result<(), String> { + let test_cases = [ + test_types::DependencyTokenTestCase { + description: "comments and escaped literals retain canonical arguments", + expected_columns: &[1], + dialect: "duckdb", + query_sql: "WITH imported AS (SELECT * FROM __ReF /* __ref('ignored') */ ('Order''s')) SELECT '__source(''ignored'')' AS label FROM imported", + expected_messages: &[ + "each __ref/__source must appear once in its own top-level import CTE", + ], + }, + test_types::DependencyTokenTestCase { + description: "quoted function names are not dependency calls", + expected_columns: &[], + dialect: "duckdb", + query_sql: "SELECT * FROM \"__ref\"('orders')", + expected_messages: &[], + }, + test_types::DependencyTokenTestCase { + description: "backtick quoting and comment-separated source arguments", + expected_columns: &[], + dialect: "bigquery", + query_sql: "WITH imported AS (SELECT * FROM __source /* ignored */ (`Orders`)) SELECT order_id FROM imported", + expected_messages: &[], + }, + test_types::DependencyTokenTestCase { + description: "dollar strings do not contribute spurious calls", + expected_columns: &[], + dialect: "postgres", + query_sql: "WITH imported AS (SELECT * FROM __ref('orders')) SELECT $$__ref('ignored')$$ AS label FROM imported", + expected_messages: &[], + }, + test_types::DependencyTokenTestCase { + description: "generic fallback retains original dialect dependencies", + expected_columns: &[], + dialect: "postgres", + query_sql: "WITH imported AS (SELECT * FROM __ref('orders')) SELECT payload:order_id FROM imported", + expected_messages: &[], + }, + test_types::DependencyTokenTestCase { + description: "normalization must not erase dependency calls", + expected_columns: &[1], + dialect: "snowflake", + query_sql: "SELECT * EXCLUDE (__ref('orders')) FROM orders", + expected_messages: &[ + "each __ref/__source must appear once in its own top-level import CTE", + ], + }, + test_types::DependencyTokenTestCase { + description: "duplicate dependencies remain ordered findings", + expected_columns: &[1, 55], + dialect: "duckdb", + query_sql: "WITH first_import AS (SELECT * FROM __ref('orders')), second_import AS (SELECT * FROM __REF('orders') WHERE order_id > 0) SELECT order_id FROM first_import", + expected_messages: &[ + "each __ref/__source must appear once in its own top-level import CTE", + "import CTE \"second_import\" contains transformation logic", + ], + }, + ]; + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let config = json!({"select": ["SQBRMODEL101"], "cache": {"enabled": false}}); + for test_case in test_cases { + let mut request: Value = serde_json::from_str(&helpers::request(&project_dir, &config)) + .map_err(|error| error.to_string())?; + request["dialect"] = json!(test_case.dialect); + request["models"][0]["query_sql"] = json!(test_case.query_sql); + // Authored SQL can differ after expansion: dependency classification has + // always used query_sql, while literal decisions use authored_sql. + request["models"][0]["authored_sql"] = json!("SELECT @orders()"); + let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + let faults = result["faults"].as_array().ok_or("missing faults")?; + let messages: Vec<&str> = faults + .iter() + .filter_map(|fault| fault["message"].as_str()) + .collect(); + assert_eq!( + messages, test_case.expected_messages, + "{}: {result}", + test_case.description + ); + assert!( + faults.iter().all(|fault| fault["unevaluated"] != true), + "{}: {result}", + test_case.description + ); + for (fault, expected_column) in faults.iter().zip(test_case.expected_columns) { + assert_eq!(fault["line"], 1, "{}", test_case.description); + assert_eq!( + fault["column"], *expected_column, + "{}", + test_case.description + ); + } + } + Ok(()) +} + #[test] fn given_conflicting_model_identity_when_evaluating_layer_rules_then_reports_independent_findings() -> Result<(), String> { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs b/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs new file mode 100644 index 0000000000..f8c2c47933 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_speculative_rows.rs @@ -0,0 +1,176 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; + +use serde_json::Value; +use tempfile::TempDir; + +use crate::engine::_helpers::memo::SpeculativeOutcome; +use crate::engine::main::answer_speculative_rows::answer_speculative_rows; +use crate::engine::main::evaluate_rows::evaluate_rows; +use crate::engine::main::start_speculative_rows::start_speculative_rows; +use crate::engine::tests::helpers::{ + SPECULATIVE_CACHE_FILE, answer_after_stop, changed_orders_models, discard_after_running, + drop_at_once, evaluation_summary, orders_models, speculative_rows_request, written_rules_files, +}; +use crate::engine::tests::test_types::{ + SpeculativeOutcomeTestCase, SpeculativeRowsTestCase, SpeculativeStopTestCase, +}; +use crate::models::SpeculativeAnswer; + +#[test] +fn given_speculative_rows_when_answering_then_result_and_writes_match_ordinary_evaluation() +-> Result<(), String> { + let test_cases = [ + SpeculativeRowsTestCase { + description: "identical cached request uses the speculation", + cache_enabled: true, + changed_query: None, + expected_used: true, + }, + SpeculativeRowsTestCase { + description: "identical uncached request uses the speculation", + cache_enabled: false, + changed_query: None, + expected_used: true, + }, + SpeculativeRowsTestCase { + description: "a model changed after the speculation started is evaluated again", + cache_enabled: true, + changed_query: Some("SELECT order_id FROM warehouse.raw.orders"), + expected_used: false, + }, + ]; + for test_case in &test_cases { + let speculated_dir = TempDir::new().map_err(|error| error.to_string())?; + let ordinary_dir = TempDir::new().map_err(|error| error.to_string())?; + let project_dir: &Path = speculated_dir.path(); + let actual_models: Vec = changed_orders_models(test_case.changed_query); + + let speculative = start_speculative_rows(speculative_rows_request( + project_dir, + test_case.cache_enabled, + &orders_models(), + )?) + .map_err(|error| error.to_string())?; + let answered = answer_speculative_rows( + speculative, + speculative_rows_request(project_dir, test_case.cache_enabled, &actual_models)?, + &AtomicBool::new(false), + ); + let expected = evaluate_rows(speculative_rows_request( + ordinary_dir.path(), + test_case.cache_enabled, + &actual_models, + )?) + .map_err(|_| "ordinary evaluation failed".to_owned())?; + + let SpeculativeAnswer::Answered { evaluation, used } = answered else { + return Err(format!("{}: answer stopped", test_case.description)); + }; + let evaluation = evaluation.map_err(|_| "speculative answer failed".to_owned())?; + let cache = |dir: &Path| std::fs::read(dir.join(SPECULATIVE_CACHE_FILE)).ok(); + assert_eq!(used, test_case.expected_used, "{}", test_case.description); + assert_eq!( + evaluation_summary(&evaluation), + evaluation_summary(&expected), + "{}", + test_case.description + ); + assert_eq!( + cache(project_dir), + cache(ordinary_dir.path()), + "{}", + test_case.description + ); + assert_eq!( + written_rules_files(project_dir).len(), + written_rules_files(ordinary_dir.path()).len(), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_speculative_rows_when_discarded_or_stopped_then_nothing_is_written() -> Result<(), String> +{ + let test_cases = [ + SpeculativeStopTestCase { + description: "discarding after it ran a while", + end: discard_after_running, + expected_stopped: false, + }, + SpeculativeStopTestCase { + description: "answering after the rules run was cancelled", + end: answer_after_stop, + expected_stopped: true, + }, + SpeculativeStopTestCase { + description: "dropping it at once", + end: drop_at_once, + expected_stopped: false, + }, + ]; + for test_case in &test_cases { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let speculative = start_speculative_rows(speculative_rows_request( + project_dir.path(), + true, + &orders_models(), + )?) + .map_err(|error| error.to_string())?; + + let stopped: bool = (test_case.end)(speculative, project_dir.path())?; + + assert_eq!( + stopped, test_case.expected_stopped, + "{}", + test_case.description + ); + assert_eq!( + written_rules_files(project_dir.path()), + Vec::::new(), + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_speculative_outcome_when_checked_then_only_failed_evaluations_are_retried() { + let test_cases = [ + SpeculativeOutcomeTestCase { + description: "memo reuse is kept", + outcome: || SpeculativeOutcome::Reused("{}".to_owned()), + expected_clean: true, + }, + SpeculativeOutcomeTestCase { + description: "evaluated response is kept", + outcome: || SpeculativeOutcome::Evaluated { + bucket: None, + response: Ok("{}".to_owned()), + memo: None, + }, + expected_clean: true, + }, + SpeculativeOutcomeTestCase { + description: "failed evaluation is retried by the real request", + outcome: || SpeculativeOutcome::Evaluated { + bucket: None, + response: Err("worker pool unavailable".to_owned()), + memo: None, + }, + expected_clean: false, + }, + ]; + for test_case in test_cases { + assert_eq!( + (test_case.outcome)().evaluated_cleanly(), + test_case.expected_clean, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index c3476b1906..6d69c0b50b 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -252,3 +252,29 @@ pub(crate) struct CachePruningTestCase { pub(crate) description: &'static str, pub(crate) expected_paths: &'static [&'static str], } +pub(crate) struct DependencyTokenTestCase { + pub description: &'static str, + pub dialect: &'static str, + pub query_sql: &'static str, + pub expected_messages: &'static [&'static str], + pub expected_columns: &'static [usize], +} + +pub(crate) struct SpeculativeRowsTestCase { + pub(crate) description: &'static str, + pub(crate) cache_enabled: bool, + pub(crate) changed_query: Option<&'static str>, + pub(crate) expected_used: bool, +} + +pub(crate) struct SpeculativeStopTestCase { + pub(crate) description: &'static str, + pub(crate) end: fn(crate::models::SpeculativeRows, &std::path::Path) -> Result, + pub(crate) expected_stopped: bool, +} + +pub(crate) struct SpeculativeOutcomeTestCase { + pub(crate) description: &'static str, + pub(crate) outcome: fn() -> crate::engine::_helpers::memo::SpeculativeOutcome, + pub(crate) expected_clean: bool, +} diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 5b57a1a7d4..c98860e8ff 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -1,4 +1,6 @@ use serde::{Deserialize, Serialize}; + +use crate::errors::RowsError; use serde_json::Value; use std::collections::BTreeMap; @@ -926,6 +928,30 @@ pub struct RowsRequest { pub memo: Option<(String, std::path::PathBuf)>, } +/// A built-in rules evaluation running speculatively on its own thread; dropping it cancels it. +#[derive(Debug)] +pub struct SpeculativeRows { + pub(crate) key: std::sync::Mutex>>, + pub(crate) thread: Option< + std::thread::JoinHandle< + Result, String>, + >, + >, + pub(crate) cancel: std::sync::Arc, +} + +/// How a request was answered beside a speculative evaluation. +#[derive(Debug)] +pub enum SpeculativeAnswer { + /// The request's evaluation, and whether the speculation evaluated exactly this request. + Answered { + evaluation: Result, + used: bool, + }, + /// `stop` was set while waiting; nothing was evaluated or written. + Stopped, +} + /// A rows request's decoded response. #[derive(Debug, Deserialize)] pub struct RowsEvaluation { diff --git a/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs index 50033d3c68..78503eda68 100644 --- a/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/digest.rs @@ -1,19 +1,17 @@ -//! SHA-256 over length-prefixed fields, so no two field lists share a key. - -use sha2::{Digest, Sha256}; +//! BLAKE3 over length-prefixed fields, so no two field lists share a key. use crate::constants::NATIVE_BUILD_IDENTITY; use crate::rule_cache::constants::CACHE_VERSION; /// The hex digest of the cache version, the native build and each field in order. pub(crate) fn field_digest(kind: &str, fields: &[&str]) -> String { - let mut digest = Sha256::new(); + let mut digest = blake3::Hasher::new(); for field in [CACHE_VERSION, NATIVE_BUILD_IDENTITY, kind] .iter() .chain(fields.iter()) { - digest.update((field.len() as u64).to_le_bytes()); + digest.update(&(field.len() as u64).to_le_bytes()); digest.update(field.as_bytes()); } - format!("{:x}", digest.finalize()) + digest.finalize().to_hex().to_string() } diff --git a/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs index 71d61e1a07..0898810f52 100644 --- a/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod digest; +pub(crate) mod model_rule_key; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs b/crates/sqlbuild-rules/src/rule_cache/_helpers/model_rule_key.rs similarity index 86% rename from crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs rename to crates/sqlbuild-rules/src/rule_cache/_helpers/model_rule_key.rs index 99df4d0c9e..ccec4a557f 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_key.rs +++ b/crates/sqlbuild-rules/src/rule_cache/_helpers/model_rule_key.rs @@ -6,7 +6,11 @@ use crate::rule_cache::models::ModelKeyInput; /// A digest of the lint key, selected codes, authored and compiled SQL and dynamic contract. #[must_use] -pub fn model_rule_key(lint_key: &str, codes: &[String], model: &ModelKeyInput<'_>) -> String { +pub(crate) fn model_rule_key( + lint_key: &str, + codes: &[String], + model: &ModelKeyInput<'_>, +) -> String { let codes: String = codes.join("\0"); let contract: &str = model.contract.unwrap_or(NO_CONTRACT); let has_contract: &str = if model.contract.is_some() { "1" } else { "0" }; diff --git a/crates/sqlbuild-rules/src/rule_cache/constants.rs b/crates/sqlbuild-rules/src/rule_cache/constants.rs index 4b5361cf03..8e6fff224f 100644 --- a/crates/sqlbuild-rules/src/rule_cache/constants.rs +++ b/crates/sqlbuild-rules/src/rule_cache/constants.rs @@ -1,7 +1,7 @@ //! The SQL rule result cache format and the inputs its keys separate. /// Bumped whenever key derivation or the stored finding shape changes. -pub(crate) const CACHE_VERSION: &str = "sql-rules-v6"; +pub(crate) const CACHE_VERSION: &str = "sql-rules-v7"; pub(crate) const MODEL_KEY_KIND: &str = "model"; pub(crate) const FILE_KEY_KIND: &str = "file"; pub(crate) const LINT_KEY_KIND: &str = "lint"; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs index 20a4012180..3a70ebde30 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/main/mod.rs @@ -1,7 +1,7 @@ pub mod file_rule_key; pub mod lint_key; pub mod lookup_rule_cache; -pub mod model_rule_key; +pub mod model_rule_keys; pub mod parse_rule_cache; pub mod rule_cache_text; pub mod rules_bulk_cache_path; diff --git a/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs new file mode 100644 index 0000000000..91093063ad --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/main/model_rule_keys.rs @@ -0,0 +1,19 @@ +//! `model_rule_keys`: the SQL rule cache keys of many models, hashed in parallel. + +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +use crate::rule_cache::_helpers::model_rule_key::model_rule_key; +use crate::rule_cache::models::ModelKeyInput; + +/// Each model's `model_rule_key`, in input order. +#[must_use] +pub fn model_rule_keys( + lint_key: &str, + codes: &[String], + models: &[ModelKeyInput<'_>], +) -> Vec { + models + .par_iter() + .map(|model| model_rule_key(lint_key, codes, model)) + .collect() +} diff --git a/crates/sqlbuild-rules/src/rule_cache/mod.rs b/crates/sqlbuild-rules/src/rule_cache/mod.rs index 76c842f679..1644773e62 100644 --- a/crates/sqlbuild-rules/src/rule_cache/mod.rs +++ b/crates/sqlbuild-rules/src/rule_cache/mod.rs @@ -3,3 +3,6 @@ pub(crate) mod constants; pub mod main; pub mod models; pub mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/rule_cache/tests.rs b/crates/sqlbuild-rules/src/rule_cache/tests.rs new file mode 100644 index 0000000000..18a0eb0074 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_rule_cache.rs"] +mod rule_cache; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs b/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs new file mode 100644 index 0000000000..4d2c72069a --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/tests/test_rule_cache.rs @@ -0,0 +1,55 @@ +use crate::rule_cache::_helpers::model_rule_key::model_rule_key; +use crate::rule_cache::main::model_rule_keys::model_rule_keys; +use crate::rule_cache::models::ModelKeyInput; +use crate::rule_cache::tests::test_types::ModelRuleKeysTestCase; + +#[test] +fn given_models_when_hashing_rule_keys_together_then_each_key_matches_its_single_model_key_in_order() + { + let test_cases = [ + ModelRuleKeysTestCase { + description: "no models give no keys", + models: vec![], + expected_key_count: 0, + }, + ModelRuleKeysTestCase { + description: "many models keep their order and single-model keys", + models: (0..64) + .map(|index| { + let sql: &'static str = + Box::leak(format!("select {index} as id").into_boxed_str()); + let contract: Option<&'static str> = (index % 3 == 0).then_some("id:integer"); + (sql, sql, contract) + }) + .collect(), + expected_key_count: 64, + }, + ]; + let codes: Vec = vec!["SQBRSQL001".to_owned(), "SQBRSQL033".to_owned()]; + + for test_case in test_cases { + let inputs: Vec> = test_case + .models + .iter() + .map(|(authored_sql, query_sql, contract)| ModelKeyInput { + authored_sql, + query_sql, + contract: *contract, + }) + .collect(); + let expected: Vec = inputs + .iter() + .map(|model| model_rule_key("lint-key", &codes, model)) + .collect(); + + let keys: Vec = model_rule_keys("lint-key", &codes, &inputs); + + assert_eq!( + keys.len(), + test_case.expected_key_count, + "{}", + test_case.description + ); + assert_eq!(keys, expected, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs b/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs new file mode 100644 index 0000000000..e267ac3f3d --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_cache/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(crate) struct ModelRuleKeysTestCase { + pub(crate) description: &'static str, + pub(crate) models: Vec<(&'static str, &'static str, Option<&'static str>)>, + pub(crate) expected_key_count: usize, +} diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 339a58906f..621c051d3d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -155,8 +155,13 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result) -> Result dependencies, + None => dependency_calls(&model.query_sql, &request.dialect)?, + }; + let classification = classify_model(&query, model, dependencies); let parsed = ParsedModel { query, model, @@ -284,13 +293,21 @@ fn requires_model_query(rule: &RuleMetadata) -> bool { pub(super) fn parse_rule_statements( sql: &str, dialect_name: &str, +) -> Result, ParserError> { + parse_rule_statements_with_dependencies(sql, dialect_name, None) +} + +fn parse_rule_statements_with_dependencies( + sql: &str, + dialect_name: &str, + dependencies: Option<&mut Option>>, ) -> Result, ParserError> { let dialect = rules_dialect(dialect_name); - match parse_with_dialect(sql, dialect.as_ref()) { + match parse_with_dialect(sql, dialect.as_ref(), dependencies) { Ok(statements) => Ok(statements), Err(primary) if !dialect_name.eq_ignore_ascii_case("generic") => { let fallback_sql = normalize_generic_fallback(sql); - parse_with_dialect(&fallback_sql, &GenericDialect {}).map_err(|fallback| { + parse_with_dialect(&fallback_sql, &GenericDialect {}, None).map_err(|fallback| { ParserError::ParserError(format!( "{primary}; generic fallback also failed: {fallback}" )) @@ -331,11 +348,21 @@ fn normalize_generic_fallback(sql: &str) -> String { String::from_utf8(bytes).unwrap_or_else(|_| sql.to_owned()) } -fn parse_with_dialect(sql: &str, dialect: &dyn Dialect) -> Result, ParserError> { +fn parse_with_dialect( + sql: &str, + dialect: &dyn Dialect, + dependencies: Option<&mut Option>>, +) -> Result, ParserError> { + let tokens = Tokenizer::new(dialect, sql).tokenize_with_location()?; + if let Some(dependencies) = dependencies { + *dependencies = Some(dependency_calls_from_tokens( + tokens.iter().map(|token| &token.token), + )); + } let mut parser = Parser::new(dialect) .with_options(ParserOptions::new().with_trailing_commas(true)) .with_recursion_limit(RULES_PARSER_RECURSION_LIMIT) - .try_with_sql(sql)?; + .with_tokens_with_locations(tokens); parser.parse_statements() } @@ -692,9 +719,8 @@ pub(super) fn sole_table_name(query: &Query) -> Option { fn classify_model( query: &Query, model: &Model, - dialect_name: &str, -) -> Result { - let authored_dependencies = dependency_calls(&model.query_sql, dialect_name)?; + authored_dependencies: Vec, +) -> ModelClassification { let mut logical_seen = false; let ctes = top_ctes(query) .iter() @@ -738,11 +764,11 @@ fn classify_model( && root_select(query).is_some_and(|select| plain_select(select, true)) }); - Ok(ModelClassification { + ModelClassification { authored_dependencies, ctes, passthrough, - }) + } } fn dependency_calls(source: &str, dialect_name: &str) -> Result, String> { @@ -750,8 +776,11 @@ fn dependency_calls(source: &str, dialect_name: &str) -> Result, Str let tokens = Tokenizer::new(dialect.as_ref(), source) .tokenize() .map_err(|error| format!("could not tokenize model dependencies: {error}"))?; + Ok(dependency_calls_from_tokens(tokens.iter())) +} + +fn dependency_calls_from_tokens<'a>(tokens: impl Iterator) -> Vec { let tokens = tokens - .iter() .filter(|token| !matches!(token, Token::Whitespace(_))) .collect::>(); let mut calls: Vec = Vec::new(); @@ -794,7 +823,7 @@ fn dependency_calls(source: &str, dialect_name: &str) -> Result, Str } index = cursor; } - Ok(calls) + calls } fn query_dependency_calls(query: &Query) -> Vec { diff --git a/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs b/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs index 8e8bdaf663..b104206872 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/main/non_code_end.rs @@ -6,16 +6,16 @@ use crate::sql_scan::main::quote_end::quote_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; /// Return the end of a comment or quoted text starting at `index`, if one starts there. +#[inline] pub fn non_code_end( sql: &[u8], index: usize, policy: QuotePolicy, ) -> Result, Unclosed> { - if let Some(end) = comment_end(sql, index)? { - return Ok(Some(end)); - } match sql.get(index) { - Some(byte) if policy.is_quote(*byte) => quote_end(sql, index, policy).map(Some), + Some(b'-' | b'/') => comment_end(sql, index), + Some(b'\'' | b'"') => quote_end(sql, index, policy).map(Some), + Some(b'`') if policy.backtick_identifiers => quote_end(sql, index, policy).map(Some), Some(b'$') if policy.dollar_quotes => dollar_quote_end(sql, index), _ => Ok(None), } diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs index 4adf633977..911a581657 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/helpers.rs @@ -1,6 +1,82 @@ //! Shared lexical syntaxes for SQL scanning tests. -use crate::sql_scan::models::LexicalSyntax; +use crate::sql_scan::_helpers::dollar::dollar_quote_end; +use crate::sql_scan::main::comment_end::comment_end; +use crate::sql_scan::main::quote_end::quote_end; +use crate::sql_scan::models::{LexicalSyntax, QuotePolicy, Unclosed}; +use crate::sql_scan::tests::test_types::DispatchTestCase; + +// The pre-optimization comment-first dispatch is the differential oracle. Keep it +// independent of non_code_end so its first-byte dispatch is what the tests exercise. +fn comment_first_end( + sql: &[u8], + index: usize, + policy: QuotePolicy, +) -> Result, Unclosed> { + comment_end(sql, index)?.map_or_else( + || { + sql.get(index) + .filter(|byte| policy.is_quote(**byte)) + .map_or_else( + || { + sql.get(index) + .filter(|byte| **byte == b'$' && policy.dollar_quotes) + .map_or(Ok(None), |_| dollar_quote_end(sql, index)) + }, + |_| quote_end(sql, index, policy).map(Some), + ) + }, + |end| Ok(Some(end)), + ) +} + +pub(crate) fn dispatch_truncation_cases(input: &[u8]) -> Vec { + let mut cases = Vec::new(); + for flags in 0..16 { + let policy = QuotePolicy { + backtick_identifiers: flags & 1 != 0, + single_quote_backslash_escapes: flags & 2 != 0, + double_quote_backslash_escapes: flags & 4 != 0, + dollar_quotes: flags & 8 != 0, + }; + // The byte API must also handle truncations inside a multibyte character. + for length in 0..=input.len() { + let sql = &input[..length]; + for index in 0..=length + 1 { + cases.push(DispatchTestCase { + description: format!( + "input={input:?}, length={length}, index={index}, policy={policy:?}" + ), + sql: sql.to_vec(), + index, + policy, + expected_end: comment_first_end(sql, index, policy), + }); + } + } + } + cases +} + +pub(crate) fn ordinary_byte_cases(suffix: &[u8]) -> Vec { + let mut cases = Vec::new(); + for byte in + (u8::MIN..=u8::MAX).filter(|byte| !matches!(byte, b'-' | b'/' | b'\'' | b'"' | b'`' | b'$')) + { + for policy in [QuotePolicy::COMPILER, QuotePolicy::SQL_LINT] { + let mut sql = vec![byte]; + sql.extend_from_slice(suffix); + cases.push(DispatchTestCase { + description: format!("byte={byte}, suffix={suffix:?}, policy={policy:?}"), + sql, + index: 0, + policy, + expected_end: Ok(None), + }); + } + } + cases +} pub(crate) fn generic() -> LexicalSyntax { LexicalSyntax { diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs index 371063385b..d1b333d3a3 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_sql_scan.rs @@ -2,7 +2,10 @@ use crate::sql_scan::main::matching_paren::matching_paren; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::QuotePolicy; use crate::sql_scan::models::Unclosed; -use crate::sql_scan::tests::test_types::{MatchingParenPolicyTestCase, NonCodeEndTestCase}; +use crate::sql_scan::tests::helpers::{dispatch_truncation_cases, ordinary_byte_cases}; +use crate::sql_scan::tests::test_types::{ + DispatchInputTestCase, MatchingParenPolicyTestCase, NonCodeEndTestCase, +}; const QUOTES_ONLY: QuotePolicy = QuotePolicy { backtick_identifiers: false, @@ -18,6 +21,100 @@ const BACKSLASH_ONLY: QuotePolicy = QuotePolicy { dollar_quotes: false, }; +#[test] +fn given_every_truncation_and_policy_when_dispatching_then_matches_comment_first_scanning() { + let test_cases = [ + DispatchInputTestCase { + description: "ordinary code and non-nesting comments", + expected_cases: dispatch_truncation_cases, + sql: b"SELECT orders.id - 1 / 2; -- note\r\n/* outer /* inner */ tail */", + }, + DispatchInputTestCase { + description: "multibyte text and doubled quotes", + expected_cases: dispatch_truncation_cases, + sql: "SELECT 'é''ü', \"ñ\"\"名\", `名``é`, $$ ' -- /* ü $$".as_bytes(), + }, + DispatchInputTestCase { + description: "backslash escapes and tagged dollar quotes", + expected_cases: dispatch_truncation_cases, + sql: br#"'a\'b' "c\"d" `e\`f` $tag_2$ /* -- ' " $other$ $tag_2$"#, + }, + DispatchInputTestCase { + description: "invalid dollar tags and unclosed quotes", + expected_cases: dispatch_truncation_cases, + sql: b"$ $1 $1$ $tag $tag$unterminated $$unterminated", + }, + DispatchInputTestCase { + description: "dollar tag word boundaries", + expected_cases: dispatch_truncation_cases, + sql: "word$tag$ é$tag$ _$$ $tag$body$tag$".as_bytes(), + }, + DispatchInputTestCase { + description: "truncated opening delimiters", + expected_cases: dispatch_truncation_cases, + sql: b"- / -- /* ' \" ` \\", + }, + ]; + for test_case in test_cases { + for sample in (test_case.expected_cases)(test_case.sql) { + assert_eq!( + non_code_end(&sample.sql, sample.index, sample.policy), + sample.expected_end, + "{}: {}", + test_case.description, + sample.description, + ); + } + } +} + +#[test] +fn given_non_opening_bytes_when_dispatching_then_returns_no_boundary() { + let test_cases = [ + DispatchInputTestCase { + description: "before a line comment", + expected_cases: ordinary_byte_cases, + sql: b"--", + }, + DispatchInputTestCase { + description: "before a block comment", + expected_cases: ordinary_byte_cases, + sql: b"/*", + }, + DispatchInputTestCase { + description: "before a single quote", + expected_cases: ordinary_byte_cases, + sql: b"'", + }, + DispatchInputTestCase { + description: "before a double quote", + expected_cases: ordinary_byte_cases, + sql: b"\"", + }, + DispatchInputTestCase { + description: "before a backtick", + expected_cases: ordinary_byte_cases, + sql: b"`", + }, + DispatchInputTestCase { + description: "before a dollar quote", + expected_cases: ordinary_byte_cases, + sql: b"$$", + }, + ]; + for test_case in test_cases { + for sample in (test_case.expected_cases)(test_case.sql) { + assert_eq!( + non_code_end(&sample.sql, sample.index, sample.policy), + sample.expected_end, + "{}: {}", + test_case.description, + sample.description, + ); + } + } +} + #[test] fn given_quote_policies_when_matching_parentheses_then_policy_decides_quote_boundaries() { let test_cases = [ diff --git a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs index 4507180b90..42ebb866b3 100644 --- a/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/sql_scan/tests/test_types.rs @@ -15,6 +15,20 @@ pub(crate) struct NonCodeEndTestCase { pub(crate) expected_end: Result, Unclosed>, } +pub(crate) struct DispatchTestCase { + pub(crate) description: String, + pub(crate) sql: Vec, + pub(crate) index: usize, + pub(crate) policy: QuotePolicy, + pub(crate) expected_end: Result, Unclosed>, +} + +pub(crate) struct DispatchInputTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static [u8], + pub(crate) expected_cases: fn(&[u8]) -> Vec, +} + pub(crate) struct DialectNonCodeEndTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, diff --git a/fensu.toml b/fensu.toml index 220546fd6c..ffd3236e52 100644 --- a/fensu.toml +++ b/fensu.toml @@ -603,7 +603,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=1", @@ -612,20 +612,20 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_named.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs=3", @@ -810,7 +810,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/model_config/described_models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/errors.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_columns.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/template_data.rs=4", @@ -999,7 +999,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/guidance.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_expansion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_headers.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/rules/lint_preparation.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/rules/native_header_parser.rs=1", @@ -1145,17 +1145,17 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=24", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=25", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_config.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_contracts.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_header.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_opt_out.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_python_hooks.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_schema.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs=27", @@ -1193,7 +1193,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_modules.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/imported_python_nodes.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=23", + "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_file_record.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/discovery/model_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", @@ -1230,7 +1230,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/macro_bridge/store_keys.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/microbatches/conversion.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/model_config/config_templates.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/model_config/header_metadata.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/model_config/model_config_builder.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/dispatcher.rs=9", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index cbdbaa803f..0107e1a80e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -87,12 +87,16 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "cache_publication_ms", "physical_write_ms", "stale_traversal_ms", + "early_rules_build_ms", + "early_rules_wait_ms", "analysis_batch_cache_hits", "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", "sql_test_scan_cache_hits", "sql_test_scan_cache_misses", + "early_rules_hits", + "early_rules_misses", "project_reuse_hits", "project_reuse_misses", "project_reuse_bypasses", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index f25c77f4dc..7ba3c3653d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1023,3 +1023,28 @@ class DollarQuotedProjectVarCompileCase: description: str model_sql: str expected_fragment: str + + +@dataclass(frozen=True) +class EarlyBuiltinRulesTestCase: + """A rules-enabled project, the compile arguments and the early rules outcome it must report.""" + + description: str + repo_files: dict[str, str] + compile_arguments: tuple[str, ...] + expected_exit_code: int + expected_diagnostics: tuple[tuple[str, str, int, int], ...] + expected_early_hits: int + expected_early_misses: int + + +@dataclass(frozen=True) +class EarlyBuiltinRulesEditTestCase: + """A model edit between compiles and the early rules and rule cache outcomes of each compile.""" + + description: str + edited_path: str + edited_sql: str + expected_diagnostics: tuple[tuple[str, str, int, int], ...] + expected_early_outcomes: tuple[tuple[int, int], ...] + expected_rule_cache_outcomes: tuple[tuple[int, int], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 489c5722af..284f856294 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2964,6 +2964,31 @@ def compile_json_payload(*, project_dir: Path) -> tuple[int, dict[str, Any]]: return result.returncode, json.loads(result.stdout) +def compile_json_with( + *, project_dir: Path, arguments: tuple[str, ...] +) -> tuple[int, dict[str, Any]]: + """Run one JSON compile with extra arguments through the CLI and return its exit and payload.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json", *arguments), project_dir=project_dir + ) + return result.returncode, json.loads(result.stdout) + + +def early_rules_outcome(payload: dict[str, Any]) -> tuple[int, int]: + """The early built-in rules hits and misses one compile payload reports.""" + + timings: dict[str, Any] = payload["compile_timings"] + return timings["early_rules_hits"], timings["early_rules_misses"] + + +def rule_cache_outcome(payload: dict[str, Any]) -> tuple[int, int]: + """The rule cache hits and misses one compile payload reports.""" + + timings: dict[str, Any] = payload["compile_timings"] + return timings["rule_cache_hits"], timings["rule_cache_misses"] + + def diagnostic_location_keys(payload: dict[str, Any]) -> tuple[tuple[str, str, int, int], ...]: """Sorted code, path, line, and column of every diagnostic in a compile payload.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_builtin_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_builtin_rules.py new file mode 100644 index 0000000000..3f62424a1f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_early_builtin_rules.py @@ -0,0 +1,169 @@ +"""E2E tests for built-in rules evaluated speculatively during compilation.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + EarlyBuiltinRulesEditTestCase, + EarlyBuiltinRulesTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + compile_json_with, + diagnostic_location_keys, + early_rules_outcome, + rule_cache_outcome, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project + +_BUILTIN_PROJECT_TOML: str = ( + 'name = "commerce"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRMODEL102"]\n' +) +_CUSTOM_PROJECT_TOML: str = ( + 'name = "commerce"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRMODEL102", "XSQBR"]\n' + "\n[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" +) +_ORDERS_SQL: str = 'MODEL (description "Orders");\nSELECT 1 AS order_id, 2 AS amount\n' +_STAR_TOTALS_SQL: str = 'MODEL (description "Order totals");\nSELECT * FROM __ref("orders")\n' +_BROKEN_TOTALS_SQL: str = ( + 'MODEL (description "Order totals");\nSELECT order_id, missing_column FROM __ref("orders")\n' +) +_LONG_NAME_RULE: str = ( + "from sqlbuild.rules import Finding, Model, RuleContext, rule\n\n\n" + '@rule(code="XSQBRNAME001", message="Model names stay short", ' + 'remediation="Rename the model.")\n' + "def short_names(*, model: Model, ctx: RuleContext) -> list[Finding]:\n" + " if len(model.name) <= 8:\n" + " return []\n" + " return [ctx.finding(subject=model)]\n" +) +_STAR_FINDING: tuple[str, str, int, int] = ("SQBRMODEL102", "models/order_totals.sql", 1, 8) + + +@pytest.mark.parametrize( + "test_case", + ( + EarlyBuiltinRulesTestCase( + description="clean graph answers built-in rules from the early evaluation", + repo_files={ + "sqlbuild_project.toml": _BUILTIN_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _STAR_TOTALS_SQL, + }, + compile_arguments=(), + expected_exit_code=1, + expected_diagnostics=(_STAR_FINDING,), + expected_early_hits=1, + expected_early_misses=0, + ), + EarlyBuiltinRulesTestCase( + description="graph error skips rules and discards the early evaluation", + repo_files={ + "sqlbuild_project.toml": _BUILTIN_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _BROKEN_TOTALS_SQL, + }, + compile_arguments=(), + expected_exit_code=1, + expected_diagnostics=(("B002", "models/order_totals.sql", 2, 18),), + expected_early_hits=0, + expected_early_misses=0, + ), + EarlyBuiltinRulesTestCase( + description="model selection evaluates rules without an early evaluation", + repo_files={ + "sqlbuild_project.toml": _BUILTIN_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _STAR_TOTALS_SQL, + }, + compile_arguments=("--select", "order_totals"), + expected_exit_code=1, + expected_diagnostics=(_STAR_FINDING,), + expected_early_hits=0, + expected_early_misses=0, + ), + EarlyBuiltinRulesTestCase( + description="custom rules keep built-in rules in the rules phase", + repo_files={ + "sqlbuild_project.toml": _CUSTOM_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _STAR_TOTALS_SQL, + "rules/naming.py": _LONG_NAME_RULE, + }, + compile_arguments=(), + expected_exit_code=1, + expected_diagnostics=( + _STAR_FINDING, + ("XSQBRNAME001", "models/order_totals.sql", 1, 1), + ), + expected_early_hits=0, + expected_early_misses=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_rules_project_when_compiling_cold_and_warm_then_early_rules_keep_findings( + test_case: EarlyBuiltinRulesTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="commerce", repo_files=test_case.repo_files + ) + + runs: list[tuple[int, dict[str, Any]]] = [ + compile_json_with(project_dir=project_dir, arguments=test_case.compile_arguments) + for _ in range(2) + ] + + for exit_code, payload in runs: + assert exit_code == test_case.expected_exit_code + assert diagnostic_location_keys(payload) == test_case.expected_diagnostics + assert early_rules_outcome(payload) == ( + test_case.expected_early_hits, + test_case.expected_early_misses, + ) + + +@pytest.mark.parametrize( + "test_case", + ( + EarlyBuiltinRulesEditTestCase( + description="editing one model reuses the other model's cached findings", + edited_path="models/orders.sql", + edited_sql='MODEL (description "Orders");\nSELECT 1 AS order_id, 3 AS amount\n', + expected_diagnostics=(_STAR_FINDING,), + expected_early_outcomes=((1, 0), (1, 0), (1, 0)), + expected_rule_cache_outcomes=((0, 2), (2, 0), (1, 1)), + ), + ), + ids=lambda case: case.description, +) +def test_given_early_rules_when_compiling_cold_warm_and_edited_then_rule_cache_still_hits( + test_case: EarlyBuiltinRulesEditTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="commerce", + repo_files={ + "sqlbuild_project.toml": _BUILTIN_PROJECT_TOML, + "models/orders.sql": _ORDERS_SQL, + "models/order_totals.sql": _STAR_TOTALS_SQL, + }, + ) + + payloads: list[dict[str, Any]] = [compile_json_with(project_dir=project_dir, arguments=())[1]] + payloads.append(compile_json_with(project_dir=project_dir, arguments=())[1]) + (project_dir / test_case.edited_path).write_text(test_case.edited_sql, encoding="utf-8") + payloads.append(compile_json_with(project_dir=project_dir, arguments=())[1]) + + assert diagnostic_location_keys(payloads[-1]) == test_case.expected_diagnostics + assert tuple(early_rules_outcome(payload) for payload in payloads) == ( + test_case.expected_early_outcomes + ) + assert tuple(rule_cache_outcome(payload) for payload in payloads) == ( + test_case.expected_rule_cache_outcomes + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py index 5b50be7a74..ded3f76be0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_commented_fixtures.py @@ -86,6 +86,63 @@ @pytest.mark.parametrize( "test_case", [ + CommentedFixtureE2ETestCase( + description="bounded CTE casts joins and unions execute with mocked sources", + test_files={ + "macros/status.py": "", + "models/orders.sql": ( + "MODEL (description 'Orders through projected CTEs.');\n" + 'WITH imported AS (SELECT * FROM __source("raw_orders")),\n' + "projected AS (SELECT CAST(COALESCE(b.order_id, 0) + 0 AS INTEGER) AS order_id,\n" + "b.status AS status FROM imported AS b\n" + "LEFT JOIN imported AS c ON c.order_id = b.order_id),\n" + "combined AS (SELECT order_id, status FROM projected\n" + "UNION ALL BY NAME SELECT order_id, status FROM projected WHERE order_id = 0),\n" + "final AS (SELECT CAST(order_id AS INTEGER) AS order_id, status FROM combined)\n" + "SELECT order_id, status FROM final\n" + ), + "tests/unit/test_orders.sql": ( + 'TEST (name "projected_orders");\nWITH\n' + "__source__raw_orders AS (\n" + "SELECT 1 AS order_id, 'paid' AS status UNION ALL SELECT 2, 'void'\n" + "),\n" + _LINE_COMMENTED_EXPECTED + ), + }, + expected_exit_code=0, + expected_output_fragments=("PASS=1", "FAIL=0"), + ), + CommentedFixtureE2ETestCase( + description="nested model CTEs and marker lookalikes preserve mocked source results", + test_files={ + "macros/status.py": "", + "models/_macros/status.py": _PROJECT_FILES["macros/status.py"], + "models/orders.sql": ( + "MODEL (description 'Orders through nested CTEs.');\n" + 'WITH "recent" AS (\n' + " WITH nested AS (SELECT order_id, status, '__ref(\"ignored\")''s' AS note\n" + ' FROM __source("raw_orders"))\n' + " SELECT order_id, status FROM nested\n" + ")\n" + 'SELECT order_id, @normalize_status("status") AS status\n' + 'FROM "recent" /* __seed("ignored") */\n' + ), + "tests/unit/test_orders.sql": ( + 'TEST (name "nested_orders");\nWITH\n' + + _LINE_COMMENTED_SOURCE + + _LINE_COMMENTED_EXPECTED + ), + "tests/unit/test_order_counts.sql": ( + 'TEST (name "nested_order_counts");\nWITH\n' + + _LINE_COMMENTED_SOURCE + + "__expected__order_counts AS (\n" + " SELECT 'paid' AS status, CAST(1 AS BIGINT) AS order_count\n" + " UNION ALL SELECT 'void' AS status, CAST(1 AS BIGINT) AS order_count\n" + ") SELECT 1\n" + ), + }, + expected_exit_code=0, + expected_output_fragments=("PASS=2", "FAIL=0"), + ), CommentedFixtureE2ETestCase( description="line and block comments around fixture branches pass", test_files={ diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 0b688f3b54..28b083bba4 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -164,6 +164,16 @@ class RulesIntegrationTestCase: expected_code: str +@dataclass(frozen=True) +class EmptyRuleSelectionTestCase: + """A compile that selects no rules, with the Rules configuration it still validates.""" + + description: str + configuration: str + expected_exit_code: int + expected_messages: tuple[str, ...] + + @dataclass(frozen=True) class CustomHostSplitIntegrationTestCase: """A compile whose custom rules run on several hosts and must match one host.""" @@ -1138,3 +1148,32 @@ class ProviderAnnotationToleranceTestCase: annotation: str expected_exit_code: int expected_annotation_class_name: str | None + + +@dataclass(frozen=True) +class FirstModelErrorTestCase: + """Model files in path order and the one error compile reports for them.""" + + description: str + models: dict[str, str] + expected_message: str + + +@dataclass(frozen=True) +class HeaderColumnAuditBuildTestCase: + """Accepted values of a second model and whether its audits pass in DuckDB.""" + + description: str + accepted_values: str + expected_exit_code: int + expected_failed_models: tuple[str, ...] + + +@dataclass(frozen=True) +class TemplatedHeaderDiagnosticsTestCase: + """A second model's templated column and where the first model's diagnostic points.""" + + description: str + amount_type: str + amount_description: str + expected_location: dict[str, object] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 6ec216670e..5a2b39a3e8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -951,3 +951,33 @@ def write_provider_task_project(*, project_dir: Path, annotation: str) -> None: path: Path = project_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(text, encoding="utf-8") + + +def write_header_project(project_dir: Path, models: dict[str, str], project_toml: str) -> None: + """Write a DuckDB project with one SQL model file per name.""" + + _ = (project_dir / "sqlbuild_project.toml").write_text(project_toml) + model_dir: Path = project_dir / "models" + model_dir.mkdir() + for name, contents in models.items(): + _ = (model_dir / f"{name}.sql").write_text(contents) + + +def compile_diagnostics( + project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[int, list[dict[str, object]]]: + """Run one real CLI JSON compile and return its exit code and diagnostics.""" + + exit_code: int = main(["--project-dir", str(project_dir), "compile", "--json", "--no-cache"]) + payload: dict[str, object] = json.loads(capsys.readouterr().out) + return exit_code, cast(list[dict[str, object]], payload["diagnostics"]) + + +def diagnostics_by_model(diagnostics: list[dict[str, object]]) -> dict[str, str]: + """Each model's diagnostic as JSON with its own name replaced, to compare across models.""" + + by_model: dict[str, str] = {} + for item in diagnostics: + name: str = cast(str, item["resource_name"]) + by_model[name] = json.dumps(item, sort_keys=True).replace(name, "") + return by_model diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py new file mode 100644 index 0000000000..75be4255f7 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_model_headers.py @@ -0,0 +1,200 @@ +"""Real CLI and DuckDB coverage for header columns read natively before the ordered model loop.""" + +from __future__ import annotations + +from pathlib import Path + +import duckdb +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + FirstModelErrorTestCase, + HeaderColumnAuditBuildTestCase, + TemplatedHeaderDiagnosticsTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + compile_diagnostics, + diagnostics_by_model, + write_header_project, +) + +PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nlabel = "Order amount"\n\n' + 'amount_type = "VARCHAR"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +HOOKED_MODEL: str = ( + "MODEL (description 'Hooked orders.', pre_hooks [inline_sql('SELECT 1')], " + "columns (id (type INTEGER)));\nSELECT CAST(1 AS INTEGER) AS id\n" +) +MACRO_COLUMN_MODEL: str = ( + "MODEL (description 'Orders.', columns (id (type INTEGER, description '@cents(id)')));\n" + "SELECT CAST(1 AS INTEGER) AS id\n" +) +DUPLICATE_COLUMN_MODEL: str = ( + "MODEL (description 'Orders.', columns (id (type INTEGER), ID (type INTEGER)));\n" + "SELECT CAST(1 AS INTEGER) AS id\n" +) +NULLABLE_NOT_NULL_MODEL: str = ( + "MODEL (description 'Orders.', columns (id (nullable true, audits [not_null])));\n" + "SELECT CAST(1 AS INTEGER) AS id\n" +) +UNKNOWN_TEMPLATE_MODEL: str = ( + "MODEL (description 'Orders.', columns (id (type INTEGER, description '${missing}')));\n" + "SELECT CAST(1 AS INTEGER) AS id\n" +) +CONTRACT_MODEL: str = ( + "MODEL (description 'Orders.', contract enforced, columns (\n" + " id (type INTEGER, nullable false, audits [not_null, accepted_values (values [1, 2])]),\n" + " amount (type {amount_type}, description {description}),\n" + "));\n" + "SELECT CAST(1 AS INTEGER) AS id, CAST(2.0 AS DOUBLE) AS amount\n" +) + + +@pytest.mark.parametrize( + "test_case", + ( + FirstModelErrorTestCase( + description="a macro in a column stops before a later duplicate column", + models={ + "a_orders": HOOKED_MODEL, + "b_orders": MACRO_COLUMN_MODEL, + "c_orders": DUPLICATE_COLUMN_MODEL, + }, + expected_message="model config field 'columns.id.description' does not allow macros", + ), + FirstModelErrorTestCase( + description="a duplicate column stops before a later macro", + models={ + "a_orders": HOOKED_MODEL, + "b_orders": DUPLICATE_COLUMN_MODEL, + "c_orders": MACRO_COLUMN_MODEL, + }, + expected_message=( + "models/b_orders.sql model has duplicate column 'ID' " + "(column names are case-insensitive)" + ), + ), + FirstModelErrorTestCase( + description="a nullable not_null column stops before a later duplicate column", + models={"b_orders": NULLABLE_NOT_NULL_MODEL, "c_orders": DUPLICATE_COLUMN_MODEL}, + expected_message=( + "models/b_orders.sql column 'id' cannot set nullable = true and audit not_null" + ), + ), + FirstModelErrorTestCase( + description="a templated column read by Python stops before a later duplicate column", + models={"b_orders": UNKNOWN_TEMPLATE_MODEL, "c_orders": DUPLICATE_COLUMN_MODEL}, + expected_message="model config references unknown variable 'missing'", + ), + ), + ids=lambda case: case.description, +) +def test_given_several_failing_models_when_compiling_then_the_first_in_model_order_is_reported( + test_case: FirstModelErrorTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_header_project(tmp_path, test_case.models, PROJECT_TOML) + + exit_code, diagnostics = compile_diagnostics(tmp_path, capsys) + + assert exit_code == 1 + assert [item["message"] for item in diagnostics] == [test_case.expected_message] + + +@pytest.mark.parametrize( + "test_case", + ( + TemplatedHeaderDiagnosticsTestCase( + description="a templated column description", + amount_type="VARCHAR", + amount_description="'${label}'", + expected_location={ + "path": "models/a_orders.sql", + "line": 3, + "column": 3, + "end_line": 3, + "end_column": 9, + }, + ), + TemplatedHeaderDiagnosticsTestCase( + description="a templated column type", + amount_type="'${amount_type}'", + amount_description="'Order amount'", + expected_location={ + "path": "models/a_orders.sql", + "line": 3, + "column": 3, + "end_line": 3, + "end_column": 9, + }, + ), + ), + ids=lambda case: case.description, +) +def test_given_native_and_templated_header_columns_when_compiling_then_diagnostics_agree( + test_case: TemplatedHeaderDiagnosticsTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_header_project( + tmp_path, + { + "a_orders": CONTRACT_MODEL.format(amount_type="VARCHAR", description="'Order amount'"), + "b_orders": CONTRACT_MODEL.format( + amount_type=test_case.amount_type, description=test_case.amount_description + ), + }, + PROJECT_TOML, + ) + + exit_code, diagnostics = compile_diagnostics(tmp_path, capsys) + + by_model: dict[str, str] = diagnostics_by_model(diagnostics) + assert exit_code == 1 + assert sorted(by_model) == ["a_orders", "b_orders"] + assert by_model["a_orders"] == by_model["b_orders"] + assert diagnostics[0]["location"] == test_case.expected_location + + +@pytest.mark.parametrize( + "test_case", + ( + HeaderColumnAuditBuildTestCase( + description="native column audits pass", + accepted_values="[1, 2]", + expected_exit_code=0, + expected_failed_models=(), + ), + HeaderColumnAuditBuildTestCase( + description="a native accepted_values audit fails the build", + accepted_values="[2, 3]", + expected_exit_code=1, + expected_failed_models=("b_orders",), + ), + ), + ids=lambda case: case.description, +) +def test_given_native_header_column_audits_when_building_then_duckdb_runs_them( + test_case: HeaderColumnAuditBuildTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + contract: str = CONTRACT_MODEL.format(amount_type="DOUBLE", description="'Order amount'") + write_header_project( + tmp_path, + { + "a_orders": contract, + "b_orders": contract.replace("[1, 2]", test_case.accepted_values), + }, + PROJECT_TOML, + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "--no-color", "build"]) + output: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, output + for name in test_case.expected_failed_models: + assert f"final audit for '{name}' failed" in output + with duckdb.connect(str(tmp_path / "orders.duckdb"), read_only=True) as connection: + assert connection.execute("SELECT id, amount FROM a_orders").fetchall() == [(1, 2.0)] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d0d62ffc72..2ea7a60274 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -29,6 +29,7 @@ CustomRuleCacheSequenceTestCase, DynamicPivotRulesIntegrationTestCase, EarlyLintInputsTestCase, + EmptyRuleSelectionTestCase, ExplicitContractOutputRuleIntegrationTestCase, ImplicitAliasRuleIntegrationTestCase, NumericRangeDecisionIntegrationTestCase, @@ -2845,6 +2846,75 @@ def required_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]: assert payload["diagnostics"] == [] +@pytest.mark.parametrize( + "test_case", + [ + EmptyRuleSelectionTestCase( + "valid exception for an unevaluated rule compiles", + "[[rules.rule_exceptions]]\n" + 'rule = "SQBRCONTRACT101"\npath = "models/orders.sql"\nreason = "Legacy."\n', + 0, + (), + ), + EmptyRuleSelectionTestCase( + "unknown exception rule is rejected", + '[[rules.rule_exceptions]]\nrule = "SQBRNOPE999"\npath = "models/orders.sql"\n' + 'reason = "Legacy."\n', + 1, + ("rule exceptions target unknown rules: SQBRNOPE999",), + ), + EmptyRuleSelectionTestCase( + "missing exception path is rejected", + '[[rules.rule_exceptions]]\nrule = "SQBRCONTRACT101"\npath = "models/missing.sql"\n' + 'reason = "Legacy."\n', + 1, + ("rule exception path does not exist: models/missing.sql",), + ), + EmptyRuleSelectionTestCase( + "ignore selector matching no rule is rejected", + '[[rules.rule_ignores]]\nrules = ["SQBRZZZ"]\npaths = ["models/orders.sql"]\n' + 'reason = "Legacy."\n', + 1, + ("rule-ignore selector matches no rules: SQBRZZZ",), + ), + EmptyRuleSelectionTestCase( + "unknown built-in rule option is rejected", + "[rules.rule_options.SQBRSQL004]\nnope = 3\n", + 1, + ("rule SQBRSQL004 has unknown options: nope",), + ), + ], + ids=lambda case: case.description, +) +def test_given_empty_rule_selection_when_compiling_then_only_configuration_is_validated( + test_case: EmptyRuleSelectionTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = []\n\n{test_case.configuration}', + encoding="utf-8", + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text( + "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", encoding="utf-8" + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + timings: dict[str, int] = payload.get("compile_timings", {}) + + assert exit_code == test_case.expected_exit_code + assert [diagnostic["message"] for diagnostic in payload["diagnostics"]] == list( + test_case.expected_messages + ) + assert not rules_bulk_cache_path( + project_dir=tmp_path, file_name="native-response.json" + ).exists() + assert (timings.get("rule_cache_hits", 0), timings.get("rule_cache_misses", 0)) == (0, 0) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 4055ecad6e..681d5d2618 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -5,6 +5,16 @@ from dataclasses import dataclass, field +@dataclass(frozen=True) +class HeaderProjectionTestCase: + """Native header columns projected after real compilation and audit attachment.""" + + description: str + files: dict[str, str] + model_name: str + expected_columns: tuple[str, ...] + + @dataclass(frozen=True) class RetainedRulesRowsTestCase: """A real compile whose rules request reads model facts the compile-input stage retained.""" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py index 0dd0ca946b..7239f6f3ad 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_native_compiled_project.py @@ -13,14 +13,17 @@ from sqlbuild._native import ( InferredColumn, InvalidQualifiedIdentityError, + SchemaColumn, + SchemaModelEntry, build_compiled_project, discover_project_inputs, ) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import CompiledProjectView +from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + HeaderProjectionTestCase, ImmediateSyntaxErrorTestCase, ModelLoopErrorTestCase, ModelLoopParityTestCase, @@ -54,6 +57,49 @@ from sqlbuild._native import CompileInputsView +@pytest.mark.parametrize( + "test_case", + [ + HeaderProjectionTestCase( + "enum and factory audits", HEADER_CONTRACT_FILES, "order_statuses", ("status", "total") + ) + ], + ids=lambda case: case.description, +) +def test_given_header_contracts_when_compiling_then_columns_project_only_on_public_access( + test_case: HeaderProjectionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_files(project_dir=tmp_path, files=test_case.files) + constructed: list[SchemaColumn] = [] + + def counted(self: SchemaColumn, *args: Any, **kwargs: Any) -> None: + # Native records initialize their fields in __new__. + object.__init__(self) + constructed.append(self) + + monkeypatch.setattr(SchemaColumn, "__init__", counted) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + assert constructed == [] + models: dict[str, CompiledModelView] = {model.name: model for model in project.models} + model: CompiledModelView = models[test_case.model_name] + entry: SchemaModelEntry | None = model.schema_entry + assert entry is not None + columns: tuple[SchemaColumn, ...] = entry.columns + assert tuple(column.name for column in columns) == test_case.expected_columns + assert columns[0].type == "VARCHAR" + assert [audit.definition_name for audit in columns[0].audits] == ["not_null", "accepted_values"] + assert columns[0].audits[1].arguments == {"values": ("PLACED", "SHIPPED")} + assert entry.audits[0].name == "positive_total" + assert len(constructed) == 2 + assert model.schema_entry is entry + assert model.schema_entry.columns is columns + assert pickle.loads(pickle.dumps(columns)) == columns + + @pytest.mark.parametrize( "test_case", [SharedInferredColumnsTestCase("repeated output schemas share pickle entries", 24, 3)],