diff --git a/crates/sqlbuild-rules-native/Cargo.toml b/crates/sqlbuild-rules-native/Cargo.toml index de782e7349..dccde11346 100644 --- a/crates/sqlbuild-rules-native/Cargo.toml +++ b/crates/sqlbuild-rules-native/Cargo.toml @@ -30,6 +30,9 @@ stacker.workspace = true sqlparser.workspace = true toml.workspace = true +[build-dependencies] +sha2.workspace = true + [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-rules-native/build.rs b/crates/sqlbuild-rules-native/build.rs new file mode 100644 index 0000000000..a95c309a28 --- /dev/null +++ b/crates/sqlbuild-rules-native/build.rs @@ -0,0 +1,81 @@ +//! Derive a content identity for this native build so local caches notice source changes. +//! +//! The identity hashes every file under the listed crate entries plus the workspace lockfile, +//! keyed by `/`-separated paths relative to the crate root so it matches across platforms. +//! Entries that do not exist (for example the lockfile outside a workspace checkout) are left +//! out of both the hash and the `rerun-if-changed` list, because Cargo reruns a build script on +//! every build when a watched path is missing. + +use sha2::{Digest, Sha256}; +use std::path::{Component, Path, PathBuf}; + +const HASHED_ENTRIES: [&str; 3] = ["Cargo.toml", "src", "function_names"]; +const WORKSPACE_LOCKFILE: &str = "../../Cargo.lock"; + +fn main() -> Result<(), String> { + let crate_dir = + PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").map_err(|error| error.to_string())?); + let mut files: Vec<(String, PathBuf)> = Vec::new(); + for entry in HASHED_ENTRIES { + let path = crate_dir.join(entry); + if path.exists() { + println!("cargo:rerun-if-changed={}", path.display()); + collect_files(&crate_dir, &path, &mut files)?; + } + } + files.sort(); + let lockfile = crate_dir.join(WORKSPACE_LOCKFILE); + if lockfile.is_file() { + println!("cargo:rerun-if-changed={}", lockfile.display()); + files.push((WORKSPACE_LOCKFILE.to_owned(), lockfile)); + } + let mut digest = Sha256::new(); + for (label, file) in &files { + digest.update(label.as_bytes()); + digest.update([0]); + digest.update(std::fs::read(file).map_err(|error| error.to_string())?); + digest.update([0]); + } + let identity = format!("{:x}", digest.finalize()); + println!( + "cargo:rustc-env=SQLBUILD_NATIVE_SOURCE_HASH={}", + &identity[..16] + ); + Ok(()) +} + +fn portable_label(crate_dir: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix(crate_dir) + .map_err(|error| error.to_string())?; + let parts: Vec = relative + .components() + .map(|component| match component { + Component::Normal(part) => Ok(part.to_string_lossy().into_owned()), + other => Err(format!("unexpected path component {other:?}")), + }) + .collect::>()?; + Ok(parts.join("/")) +} + +fn collect_files( + crate_dir: &Path, + path: &Path, + files: &mut Vec<(String, PathBuf)>, +) -> Result<(), String> { + if path.is_file() { + files.push((portable_label(crate_dir, path)?, path.to_path_buf())); + return Ok(()); + } + if !path.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? { + collect_files( + crate_dir, + &entry.map_err(|error| error.to_string())?.path(), + files, + )?; + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 28598b4d26..696e9578db 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use crate::configuration::main::load; -use crate::constants::API_VERSION; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::engine::main::evaluate; use crate::models::CatalogueResponse; use crate::rules::main::{catalogue, selected_codes}; @@ -469,5 +469,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; module.add("API_VERSION", API_VERSION)?; + module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) } diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs index 4d59240106..f26728038a 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs @@ -242,7 +242,7 @@ fn is_generated_cte_name(name: &str) -> bool { /// Distinct lowercase identifiers outside strings and comments, in first-appearance order. fn identifier_tokens(sql: &str, patterns: &SqlTestPatterns) -> Vec { - let protected = protected_ranges(&patterns.protected, sql); + let protected = protected_ranges(&patterns.lexical, sql); let mut seen: HashSet = HashSet::new(); let mut tokens: Vec = Vec::new(); let mut push = |token: &str| { diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs index ba1cda0325..de59e2f7ad 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs @@ -5,21 +5,21 @@ use std::collections::HashSet; use regex::{Captures, Regex}; use crate::compiler::_helpers::sql_tests::planning::compile_error; -use crate::sql_scan::main::matching_paren::matching_paren; +use crate::sql_scan::main::dialect_matching_paren::dialect_matching_paren; +use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use crate::sql_scan::main::skip_whitespace::skip_whitespace; -use crate::sql_scan::models::QuotePolicy; -use crate::sql_scan::models::Unclosed; +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; pub(crate) fn replace_callable_markers( sql: &str, pattern: &Regex, - protected_pattern: &Regex, + syntax: &LexicalSyntax, mut replacement: F, ) -> Result<(String, HashSet), String> where F: FnMut(&str, &str) -> Option<(String, bool)>, { - let protected = protected_ranges(protected_pattern, sql); + let protected = protected_ranges(syntax, sql); let mut output = String::with_capacity(sql.len()); let mut reached: HashSet = HashSet::new(); let mut cursor = 0; @@ -34,7 +34,7 @@ where if !sql[suffix_start..].starts_with('(') { continue; } - let suffix_end = matching_paren_end(sql, suffix_start)?; + let suffix_end = matching_paren_end(sql, suffix_start, syntax)?; let Some(name) = captures.get(1).map(|value| value.as_str()) else { continue; }; @@ -53,8 +53,8 @@ where Ok((output, reached)) } -fn matching_paren_end(sql: &str, open: usize) -> Result { - matching_paren(sql.as_bytes(), open, QuotePolicy::COMPILER) +fn matching_paren_end(sql: &str, open: usize, syntax: &LexicalSyntax) -> Result { + dialect_matching_paren(sql.as_bytes(), open, syntax) .map(|close| close + 1) .map_err(|error| { compile_error(match error { @@ -68,13 +68,13 @@ fn matching_paren_end(sql: &str, open: usize) -> Result { pub(crate) fn replace_named_markers( sql: &str, pattern: &Regex, - protected_pattern: &Regex, + syntax: &LexicalSyntax, mut replacement: F, ) -> String where F: FnMut(&str) -> Option, { - replace_marker_captures(sql, pattern, protected_pattern, |captures| { + replace_marker_captures(sql, pattern, syntax, |captures| { replacement(captures.get(1)?.as_str()) }) } @@ -82,13 +82,13 @@ where pub(crate) fn replace_dbt_ref_markers( sql: &str, pattern: &Regex, - protected_pattern: &Regex, + syntax: &LexicalSyntax, mut replacement: F, ) -> String where F: FnMut(&str) -> Option, { - replace_marker_captures(sql, pattern, protected_pattern, |captures| { + replace_marker_captures(sql, pattern, syntax, |captures| { let first = captures.get(1)?.as_str(); let name = captures.get(2).map_or_else( || first.to_string(), @@ -101,13 +101,13 @@ where fn replace_marker_captures( sql: &str, pattern: &Regex, - protected_pattern: &Regex, + syntax: &LexicalSyntax, mut replacement: F, ) -> String where F: FnMut(&Captures<'_>) -> Option, { - let protected = protected_ranges(protected_pattern, sql); + let protected = protected_ranges(syntax, sql); let mut output = String::with_capacity(sql.len()); let mut cursor = 0; for captures in pattern.captures_iter(sql) { @@ -128,8 +128,8 @@ where output } -pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str) -> Vec { - let protected = protected_ranges(protected_pattern, sql); +pub(crate) fn marker_names(pattern: &Regex, syntax: &LexicalSyntax, sql: &str) -> Vec { + let protected = protected_ranges(syntax, sql); let mut names: Vec = Vec::new(); for captures in pattern.captures_iter(sql) { let Some(full) = captures.get(0) else { @@ -145,11 +145,8 @@ pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str names } -pub(crate) fn protected_ranges(pattern: &Regex, sql: &str) -> Vec<(usize, usize)> { - pattern - .find_iter(sql) - .map(|value| (value.start(), value.end())) - .collect() +pub(crate) fn protected_ranges(syntax: &LexicalSyntax, sql: &str) -> Vec<(usize, usize)> { + dialect_non_code_ranges(sql, syntax) } pub(crate) fn in_protected_range(index: usize, ranges: &[(usize, usize)]) -> bool { diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs index d0203ec5aa..7ca1946254 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs @@ -27,6 +27,7 @@ use crate::compiler::_helpers::sql_tests::rendering::{ rendered_chain_steps, }; use crate::constants::{TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE}; +use crate::sql_scan::models::LexicalSyntax; const DEFAULT_WORKERS: usize = 4; const MAX_WORKERS: usize = 4; @@ -73,6 +74,7 @@ struct PlanBatchRequest { render_sql: bool, #[serde(default = "default_true")] include_plan: bool, + lexical_syntax: LexicalSyntax, } #[derive(Debug, Deserialize)] @@ -80,6 +82,7 @@ struct PlanBatchRequest { struct ChainBatchRequest { models: Vec, tests: Vec, + lexical_syntax: LexicalSyntax, } /// Chain ordering reads only declared dependencies, so chain requests omit model SQL. @@ -219,12 +222,12 @@ pub(crate) struct SqlTestPatterns { table_function: Regex, dbt_reference: Regex, test_reference: Regex, - pub(crate) protected: Regex, + pub(crate) lexical: LexicalSyntax, pub(crate) identifier: Regex, } impl SqlTestPatterns { - fn new() -> Result { + fn new(lexical: LexicalSyntax) -> Result { Ok(Self { reference: compile_pattern(r#"(?i)__ref\(\"([^\"]+)\"\)"#)?, source: compile_pattern(r#"(?i)__source\(\"([^\"]+)\"\)"#)?, @@ -235,9 +238,7 @@ impl SqlTestPatterns { r#"(?i)__dbt_ref\(\s*\"([^\"]+)\"\s*(?:,\s*\"([^\"]+)\"\s*)?\)"#, )?, test_reference: compile_pattern(r#"(?i)__(?:ref|source|seed|dbt_ref|table_fn)\("#)?, - protected: compile_pattern( - r#"(?s)'(?:''|[^'])*'|\"(?:\"\"|[^\"])*\"|`(?:``|[^`])*`|\$\$.*?\$\$|--[^\n]*|/\*.*?\*/"#, - )?, + lexical, identifier: compile_pattern(r"[A-Za-z_][A-Za-z0-9_$]*")?, }) } @@ -532,7 +533,7 @@ impl GeneratedCteState { pub(crate) fn resolve_chains_json(request_json: &str) -> Result { let request: ChainBatchRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; - let patterns = SqlTestPatterns::new()?; + let patterns = SqlTestPatterns::new(request.lexical_syntax)?; let models: HashMap = request .models .into_iter() @@ -616,7 +617,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result set_difference_operator: request.set_difference_operator, requires_derived_table_aliases: request.requires_derived_table_aliases, analysis_templates: Arc::new(Mutex::new(HashMap::new())), - patterns: SqlTestPatterns::new()?, + patterns: SqlTestPatterns::new(request.lexical_syntax)?, render_dialect, rejects_nested_with, }; @@ -1053,7 +1054,7 @@ fn resolve_assertion_textual_sql( let mut seen: HashSet = HashSet::new(); for name in marker_names( &request.patterns.reference, - &request.patterns.protected, + &request.patterns.lexical, request.assertion_sql, ) { if !seen.insert(name.clone()) { @@ -1165,7 +1166,7 @@ fn topo_sort_model_chain(request: TopoSortRequest<'_>) -> Vec { } if let Some(model) = models.get(name) { let mut dependencies = if let Some(override_sql) = overrides.get(name) { - marker_names(&patterns.reference, &patterns.protected, override_sql) + marker_names(&patterns.reference, &patterns.lexical, override_sql) } else { model.model_dependencies.clone() }; @@ -1348,7 +1349,7 @@ fn resolve_textual_sql(request: TextualResolutionRequest<'_>) -> Result) -> Result) -> Result) -> Result) -> Vec }); } }; - for name in marker_names(&patterns.reference, &patterns.protected, sql) { + for name in marker_names(&patterns.reference, &patterns.lexical, sql) { let message = format!( "test '{test_name}': model '{model_name}' references __ref('{name}') which has no mock and is not in the expected chain" ); @@ -1629,14 +1630,14 @@ fn unresolved_reference_warnings(request: UnresolvedReferenceRequest<'_>) -> Vec (&patterns.source, SOURCE_FUNCTION), (&patterns.seed, SEED_FUNCTION), ] { - for name in marker_names(pattern, &patterns.protected, sql) { + for name in marker_names(pattern, &patterns.lexical, sql) { let message = format!( "test '{test_name}': model '{model_name}' references {function_name}('{name}') which has no mock" ); warn(function_name, name, message); } } - let protected = protected_ranges(&patterns.protected, sql); + let protected = protected_ranges(&patterns.lexical, sql); for captures in patterns.dbt_reference.captures_iter(sql) { let Some(full) = captures.get(0) else { continue; @@ -1698,7 +1699,7 @@ fn assertion_ref_targets( ) -> Vec { let mut targets: Vec = Vec::new(); for (_, sql) in assertions { - targets.extend(marker_names(&patterns.reference, &patterns.protected, sql)); + targets.extend(marker_names(&patterns.reference, &patterns.lexical, sql)); } dedupe(targets) } @@ -1707,9 +1708,9 @@ fn has_unresolved_test_reference(sql: &str, patterns: &SqlTestPatterns) -> bool if !patterns.test_reference.is_match(sql) { return false; } - !marker_names(&patterns.reference, &patterns.protected, sql).is_empty() - || !marker_names(&patterns.source, &patterns.protected, sql).is_empty() - || !marker_names(&patterns.seed, &patterns.protected, sql).is_empty() + !marker_names(&patterns.reference, &patterns.lexical, sql).is_empty() + || !marker_names(&patterns.source, &patterns.lexical, sql).is_empty() + || !marker_names(&patterns.seed, &patterns.lexical, sql).is_empty() || patterns.dbt_reference.is_match(sql) || patterns.table_function.is_match(sql) } diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs index e2dc0549c6..fcfaeb3e5a 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs @@ -393,6 +393,7 @@ pub(crate) fn model_test_batch_returns_ordered_artifact() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ { "name": "stg_orders", @@ -455,6 +456,7 @@ pub(crate) fn unicode_cte_after_leading_with_preserves_identifier() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [{ "name": "orders", "querySql": "WITH \"订单行\" AS (SELECT order_id FROM __source(\"raw_orders\")) SELECT order_id FROM \"订单行\"", @@ -499,6 +501,7 @@ pub(crate) fn unresolved_reference_fast_rejection_preserves_warning() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [{ "name": "orders", "querySql": "SELECT * FROM __SOURCE(\"missing_orders\")", @@ -588,6 +591,7 @@ pub(crate) fn shared_textual_chain_renders_each_model_once() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": models, "functions": [], "tests": [{ @@ -670,6 +674,7 @@ fn plan_deep_diamond_with_missing_mock(sql_analysis_enabled: bool) -> Value { serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": deep_diamond_models(LAYERS), "tests": [{ "name": "orders_totals", @@ -718,6 +723,7 @@ pub(crate) fn plan_without_rendering_returns_executable_steps() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ { "name": "stg_orders", @@ -797,6 +803,7 @@ pub(crate) fn textual_assertion_with_clause_merges_lifted_ctes() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ { "name": "stg_orders", @@ -859,6 +866,7 @@ pub(crate) fn textual_assertion_with_clause_merges_lifted_ctes() -> bool { pub(crate) fn upstream_fallback_resolves() -> bool { let request = json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ {"name": "stg_orders", "querySql": "SELECT __udf(\"identity_value\")(order_id) AS order_id FROM __source(\"raw_orders\")"}, {"name": "orders", "querySql": "SELECT * FROM __ref(\"stg_orders\")", "modelDependencies": ["stg_orders"]} @@ -911,6 +919,7 @@ pub(crate) fn chain_resolution_orders_unmocked_models() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_chain_resolution::resolve_chains_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": deep_diamond_models(2), "tests": [ { @@ -1034,6 +1043,7 @@ pub(crate) fn long_chain_plan_output_stays_linear() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": models, "tests": [{ "name": "orders_tip", @@ -1193,6 +1203,7 @@ pub(crate) fn snowflake_plan_keeps_quoted_expected_columns() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [{ "name": "orders", "querySql": "SELECT order_id AS \"Order Id\", status, amount FROM __source(\"raw_orders\")", @@ -1240,6 +1251,7 @@ fn plan_helper_scope_request( ) -> Result { crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ { "name": "orders", @@ -1368,6 +1380,7 @@ pub(crate) fn mock_read_through_helper_brings_its_mock_dependencies_into_scope() let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ {"name": "raw_orders", "querySql": "SELECT 1 AS order_id", "modelDependencies": []}, {"name": "stg_orders", "querySql": "SELECT order_id FROM __ref(\"raw_orders\")", "modelDependencies": ["raw_orders"]}, @@ -1633,3 +1646,14 @@ pub(super) fn snowflake_function_synonyms_stay_as_authored() -> bool { && !sql.contains("STARTS_WITH") && !sql.contains("SUBSTRING") } + +fn generic_lexical_syntax() -> Value { + json!({ + "backslashEscapeQuotes": [], + "escapeStringPrefix": false, + "rawStringPrefix": false, + "tripleQuotedStrings": false, + "nestedBlockComments": false, + "lineCommentPrefixes": ["--"] + }) +} diff --git a/crates/sqlbuild-rules-native/src/constants.rs b/crates/sqlbuild-rules-native/src/constants.rs index bbd77bbedb..1d2d0a3c31 100644 --- a/crates/sqlbuild-rules-native/src/constants.rs +++ b/crates/sqlbuild-rules-native/src/constants.rs @@ -1,4 +1,9 @@ pub(crate) const API_VERSION: u32 = 1; +pub(crate) const NATIVE_BUILD_IDENTITY: &str = concat!( + env!("CARGO_PKG_VERSION"), + "+", + env!("SQLBUILD_NATIVE_SOURCE_HASH") +); pub(crate) const SCOPE_METADATA_SCHEMA_VERSION: u32 = 3; pub(crate) const BUILT_IN_RULE_NAMESPACE: &str = "SQBR"; pub(crate) const PIPELINE_SUBJECT: &str = "pipeline"; diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs index 8a894db89b..4fad5b6fc6 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs @@ -1,7 +1,7 @@ use crate::configuration::main::validate as config; use crate::constants::{ - API_VERSION, CUSTOM_HOST_REQUIRED_ERROR, GIT_DIRECTORY, LOGS_DIRECTORY, PYTHON_EXTENSION, - RULES_DIRECTORY, TARGET_DIRECTORY, + API_VERSION, CUSTOM_HOST_REQUIRED_ERROR, GIT_DIRECTORY, LOGS_DIRECTORY, NATIVE_BUILD_IDENTITY, + PYTHON_EXTENSION, RULES_DIRECTORY, TARGET_DIRECTORY, }; use crate::engine::_helpers::cache::{Cache, RuleCacheBucket, RuleCacheEntry}; use crate::models::{ @@ -552,7 +552,7 @@ fn custom_model_fact_identities( let mut digest = Sha256::new(); digest.update(b"custom-rule-model-facts-v1"); digest.update(ANALYSIS_BATCH_SCHEMA_VERSION.to_le_bytes()); - digest.update(env!("CARGO_PKG_VERSION").as_bytes()); + digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); digest.update(serde_json::to_vec(model).map_err(|error| error.to_string())?); Ok(( model.relative_path.clone(), @@ -575,6 +575,7 @@ fn custom_model_fact_identities( fn custom_rule_identity(rule: &RuleMetadata, request: &EvaluateRequest) -> Result { let mut digest = Sha256::new(); digest.update(b"custom-rule-identity-v1"); + digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); digest.update(serde_json::to_vec(rule).map_err(|error| error.to_string())?); digest.update( serde_json::to_vec(&request.config.rule_options.get(&rule.code)) @@ -623,7 +624,7 @@ fn model_cache_identity( identity: RuntimeIdentity { producer: "sqlbuild-rules".to_owned(), fact_schema: "sqlbuild-rules-model-v1".to_owned(), - runtime: env!("CARGO_PKG_VERSION").to_owned(), + runtime: NATIVE_BUILD_IDENTITY.to_owned(), rule_pack: ruleset.to_owned(), configuration: project.unwrap_or("model-local").to_owned(), }, diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs index 09ec3df1ec..ded80cec80 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs @@ -1,6 +1,7 @@ use serde_json::{Value, json}; use tempfile::TempDir; +use crate::constants::NATIVE_BUILD_IDENTITY; use crate::engine::main::evaluate::evaluate_json; use crate::engine::tests::helpers; use crate::engine::tests::test_types; @@ -1343,3 +1344,22 @@ fn given_threshold_override_change_when_evaluating_then_ruleset_fingerprint_chan } Ok(()) } + +#[test] +fn given_native_build_when_reading_cache_identity_then_version_carries_source_hash() { + let test_cases = [test_types::BuildIdentityTestCase { + description: "cache identity is the package version plus a source content hash", + expected_prefix: concat!(env!("CARGO_PKG_VERSION"), "+"), + expected_hash_length: 16, + }]; + for test_case in &test_cases { + let hash = NATIVE_BUILD_IDENTITY.strip_prefix(test_case.expected_prefix); + + assert!( + hash.is_some_and(|hash| hash.len() == test_case.expected_hash_length + && hash.bytes().all(|byte| byte.is_ascii_hexdigit())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs index bf1a75ccb9..f2410357e2 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs @@ -106,6 +106,12 @@ pub(crate) struct ThresholdEvaluationTestCase { pub(crate) expected_codes: &'static [&'static str], } +pub(crate) struct BuildIdentityTestCase { + pub(crate) description: &'static str, + pub(crate) expected_prefix: &'static str, + pub(crate) expected_hash_length: usize, +} + pub(crate) struct ThresholdFingerprintTestCase { pub(crate) description: &'static str, pub(crate) base_config: Value, diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs index 5e19697f5e..f5105ad6cd 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs @@ -1,4 +1,4 @@ -use crate::constants::API_VERSION; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::models::RulesCodeGrammar; use crate::models::{CustomRule, ResolveRulesRequest, RuleGuidance, RuleMetadata}; use fensu_policy::policy::errors::PolicyError; @@ -487,6 +487,7 @@ pub(crate) fn fingerprint( config: &crate::models::RulesConfig, dialect: &str, ) -> Result { - let payload = serde_json::to_vec(&(rules, config, dialect)).map_err(|e| e.to_string())?; + let payload = serde_json::to_vec(&(rules, config, dialect, NATIVE_BUILD_IDENTITY)) + .map_err(|e| e.to_string())?; Ok(format!("{:x}", Sha256::digest(payload))) } diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs index 0d15bb9956..ccce73474a 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs @@ -172,11 +172,6 @@ pub(super) fn transform_statements( if let Some(with) = &mut select.with { with.ctes = fold_ctes(std::mem::take(&mut with.ctes), &authored); } - for join in &mut select.joins { - for identifier in &mut join.using { - *identifier = fold(identifier.clone(), &authored); - } - } if let Some(windows) = &mut select.windows { for window in windows { window.name = fold(window.name.clone(), &authored); diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs new file mode 100644 index 0000000000..4d4f6ee5bb --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs @@ -0,0 +1,141 @@ +//! Dialect-aware comment and quoted-text boundaries, mirroring Python `dialect_non_code_end`. + +use crate::sql_scan::_helpers::dollar::{continues_dollar_word, dollar_quote_end}; +use crate::sql_scan::_helpers::parens::matching_paren_with; +use crate::sql_scan::constants::{ + STRING_PREFIX_CHARACTERS, STRING_PREFIX_MAX_LENGTH, TRIPLE_QUOTE_LENGTH, +}; +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; + +/// Return the end of a comment or quoted text starting at `index` under the dialect's rules. +pub(crate) fn non_code_end( + sql: &[u8], + index: usize, + syntax: &LexicalSyntax, +) -> Result, Unclosed> { + if sql[index..].starts_with(b"/*") { + return block_comment_end(sql, index, syntax).map(Some); + } + if syntax + .line_comment_prefixes + .iter() + .any(|prefix| sql[index..].starts_with(prefix.as_bytes())) + { + return Ok(Some( + sql[index..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(sql.len(), |offset| index + offset + 1), + )); + } + match sql.get(index) { + Some(b'$') => dollar_quote_end(sql, index), + Some(b'\'' | b'"' | b'`') => quoted_text_end(sql, index, syntax).map(Some), + _ => Ok(None), + } +} + +/// Return every comment and quoted-text range in `sql`; an unclosed one extends to the end. +pub(crate) fn non_code_ranges(sql: &str, syntax: &LexicalSyntax) -> Vec<(usize, usize)> { + let bytes = sql.as_bytes(); + let mut ranges: Vec<(usize, usize)> = Vec::new(); + let mut index = 0; + while index < bytes.len() { + match non_code_end(bytes, index, syntax) { + Ok(Some(end)) => { + ranges.push((index, end)); + index = end; + } + Ok(None) => index += 1, + Err(_) => { + ranges.push((index, bytes.len())); + break; + } + } + } + ranges +} + +/// Return the index of the parenthesis closing the one at `open` under the dialect's rules. +pub(crate) fn matching_paren( + sql: &[u8], + open: usize, + syntax: &LexicalSyntax, +) -> Result { + matching_paren_with(sql, open, |sql, index| non_code_end(sql, index, syntax)) +} + +fn block_comment_end(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Result { + let mut depth = 1usize; + let mut index = start + 2; + while depth > 0 { + let close = find(sql, b"*/", index).ok_or(Unclosed::BlockComment)?; + let open = if syntax.nested_block_comments { + find(&sql[..close], b"/*", index) + } else { + None + }; + if let Some(open) = open { + depth += 1; + index = open + 2; + } else { + depth -= 1; + index = close + 2; + } + } + Ok(index) +} + +fn quoted_text_end(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Result { + let quote = sql[start]; + let prefix = string_literal_prefix(sql, start); + let mut backslash_escapes = syntax.backslash_escapes(quote) + || (syntax.escape_string_prefix && quote == b'\'' && prefix == b"e"); + if syntax.raw_string_prefix && prefix.contains(&b'r') { + backslash_escapes = false; + } + let triple = [quote; TRIPLE_QUOTE_LENGTH]; + let delimiter: &[u8] = if syntax.triple_quoted_strings && sql[start..].starts_with(&triple) { + &triple + } else { + &triple[..1] + }; + let mut index = start + delimiter.len(); + while index < sql.len() { + if backslash_escapes && sql[index] == b'\\' { + index += 2; + continue; + } + if sql[index..].starts_with(delimiter) { + if delimiter.len() == 1 && sql.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Ok(index + delimiter.len()); + } + index += 1; + } + Err(Unclosed::Quote) +} + +fn string_literal_prefix(sql: &[u8], start: usize) -> Vec { + let mut prefix_start = start; + while prefix_start > 0 + && start - prefix_start < STRING_PREFIX_MAX_LENGTH + && STRING_PREFIX_CHARACTERS.contains(&sql[prefix_start - 1]) + { + prefix_start -= 1; + } + if prefix_start > 0 && continues_dollar_word(sql[prefix_start - 1]) { + return Vec::new(); + } + sql[prefix_start..start].to_ascii_lowercase() +} + +fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { + haystack + .get(from..)? + .windows(needle.len()) + .position(|window| window == needle) + .map(|offset| from + offset) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs new file mode 100644 index 0000000000..601205157d --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs @@ -0,0 +1,31 @@ +//! Dollar-quoted literal boundaries shared by every scanning policy. + +use crate::sql_scan::models::Unclosed; + +/// Return the end of the `$$` or `$tag$` literal opening at `start`, if one opens there. +pub(crate) fn dollar_quote_end(sql: &[u8], start: usize) -> Result, Unclosed> { + if start + .checked_sub(1) + .is_some_and(|previous| continues_dollar_word(sql[previous])) + { + return Ok(None); + } + let tag_end = sql[start + 1..] + .iter() + .position(|byte| !(byte.is_ascii_alphanumeric() || *byte == b'_')) + .map_or(sql.len(), |offset| start + 1 + offset); + let tag = &sql[start + 1..tag_end]; + if sql.get(tag_end) != Some(&b'$') || tag.first().is_some_and(u8::is_ascii_digit) { + return Ok(None); + } + let delimiter = &sql[start..=tag_end]; + sql[tag_end + 1..] + .windows(delimiter.len()) + .position(|window| window == delimiter) + .map(|offset| Some(tag_end + 1 + offset + delimiter.len())) + .ok_or(Unclosed::Quote) +} + +pub(crate) fn continues_dollar_word(byte: u8) -> bool { + !byte.is_ascii() || byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs new file mode 100644 index 0000000000..efa92dc9fc --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod dialect; +pub(crate) mod dollar; +pub(crate) mod parens; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs new file mode 100644 index 0000000000..525b49e34f --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs @@ -0,0 +1,31 @@ +//! Parenthesis matching shared by every scanning policy. + +use crate::sql_scan::models::Unclosed; + +/// Return the index of the parenthesis closing the one at `open`, skipping non-code spans. +pub(crate) fn matching_paren_with( + sql: &[u8], + open: usize, + non_code_end: impl Fn(&[u8], usize) -> Result, Unclosed>, +) -> Result { + let mut depth = 0isize; + let mut index = open; + while index < sql.len() { + if let Some(end) = non_code_end(sql, index)? { + index = end; + continue; + } + match sql[index] { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Ok(index); + } + } + _ => {} + } + index += 1; + } + Err(Unclosed::Parenthesis) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/constants.rs b/crates/sqlbuild-rules-native/src/sql_scan/constants.rs new file mode 100644 index 0000000000..b9bec2ab1a --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/constants.rs @@ -0,0 +1,8 @@ +//! Lexical vocabulary shared by SQL scanning. + +/// Characters that may prefix a string literal, such as `E'...'` or `r'...'`. +pub(crate) const STRING_PREFIX_CHARACTERS: &[u8] = b"bBeErR"; +/// Longest string-literal prefix, such as `rb`. +pub(crate) const STRING_PREFIX_MAX_LENGTH: usize = 2; +/// Length of a triple-quote string delimiter. +pub(crate) const TRIPLE_QUOTE_LENGTH: usize = 3; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs new file mode 100644 index 0000000000..687a007d86 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs @@ -0,0 +1,13 @@ +//! Dialect-aware parenthesis matching. + +use crate::sql_scan::_helpers::dialect::matching_paren; +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; + +/// Return the index of the parenthesis closing the one at `open` under the dialect's rules. +pub(crate) fn dialect_matching_paren( + sql: &[u8], + open: usize, + syntax: &LexicalSyntax, +) -> Result { + matching_paren(sql, open, syntax) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs new file mode 100644 index 0000000000..321597f648 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs @@ -0,0 +1,9 @@ +//! Dialect-aware comment and quoted-text ranges of one SQL text. + +use crate::sql_scan::_helpers::dialect::non_code_ranges; +use crate::sql_scan::models::LexicalSyntax; + +/// Return every comment and quoted-text range in `sql`; an unclosed one extends to the end. +pub(crate) fn dialect_non_code_ranges(sql: &str, syntax: &LexicalSyntax) -> Vec<(usize, usize)> { + non_code_ranges(sql, syntax) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs index 89ee9b98e3..363e245e82 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs @@ -1,5 +1,6 @@ //! Quote- and comment-aware parenthesis matching. +use crate::sql_scan::_helpers::parens::matching_paren_with; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; @@ -9,24 +10,5 @@ pub(crate) fn matching_paren( open: usize, policy: QuotePolicy, ) -> Result { - let mut depth = 0isize; - let mut index = open; - while index < sql.len() { - if let Some(end) = non_code_end(sql, index, policy)? { - index = end; - continue; - } - match sql[index] { - b'(' => depth += 1, - b')' => { - depth -= 1; - if depth == 0 { - return Ok(index); - } - } - _ => {} - } - index += 1; - } - Err(Unclosed::Parenthesis) + matching_paren_with(sql, open, |sql, index| non_code_end(sql, index, policy)) } diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs index 4bf480ac79..2d272199c3 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod comment_end; +pub(crate) mod dialect_matching_paren; +pub(crate) mod dialect_non_code_ranges; pub(crate) mod matching_paren; pub(crate) mod non_code_end; pub(crate) mod quote_end; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs index bf57e1b0aa..8fbd9ba3d7 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs @@ -1,5 +1,6 @@ //! Boundaries of the comment or quoted text starting at one offset. +use crate::sql_scan::_helpers::dollar::dollar_quote_end; use crate::sql_scan::main::comment_end::comment_end; use crate::sql_scan::main::quote_end::quote_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; @@ -19,31 +20,3 @@ pub(crate) fn non_code_end( _ => Ok(None), } } - -/// Return the end of the `$$` or `$tag$` literal opening at `start`, if one opens there. -fn dollar_quote_end(sql: &[u8], start: usize) -> Result, Unclosed> { - if start - .checked_sub(1) - .is_some_and(|previous| continues_dollar_word(sql[previous])) - { - return Ok(None); - } - let tag_end = sql[start + 1..] - .iter() - .position(|byte| !(byte.is_ascii_alphanumeric() || *byte == b'_')) - .map_or(sql.len(), |offset| start + 1 + offset); - let tag = &sql[start + 1..tag_end]; - if sql.get(tag_end) != Some(&b'$') || tag.first().is_some_and(u8::is_ascii_digit) { - return Ok(None); - } - let delimiter = &sql[start..=tag_end]; - sql[tag_end + 1..] - .windows(delimiter.len()) - .position(|window| window == delimiter) - .map(|offset| Some(tag_end + 1 + offset + delimiter.len())) - .ok_or(Unclosed::Quote) -} - -fn continues_dollar_word(byte: u8) -> bool { - !byte.is_ascii() || byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') -} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/mod.rs index 650595395f..2ffdf440f3 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/mod.rs @@ -1,3 +1,5 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; pub(crate) mod main; pub(crate) mod models; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/models.rs b/crates/sqlbuild-rules-native/src/sql_scan/models.rs index ee967e3b9c..fa699b1e68 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/models.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/models.rs @@ -64,3 +64,30 @@ impl QuotePolicy { } } } + +/// One adapter's rules for where quoted text and comments end, mirroring Python `SqlLexicalSyntax`. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct LexicalSyntax { + /// Quote characters inside which a backslash escapes the next byte. + pub(crate) backslash_escape_quotes: Vec, + /// `E'...'` strings honour backslash escapes. + pub(crate) escape_string_prefix: bool, + /// `r'...'` strings disable backslash escapes. + pub(crate) raw_string_prefix: bool, + /// `'''...'''` and `"""..."""` delimit strings. + pub(crate) triple_quoted_strings: bool, + /// Block comments nest. + pub(crate) nested_block_comments: bool, + /// Prefixes that start a line comment, such as `--`, `#` or `//`. + pub(crate) line_comment_prefixes: Vec, +} + +impl LexicalSyntax { + /// Return whether a backslash escapes the next byte inside text opened by `quote`. + pub(crate) fn backslash_escapes(&self, quote: u8) -> bool { + self.backslash_escape_quotes + .iter() + .any(|value| value.as_bytes() == [quote]) + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests.rs index 0b2156d36c..ab9f7b24dc 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/tests.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests.rs @@ -1,3 +1,7 @@ +#[path = "tests/test_dialect_scan.rs"] +mod dialect_scan; +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_sql_scan.rs"] mod sql_scan; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs new file mode 100644 index 0000000000..3a9c34da1a --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs @@ -0,0 +1,43 @@ +//! Shared lexical syntaxes for SQL scanning tests. + +use crate::sql_scan::models::LexicalSyntax; + +pub(crate) fn generic() -> LexicalSyntax { + LexicalSyntax { + line_comment_prefixes: vec!["--".to_string()], + ..LexicalSyntax::default() + } +} + +pub(crate) fn single_quote_backslashes() -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: vec!["'".to_string()], + line_comment_prefixes: vec!["--".to_string(), "//".to_string()], + ..LexicalSyntax::default() + } +} + +pub(crate) fn escape_strings() -> LexicalSyntax { + LexicalSyntax { + escape_string_prefix: true, + nested_block_comments: true, + ..generic() + } +} + +pub(crate) fn triple_quotes() -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: vec!["'".to_string(), "\"".to_string()], + triple_quoted_strings: true, + line_comment_prefixes: vec!["--".to_string(), "#".to_string()], + ..LexicalSyntax::default() + } +} + +pub(crate) fn raw_strings() -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: vec!["'".to_string()], + raw_string_prefix: true, + ..generic() + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs new file mode 100644 index 0000000000..64291fbdd4 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs @@ -0,0 +1,119 @@ +use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use crate::sql_scan::tests::helpers::{ + escape_strings, generic, raw_strings, single_quote_backslashes, triple_quotes, +}; +use crate::sql_scan::tests::test_types::{ + DialectFirstNonCodeRangeTestCase, DialectNonCodeRangesTestCase, +}; + +#[test] +fn given_dialect_lexical_syntax_when_scanning_non_code_then_dialect_decides_boundaries() { + let test_cases = [ + DialectFirstNonCodeRangeTestCase { + description: "generic sql ends a quote at a backslash-escaped quote", + sql: r"'O\'Brien' x", + syntax: generic(), + expected_first_range: Some((0, 4)), + }, + DialectFirstNonCodeRangeTestCase { + description: "backslash escapes keep the quote open", + sql: r"'O\'Brien' x", + syntax: single_quote_backslashes(), + expected_first_range: Some((0, 10)), + }, + DialectFirstNonCodeRangeTestCase { + description: "escape strings honour backslashes", + sql: r"E'O\'Brien' x", + syntax: escape_strings(), + expected_first_range: Some((1, 11)), + }, + DialectFirstNonCodeRangeTestCase { + description: "plain strings keep backslashes literal under escape-string rules", + sql: r"'C:\' x", + syntax: escape_strings(), + expected_first_range: Some((0, 5)), + }, + DialectFirstNonCodeRangeTestCase { + description: "raw strings disable backslash escapes", + sql: r"r'C:\' x", + syntax: raw_strings(), + expected_first_range: Some((1, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "triple quotes contain single quotes", + sql: "'''it's''' x", + syntax: triple_quotes(), + expected_first_range: Some((0, 10)), + }, + DialectFirstNonCodeRangeTestCase { + description: "hash starts a line comment", + sql: "# note\nx", + syntax: triple_quotes(), + expected_first_range: Some((0, 7)), + }, + DialectFirstNonCodeRangeTestCase { + description: "slash pair starts a line comment", + sql: "// note\nx", + syntax: single_quote_backslashes(), + expected_first_range: Some((0, 8)), + }, + DialectFirstNonCodeRangeTestCase { + description: "nested block comments end at the outer close", + sql: "/* a /* b */ c */ x", + syntax: escape_strings(), + expected_first_range: Some((0, 17)), + }, + DialectFirstNonCodeRangeTestCase { + description: "flat block comments end at the first close", + sql: "/* a /* b */ c */ x", + syntax: generic(), + expected_first_range: Some((0, 12)), + }, + DialectFirstNonCodeRangeTestCase { + description: "an unclosed backslash-escaped quote extends to the end", + sql: r"'O\'", + syntax: single_quote_backslashes(), + expected_first_range: Some((0, 4)), + }, + ]; + for test_case in test_cases { + let ranges: Vec<(usize, usize)> = dialect_non_code_ranges(test_case.sql, &test_case.syntax); + assert_eq!( + ranges.first().copied(), + test_case.expected_first_range, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_quoted_text_before_a_marker_when_collecting_non_code_ranges_then_dialect_decides_coverage() +{ + let test_cases = [ + DialectNonCodeRangesTestCase { + description: "backslash escapes keep a later marker in code", + sql: r#"SELECT 'O\'Brien' AS name FROM __ref("orders")"#, + syntax: single_quote_backslashes(), + expected_marker_protected: false, + }, + DialectNonCodeRangesTestCase { + description: "generic sql reads a later marker as quoted text", + sql: r#"SELECT 'O\'Brien' AS name FROM __ref("orders")"#, + syntax: generic(), + expected_marker_protected: true, + }, + ]; + for test_case in test_cases { + let ranges: Vec<(usize, usize)> = dialect_non_code_ranges(test_case.sql, &test_case.syntax); + let marker: usize = test_case.sql.find("__ref").unwrap_or_default(); + let protected: bool = ranges + .iter() + .any(|(start, end)| marker >= *start && marker < *end); + assert_eq!( + protected, test_case.expected_marker_protected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs index 0951ec3e0e..ba72c5cbfb 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs @@ -14,3 +14,17 @@ pub(crate) struct NonCodeEndTestCase { pub(crate) policy: QuotePolicy, pub(crate) expected_end: Result, Unclosed>, } + +pub(crate) struct DialectFirstNonCodeRangeTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) syntax: crate::sql_scan::models::LexicalSyntax, + pub(crate) expected_first_range: Option<(usize, usize)>, +} + +pub(crate) struct DialectNonCodeRangesTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) syntax: crate::sql_scan::models::LexicalSyntax, + pub(crate) expected_marker_protected: bool, +} diff --git a/pyproject.toml b/pyproject.toml index 4b3587418c..96b517bf77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,7 @@ cache-keys = [ { file = "pyproject.toml" }, { file = "Cargo.toml" }, { file = "crates/sqlbuild-rules-native/Cargo.toml" }, + { file = "crates/sqlbuild-rules-native/build.rs" }, { file = "crates/sqlbuild-rules-native/src/**/*.rs" }, { file = "src/sqlbuild/_native.pyi" }, ] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 03525419f1..edf93ffdb7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -146,6 +146,11 @@ Returns a structured report including: - `lineage_mode` - which lineage mode was used - `artifacts` - paths to written files +If compile stops on an error before it can build the report, such as a model header that does not +parse, `--json` still prints one JSON document. It has `command`, `has_errors: true`, +`stopped: true`, and a `diagnostics` list with that error's `code`, `message`, and `help`, the same +text the error shows on stderr. The exit code is still 1, and progress stays on stderr. + ## Column lineage modes The `--lineage-mode` flag controls how column lineage is computed during compile: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index e7d71af19c..3dca51e3a8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -150,7 +150,7 @@ Table promotion mode is a project setting rather than a `MODEL()` field. Staged See [Incremental](../incremental.md) for full semantics. -Current compatibility behavior treats an unrecognized `on_schema_change` value as the default `append_new_columns` policy and an unrecognized `replay_on_change` value as `forward`. Use the documented values exactly; future validation may reject unknown values instead of applying these fallbacks. +Unknown `on_schema_change` or `replay_on_change` values fail compilation, in `MODEL()` headers, `[defaults]` and `[path_defaults]`. The error names the value and lists the valid ones. `bounded-` needs a positive duration such as `bounded-14d`. ## Snapshot fields diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md index 27d0688515..fa461cda58 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md @@ -28,7 +28,9 @@ normal compiler before treating the complete project as valid. Rules cache under `target/rules-cache`. A cache identity includes the Rule code, implementation and imported helpers, options, invocation subject, accessed compiler facts, dialect, tracked project -observations, and compatibility versions. +observations, and compatibility versions. The SQLBuild version in that identity also carries a hash of +the native engine's source, so a local development build never reuses results from a different build +of the same version. Model-subject Rules support fine-grained reuse. Editing one model invalidates affected model subjects rather than every Rule invocation. Project-subject Rules run once and are appropriate for diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index e91ebf9a39..8bd778c8c2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -101,6 +101,13 @@ The trailing `SELECT 1` is required as a ceremonial closing statement. Any CTE without one of these prefixes is treated as a helper CTE, available to mock, model, `__expected__`, and `__assert__` SQL in the test. +SQLBuild splits test files into CTEs and finds `__ref`, `__source`, `__seed`, `__dbt_ref`, +`__udf` and `__table_fn` calls outside comments and quoted text, in tests, scenarios, models, +functions, audits and hooks alike. Quoted text and comments follow the project adapter's rules: +backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, +nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, +and `//` line comments on Snowflake. + An `__expected__` CTE compares only the columns it lists, matched by name, so column order does not matter and unlisted model columns are ignored. Row counts are always compared. Listing a column the model does not output is a test error. When the expected CTE's columns are not explicit diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md index 3bdc163c2a..71cbda289c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -86,6 +86,7 @@ dlt_sources: bucket_url: "s3://my-bucket/events/" resources: - name: raw_events + reader: csv write_disposition: append ``` @@ -98,11 +99,14 @@ dlt_sources: | `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | | `table` | Source table to replicate (`sql_database` only). Required for that type. | | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | +| `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | | `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | | `primary_key` / `merge_key` | dlt keys for `merge`. | | `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | | `schema` | Per-resource schema override (falls back to the group `schema`). | +A `dlt_sources` entry accepts only `type`, `config`, `destination`, `schema`, and `resources`, and each resource accepts only the keys above for its source type. Any other key fails compilation with the nearest supported key as a hint. + Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_insert` is not available declaratively; use a Python loader for that. Config values support SQLBuild's [interpolation](../concepts/interpolation.md): `${name}` for a project variable, `${ENV:NAME}` for an environment variable, and `${CTX:...}` for context, resolved at load time, so credentials stay out of the YAML. ### Destination diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 51cd178294..118f3d2f5d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2,6 +2,8 @@ from typing import TypedDict +BUILD_IDENTITY: str + class LintPreparationRequest(TypedDict): expanded: str before_expansion: str diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 80ecaede2b..3451e6c91e 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -31,7 +31,11 @@ CompilerDiagnostic, RelatedLocation, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, DiagnosticSeverity +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries from sqlbuild.compiler.lineage.models import ProjectColumnLineage @@ -214,6 +218,27 @@ def format_compile_json( return json.dumps(result, indent=2) +def format_compile_error_json(*, code: str, message: str, help_text: str | None) -> str: + """Serialize a compile that stopped on a raised error as one JSON diagnostic report.""" + + diagnostic: CompilerDiagnostic = CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=code, + message=message, + help=help_text, + ) + result: dict[str, object] = { + "version": _sqlbuild_version(), + "command": "compile", + "offline": True, + "has_errors": True, + "stopped": True, + "diagnostics": [_diagnostic_to_json(diagnostic)], + } + return orjson.dumps(result, option=orjson.OPT_INDENT_2).decode() + + def _summary( *, graph: ProjectGraph, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py index 07fea40c29..5a8cab06bf 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py @@ -4,7 +4,10 @@ import sys from pathlib import Path -from sqlbuild.cli.commands._helpers.entry.errors import format_expected_error +from sqlbuild.cli.commands._helpers.entry.errors import ( + expected_error_parts, + format_expected_error, +) from sqlbuild.cli.commands._helpers.skills.update import maintain_sqlbuild_skills from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffExecutionError @@ -46,6 +49,7 @@ def dispatch_and_handle_errors( ) if output_error is not None: effective_error = output_error + _write_compile_machine_error(args=args, error=effective_error, fallback_code="C000") outcome_status: object | None = getattr(effective_error, "status", None) if outcome_status is not None: print(f"Query diff outcome {outcome_status}", file=sys.stderr) @@ -58,6 +62,7 @@ def dispatch_and_handle_errors( return 1 except LintError as error: logging.getLogger("sqlbuild.cli").exception("lint failed") + _write_compile_machine_error(args=args, error=error, fallback_code="L001") print( format_expected_error(error=error, fallback_code="L001", use_color=use_color), file=sys.stderr, @@ -67,6 +72,7 @@ def dispatch_and_handle_errors( if not isinstance(error, _expected_command_failure_types()): raise logging.getLogger("sqlbuild.cli").exception("command failed") + _write_compile_machine_error(args=args, error=error, fallback_code="E001") print( format_expected_error(error=error, fallback_code="E001", use_color=use_color), file=sys.stderr, @@ -118,6 +124,17 @@ def _report_skill_freshness(*, invocation: ParsedCliInvocation) -> None: print(result.message, file=sys.stderr, end="") +def _write_compile_machine_error( + *, args: CliNamespace, error: Exception, fallback_code: str +) -> None: + if args.command != CliCommand.COMPILE or not getattr(args, "json", False): + return + from sqlbuild.cli.commands._helpers.compile.output import format_compile_error_json + + code, message, help_text = expected_error_parts(error=error, fallback_code=fallback_code) + print(format_compile_error_json(code=code, message=message, help_text=help_text)) + + def _write_diff_machine_error( *, args: CliNamespace, error: CliUserError ) -> QueryDiffExecutionError | None: diff --git a/src/sqlbuild/cli/commands/_helpers/entry/errors.py b/src/sqlbuild/cli/commands/_helpers/entry/errors.py index d9ddbe03f3..e4e12a8537 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/errors.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/errors.py @@ -19,10 +19,15 @@ class ColorAwareSqlbuildArgumentParser(SqlbuildArgumentParser): return ColorAwareSqlbuildArgumentParser -def format_expected_error(*, error: Exception, fallback_code: str, use_color: bool) -> str: +def expected_error_parts(*, error: Exception, fallback_code: str) -> tuple[str, str, str | None]: code: str = str(getattr(error, "code", fallback_code)) message: str = str(getattr(error, "message", str(error))) help_text: str | None = getattr(error, "help", None) + return code, message, help_text + + +def format_expected_error(*, error: Exception, fallback_code: str, use_color: bool) -> str: + code, message, help_text = expected_error_parts(error=error, fallback_code=fallback_code) return ( format_coded_error(code=code, message=message, help=help_text, use_color=use_color) + "\n" ) diff --git a/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py b/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py new file mode 100644 index 0000000000..c87c9f902f --- /dev/null +++ b/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py @@ -0,0 +1,64 @@ +"""Validation of authored on_schema_change and replay_on_change values.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner.types import BackfillAction, OnSchemaChange +from sqlbuild.cursor_algebra.models import Duration +from sqlbuild.errors.setting_help.main.model_header_help import model_header_help +from sqlbuild.errors.setting_help.main.setting_help import setting_help + +_ON_SCHEMA_CHANGE_KEY: str = "on_schema_change" +_REPLAY_ON_CHANGE_KEY: str = "replay_on_change" +_BOUNDED_PREFIX: str = f"{BackfillAction.BOUNDED}-" +_VALID_VALUES: dict[str, tuple[str, ...]] = { + _ON_SCHEMA_CHANGE_KEY: tuple(sorted(policy.value for policy in OnSchemaChange)), + _REPLAY_ON_CHANGE_KEY: ( + BackfillAction.FORWARD_ONLY.value, + BackfillAction.FULL.value, + f"{_BOUNDED_PREFIX}", + ), +} +_EXAMPLE_VALUES: dict[str, str] = { + _ON_SCHEMA_CHANGE_KEY: OnSchemaChange.APPEND_NEW_COLUMNS.value, + _REPLAY_ON_CHANGE_KEY: f"{_BOUNDED_PREFIX}14d", +} + + +def change_policy_problem_impl(*, key: str, value: object) -> str | None: + """Describe why a change-policy value is invalid, or return None when it is valid.""" + + valid: str = ", ".join(_VALID_VALUES[key]) + if not isinstance(value, str): + return f"{key} must be a string; valid values: {valid}" + if key == _ON_SCHEMA_CHANGE_KEY and value in _VALID_VALUES[key]: + return None + if key == _REPLAY_ON_CHANGE_KEY: + if value in (BackfillAction.FORWARD_ONLY, BackfillAction.FULL): + return None + if value.startswith(_BOUNDED_PREFIX): + duration: str = value.removeprefix(_BOUNDED_PREFIX).strip() + if Duration.parse(duration) is not None: + return None + return ( + f"{key} '{value}' has an invalid duration '{duration}'; use a positive duration " + "such as 14d, 12h or 1mo" + ) + return f"unknown {key} '{value}'; valid values: {valid}" + + +def change_policy_header_help_impl(*, key: str) -> str: + """Show a valid MODEL header entry for a change-policy key.""" + + return model_header_help(purpose=f"use a valid {key}", entry=f"{key} {_EXAMPLE_VALUES[key]}") + + +def change_policy_toml_help_impl(*, key: str, file_name: str, section: str) -> str: + """Show a valid TOML setting line for a change-policy key.""" + + return setting_help( + purpose=f"use a valid {key}", + file_name=file_name, + section=section, + key=key, + value=_EXAMPLE_VALUES[key], + ) diff --git a/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py b/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py new file mode 100644 index 0000000000..9d40a3bc96 --- /dev/null +++ b/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py @@ -0,0 +1,13 @@ +"""Public MODEL-header help entrypoint for change-policy values.""" + +from __future__ import annotations + +from sqlbuild.compiler.authored_values._helpers.change_policy import ( + change_policy_header_help_impl, +) + + +def change_policy_header_help(*, key: str) -> str: + """Show a valid MODEL header entry for on_schema_change or replay_on_change.""" + + return change_policy_header_help_impl(key=key) diff --git a/src/sqlbuild/compiler/authored_values/main/_change_policy_problem.py b/src/sqlbuild/compiler/authored_values/main/_change_policy_problem.py new file mode 100644 index 0000000000..773eb94c2c --- /dev/null +++ b/src/sqlbuild/compiler/authored_values/main/_change_policy_problem.py @@ -0,0 +1,11 @@ +"""Public change-policy value validation entrypoint.""" + +from __future__ import annotations + +from sqlbuild.compiler.authored_values._helpers.change_policy import change_policy_problem_impl + + +def change_policy_problem(*, key: str, value: object) -> str | None: + """Describe why an on_schema_change or replay_on_change value is invalid, or return None.""" + + return change_policy_problem_impl(key=key, value=value) diff --git a/src/sqlbuild/compiler/authored_values/main/_change_policy_toml_help.py b/src/sqlbuild/compiler/authored_values/main/_change_policy_toml_help.py new file mode 100644 index 0000000000..814249ae11 --- /dev/null +++ b/src/sqlbuild/compiler/authored_values/main/_change_policy_toml_help.py @@ -0,0 +1,11 @@ +"""Public TOML help entrypoint for change-policy values.""" + +from __future__ import annotations + +from sqlbuild.compiler.authored_values._helpers.change_policy import change_policy_toml_help_impl + + +def change_policy_toml_help(*, key: str, file_name: str, section: str) -> str: + """Show a valid TOML setting line for on_schema_change or replay_on_change.""" + + return change_policy_toml_help_impl(key=key, file_name=file_name, section=section) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 063daf4ac3..bd2634be06 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -97,7 +97,6 @@ ) from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile._helpers.sharing.binding import shareable_binding_prekeys -from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_assertion_target_model_names from sqlbuild.compiler.compile._helpers.sql_tests.identity import ( build_sql_test_case_fingerprint, ) @@ -375,6 +374,7 @@ def assemble_compiled_project( effective_target_database=_str_or_none(effective_target_values.get("database")), effective_target_schema=_str_or_none(effective_target_values.get("schema")), sql_analysis_dialect=profile.sql_analysis_dialect, + sql_lexical_syntax=inputs.sql_lexical_syntax, compile_cache_dir=inputs.compile_cache_dir, settings=inputs.effective_settings, scenario=resolve_effective_scenario_config( @@ -1737,6 +1737,7 @@ def _assemble_compiled_sql_test( test_name: str = _resolve_test_name(test_input) compiled_payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload scope_deps: tuple[CompiledObjectKey, ...] + target_model_names: tuple[str, ...] = () if isinstance(test_input.payload, CompileDirectLogicSqlTestInputPayload): if test_input.payload.mode == SqlTestMode.MACRO: scope_deps = _macro_sql_test_scope_deps( @@ -1761,14 +1762,12 @@ def _assemble_compiled_sql_test( ) else: model_payload: CompileModelSqlTestInputPayload = test_input.payload - assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in model_payload.assertion_ctes) - ) - scope_deps = sql_test_scope_deps( - expected_model_names=tuple( - dict.fromkeys((*model_payload.expected_model_names, *assertion_target_model_names)) + target_model_names = tuple( + dict.fromkeys( + (*model_payload.expected_model_names, *model_payload.assertion_target_model_names) ) ) + scope_deps = sql_test_scope_deps(expected_model_names=target_model_names) compiled_payload = CompiledModelSqlTestPayload( authored_ctes=model_payload.authored_ctes, macro_mocks=model_payload.macro_mocks, @@ -1839,19 +1838,11 @@ def _assemble_compiled_sql_test( else () ), assertion_target_model_names=( - assertion_target_model_names - if isinstance(test_input.payload, CompileModelSqlTestInputPayload) - else () - ), - target_model_names=( - tuple( - dict.fromkeys( - (*test_input.payload.expected_model_names, *assertion_target_model_names) - ) - ) + test_input.payload.assertion_target_model_names if isinstance(test_input.payload, CompileModelSqlTestInputPayload) else () ), + target_model_names=target_model_names, tested_resources=tested_resources, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index bd184f1235..32fc8f463c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -50,6 +50,7 @@ DeclarationExpansionContext, LoadedMacro, MacroContext, + ModelInputBuildContext, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, @@ -69,6 +70,7 @@ UsageRecord, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( SchemaAuditInstance, SchemaColumn, @@ -101,6 +103,7 @@ class _AuditAttachmentContext: effective_vars: dict[str, object] macro_context: MacroContext declaration_expansion: DeclarationExpansionContext + sql_lexical_syntax: SqlLexicalSyntax scoped_declarations: dict[tuple[Path, DeclarationIdentity], DeclarationExpansionContext] = ( field(default_factory=dict, compare=False, repr=False) ) @@ -119,19 +122,21 @@ def remember_scoped_declarations( def build_audit_inputs( *, discovered_inputs: DiscoveredProjectInputs, - effective_settings: SettingsConfig, + context: ModelInputBuildContext, model_inputs: tuple[CompileModelInput, ...], source_inputs: tuple[CompileSourceInput, ...], - effective_vars: dict[str, object], - macro_context: MacroContext, - loaded_macros: dict[str, LoadedMacro], - declaration_expansion: DeclarationExpansionContext, generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] | None = None, seed_inputs: tuple[CompileSeedInput, ...] = (), ) -> tuple[CompileAuditInput, ...]: """Build compile-time audit inputs from discovered SQL audit blocks.""" + effective_settings: SettingsConfig = context.effective_settings + effective_vars: dict[str, object] = context.effective_vars + macro_context: MacroContext = context.macro_context + loaded_macros: dict[str, LoadedMacro] = context.loaded_macros + declaration_expansion: DeclarationExpansionContext = context.declaration_expansion + sql_lexical_syntax: SqlLexicalSyntax = context.sql_lexical_syntax known_model_names: set[str] = build_known_ref_names(discovered_inputs) known_seed_names: set[str] = build_known_seed_names(discovered_inputs) known_source_names: set[str] = build_known_source_names(discovered_inputs) @@ -150,6 +155,7 @@ def build_audit_inputs( effective_vars=effective_vars, macro_context=macro_context, declaration_expansion=declaration_expansion, + sql_lexical_syntax=sql_lexical_syntax, ) audit_inputs: list[CompileAuditInput] = [] audit_file: DiscoveredAuditFile @@ -204,7 +210,9 @@ def build_audit_inputs( context=f"Audit '{audit_block.name or audit_file.file_path.stem}' evidence", ) references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=_combined_references(expanded_sql_body, expanded_evidence_sql), + references=_combined_references( + expanded_sql_body, expanded_evidence_sql, syntax=sql_lexical_syntax + ), argument_references=_argument_references(expansion, evidence_expansion), ) validate_audit_references( @@ -629,7 +637,9 @@ def build_attached_audit_input( context=f"Audit '{audit_instance.definition_name}' evidence", ) references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=_combined_references(expanded_sql_body, expanded_evidence_sql), + references=_combined_references( + expanded_sql_body, expanded_evidence_sql, syntax=context.sql_lexical_syntax + ), argument_references=_argument_references(expansion, evidence_expansion), ) validate_audit_references( @@ -951,11 +961,13 @@ def _argument_references( return tuple(references) -def _combined_references(*sql_values: str | None) -> tuple[CompileSqlReference, ...]: +def _combined_references( + *sql_values: str | None, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlReference, ...]: """Extract references from independently compiled measurement/evidence queries.""" references: list[CompileSqlReference] = [] for sql in sql_values: if sql is not None: - references.extend(extract_sql_references(sql)) + references.extend(extract_sql_references(sql=sql, syntax=syntax)) return tuple(dict.fromkeys(references)) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 57ac4780ff..c2b273a7ae 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -153,6 +153,7 @@ UsageKind, VisibilityReason, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.spec.contracts.models import ( DefaultsConfig, @@ -261,6 +262,7 @@ class _HookExpansionContext: sql_hook_definitions: dict[str, DiscoveredSqlHookFile] consumer: ResourceIdentity | DeclarationIdentity facts: _HookExpansionFacts + sql_lexical_syntax: SqlLexicalSyntax @dataclass @@ -398,7 +400,9 @@ def build_model_inputs( legacy_schema_files: tuple[DiscoveredSchemaFile, ...] = tuple( schema_file for schema_file in discovered_inputs.schema_files if schema_file.model_entries ) - with cached_sql_reference_extractor(root=reference_cache_dir) as extract_references: + with cached_sql_reference_extractor( + root=reference_cache_dir, syntax=context.sql_lexical_syntax + ) as extract_references: return _build_model_inputs( discovered_inputs=discovered_inputs, context=context, @@ -629,6 +633,7 @@ def _build_model_inputs( collection_rendering=context.collection_rendering, resolver=context.declaration_resolver, ), + sql_lexical_syntax=context.sql_lexical_syntax, sql_hook_definitions=sql_hook_definitions, consumer=model_identity, ) @@ -1233,6 +1238,7 @@ def expand_model_hook_macros_result( loaded_macros: dict[str, LoadedMacro], macro_context: MacroContext, declaration_expansion: DeclarationExpansionContext, + sql_lexical_syntax: SqlLexicalSyntax, sql_hook_definitions: dict[str, DiscoveredSqlHookFile] | None = None, consumer: ResourceIdentity | None = None, ) -> HookExpansionResult: @@ -1257,6 +1263,7 @@ def expand_model_hook_macros_result( sql_hook_definitions=sql_hook_definitions or {}, consumer=consumer or ResourceIdentity(ResourceKind.MODEL, file_path.stem), facts=facts, + sql_lexical_syntax=sql_lexical_syntax, ), hook_key=hook_key, ) @@ -1494,7 +1501,9 @@ def expand_sql_macros_in_value( facts.add(expansion.usages) facts.add_references( merge_call_site_references( - references=extract_sql_references(expansion.sql), + references=extract_sql_references( + sql=expansion.sql, syntax=context.sql_lexical_syntax + ), argument_references=expansion.argument_references, ) ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index ff3c048668..53ffdb65c0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -26,12 +26,14 @@ from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax def build_declaration_scope( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: """Build one canonical index and validate it before SQL expansion.""" @@ -53,6 +55,7 @@ def build_declaration_scope( build_scope_relationship_grants( discovered_inputs=discovered_inputs, index=index, + sql_lexical_syntax=sql_lexical_syntax, compile_cache_dir=compile_cache_dir, ) if has_scoped_relationship_declarations diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index 309c614fc8..014b743b52 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -58,6 +58,7 @@ from sqlbuild.compiler.scopes.models import ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot import import_polyglot +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( DefaultsConfig, SettingsConfig, @@ -87,15 +88,16 @@ def build_sql_function_inputs( effective_vars: dict[str, object], effective_settings: SettingsConfig, target_config: TargetConfig | None, - adapter_name: str, macro_context: MacroContext, loaded_macros: dict[str, LoadedMacro], declaration_expansion: DeclarationExpansionContext, + sql_lexical_syntax: SqlLexicalSyntax, no_sql_validation: bool = False, python_functions_inherit_default_namespace: bool = True, ) -> tuple[CompileSqlFunctionInput, ...]: """Attach and validate SQL function metadata.""" + adapter_name: str = macro_context.adapter_name known_model_names: set[str] = build_known_ref_names(discovered_inputs) known_seed_names: set[str] = build_known_seed_names(discovered_inputs) known_source_names: set[str] = build_known_source_names(discovered_inputs) @@ -226,7 +228,7 @@ def build_sql_function_inputs( function_file=function_file, ) references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=extract_sql_references(expanded_body_sql), + references=extract_sql_references(sql=expanded_body_sql, syntax=sql_lexical_syntax), argument_references=expansion.argument_references, ) validate_function_references( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index 32818872f0..1593bb20b8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -45,6 +45,7 @@ VisibilityResolution, ) from sqlbuild.compiler.scopes.types import DeclarationKind, GrantKind, ResourceKind, ScopeKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax type _SharedDeclarations = dict[tuple[str, str], tuple[DeclarationRecord, ...]] @@ -53,6 +54,7 @@ def build_scope_relationship_grants( *, discovered_inputs: DiscoveredProjectInputs, index: ScopeIndex, + sql_lexical_syntax: SqlLexicalSyntax, compile_cache_dir: Path | None = None, ) -> ScopeRelationshipBuild: """Return expected-model grants while retaining independent extraction faults.""" @@ -69,11 +71,13 @@ def build_scope_relationship_grants( lookup=lookup, shared_declarations=shared_declarations, fact_cache=fact_cache, + syntax=sql_lexical_syntax, ) scenario_grants, scenario_faults = _scenario_relationship_grants( discovered_inputs=discovered_inputs, lookup=lookup, shared_declarations=shared_declarations, + syntax=sql_lexical_syntax, ) return ScopeRelationshipBuild( grants=tuple(dict.fromkeys((*test_grants, *scenario_grants))), @@ -87,12 +91,15 @@ def _test_relationship_grants( lookup: ScopeLookup, shared_declarations: _SharedDeclarations, fact_cache: FactCacheStore, + syntax: SqlLexicalSyntax, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] cache_keys: dict[int, str] = ( { - file_index: _expected_models_fact_key(test_file=test_file, fact_cache=fact_cache) + file_index: _expected_models_fact_key( + test_file=test_file, fact_cache=fact_cache, syntax=syntax + ) for file_index, test_file in enumerate(discovered_inputs.test_files) } if fact_cache.enabled @@ -119,6 +126,7 @@ def _test_relationship_grants( else extract_sql_test_expected_model_names( sql=block.sql_body, file_label=str(test_file.relative_path), + syntax=syntax, mode=block.mode, ) ) @@ -143,6 +151,7 @@ def _test_relationship_grants( ), sql=block.sql_body, file_label=str(test_file.relative_path), + syntax=syntax, ) ) except Exception as error: @@ -161,9 +170,9 @@ def _test_relationship_grants( def _expected_models_fact_key( - *, test_file: DiscoveredSqlTestFile, fact_cache: FactCacheStore + *, test_file: DiscoveredSqlTestFile, fact_cache: FactCacheStore, syntax: SqlLexicalSyntax ) -> str: - parts: list[str] = [str(test_file.relative_path)] + parts: list[str] = [str(test_file.relative_path), syntax.cache_key] for block in test_file.blocks: parts.extend((block.sql_body, block.mode.value)) return fact_cache.key(*parts) @@ -189,13 +198,14 @@ def _scenario_relationship_grants( discovered_inputs: DiscoveredProjectInputs, lookup: ScopeLookup, shared_declarations: _SharedDeclarations, + syntax: SqlLexicalSyntax, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] for scenario in discovered_inputs.scenario_files: try: expected_names: tuple[str, ...] = extract_sql_scenario_expected_model_names( - sql=scenario.sql_body, file_label=str(scenario.relative_path) + sql=scenario.sql_body, file_label=str(scenario.relative_path), syntax=syntax ) grants.extend( _expected_model_grants( @@ -288,10 +298,15 @@ def _resolve_shared_declarations( def _tested_macro_grants( - *, lookup: ScopeLookup, resource: ResourceIdentity, sql: str, file_label: str + *, + lookup: ScopeLookup, + resource: ResourceIdentity, + sql: str, + file_label: str, + syntax: SqlLexicalSyntax, ) -> list[GrantRecord]: test_ctes: tuple[CompileSqlTestCte, ...] = extract_unclassified_sql_test_ctes( - sql=sql, file_label=file_label + sql=sql, file_label=file_label, syntax=syntax ) actual_cte: CompileSqlTestCte | None = next( (cte for cte in test_ctes if cte.name == MACRO_ACTUAL_TEST_CTE_NAME), None diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index 6482191a90..f7dc043b2d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -76,6 +76,7 @@ VisibilityRecord, ) from sqlbuild.compiler.scopes.types import ResourceKind, ScopeKind, UsageKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @dataclass(frozen=True) @@ -100,6 +101,7 @@ def build_test_inputs_with_cache( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...], compile_cache_dir: Path | None, + sql_lexical_syntax: SqlLexicalSyntax, ) -> tuple[CompileSqlTestInput, ...]: """Build SQL test inputs inside the compile timing boundary.""" @@ -113,6 +115,7 @@ def build_test_inputs_with_cache( external_sql_reference_resolver=external_sql_reference_resolver, sql_function_inputs=sql_function_inputs, compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=sql_lexical_syntax, ) @@ -123,6 +126,7 @@ def build_test_inputs( macro_context: MacroContext, loaded_macros: dict[str, LoadedMacro], declaration_expansion: DeclarationExpansionContext, + sql_lexical_syntax: SqlLexicalSyntax, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = (), compile_cache_dir: Path | None = None, @@ -230,6 +234,7 @@ def build_test_inputs( test_block=expanded_test_block, test_file=test_file, test_mode=test_mode, + syntax=sql_lexical_syntax, ) tested_resource_names = _infer_tested_direct_logic_resource_names( raw_test_ctes=raw_test_ctes, @@ -238,6 +243,7 @@ def build_test_inputs( known_function_names=known_function_names, known_table_function_names=known_table_function_names, table_function_argument_counts=table_function_argument_counts, + syntax=sql_lexical_syntax, ) expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( sql=parameter_sql, @@ -284,8 +290,17 @@ def build_test_inputs( (test.sql_body, str(test.test_file.relative_path), test.mode) for test in expanded_tests ), cache_root=compile_cache_dir, + syntax=sql_lexical_syntax, ) for test, test_ctes in zip(expanded_tests, test_ctes_batch, strict=True): + assertion_target_model_names: tuple[str, ...] = ( + extract_assertion_target_model_names( + assertion_sql=tuple(cte.sql_body for cte in test_ctes.payload.assertion_ctes), + syntax=sql_lexical_syntax, + ) + if isinstance(test_ctes.payload, CompileModelSqlTestCtes) + else () + ) validate_test_ctes( test_ctes=test_ctes, test_file=test.test_file, @@ -294,6 +309,7 @@ def build_test_inputs( known_source_names=known_source_names, known_table_function_names=known_table_function_names, loaded_macros=loaded_macros, + assertion_target_model_names=assertion_target_model_names, ) test_inputs.append( CompileSqlTestInput( @@ -304,6 +320,7 @@ def build_test_inputs( payload=_build_test_input_payload( test_ctes=test_ctes, tested_resource_names=test.tested_resource_names, + assertion_target_model_names=assertion_target_model_names, ), declaration_usages=test.declaration_usages, parent_name=test.parent_name, @@ -361,7 +378,10 @@ def _macro_test_declaration_usages( def _build_test_input_payload( - *, test_ctes: CompileSqlTestCtes, tested_resource_names: tuple[str, ...] + *, + test_ctes: CompileSqlTestCtes, + tested_resource_names: tuple[str, ...], + assertion_target_model_names: tuple[str, ...], ) -> CompileModelSqlTestInputPayload | CompileDirectLogicSqlTestInputPayload: match test_ctes.payload: case CompileModelSqlTestCtes() as model_payload: @@ -377,6 +397,7 @@ def _build_test_input_payload( expected_model_names=model_payload.expected_model_names, assertion_ctes=model_payload.assertion_ctes, assertion_names=model_payload.assertion_names, + assertion_target_model_names=assertion_target_model_names, ) case CompileDirectLogicSqlTestCtes() as direct_logic_payload: return CompileDirectLogicSqlTestInputPayload( @@ -393,10 +414,12 @@ def _validate_raw_direct_logic_test_ctes( test_block: DiscoveredSqlTestBlock, test_file: DiscoveredSqlTestFile, test_mode: SqlTestMode, + syntax: SqlLexicalSyntax, ) -> CompileSqlTestCtes: return extract_sql_test_ctes( sql=test_block.sql_body, file_label=str(test_file.relative_path), + syntax=syntax, mode=test_mode, ) @@ -411,6 +434,7 @@ def _infer_tested_direct_logic_resource_names( known_function_names: set[str], known_table_function_names: set[str], table_function_argument_counts: dict[str, int], + syntax: SqlLexicalSyntax, ) -> tuple[str, ...]: if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( @@ -423,6 +447,7 @@ def _infer_tested_direct_logic_resource_names( test_file=test_file, known_function_names=known_function_names, known_table_function_names=known_table_function_names, + syntax=syntax, ) if raw_test_ctes.mode == SqlTestMode.TABLE_FN: return _infer_tested_table_function_names( @@ -431,6 +456,7 @@ def _infer_tested_direct_logic_resource_names( known_function_names=known_function_names, known_table_function_names=known_table_function_names, table_function_argument_counts=table_function_argument_counts, + syntax=syntax, ) tested_macro_names: tuple[str, ...] = find_macro_call_names( raw_test_ctes.payload.actual_cte.sql_body @@ -456,6 +482,7 @@ def _infer_tested_udf_names( test_file: DiscoveredSqlTestFile, known_function_names: set[str], known_table_function_names: set[str], + syntax: SqlLexicalSyntax, ) -> tuple[str, ...]: if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( @@ -463,7 +490,7 @@ def _infer_tested_udf_names( "__udf_actual__ CTE and exactly one __udf_expected__ CTE" ) references: tuple[CompileSqlReference, ...] = extract_sql_references( - raw_test_ctes.payload.actual_cte.sql_body + sql=raw_test_ctes.payload.actual_cte.sql_body, syntax=syntax ) tested_udf_names: tuple[str, ...] = tuple( dict.fromkeys( @@ -500,6 +527,7 @@ def _infer_tested_table_function_names( known_function_names: set[str], known_table_function_names: set[str], table_function_argument_counts: dict[str, int], + syntax: SqlLexicalSyntax, ) -> tuple[str, ...]: if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( @@ -507,7 +535,7 @@ def _infer_tested_table_function_names( "__table_fn_actual__ CTE and exactly one __table_fn_expected__ CTE" ) references: tuple[CompileSqlReference, ...] = extract_sql_references( - raw_test_ctes.payload.actual_cte.sql_body + sql=raw_test_ctes.payload.actual_cte.sql_body, syntax=syntax ) validate_table_function_reference_arities( references=references, @@ -550,6 +578,7 @@ def build_scenario_inputs( macro_context: MacroContext, loaded_macros: dict[str, LoadedMacro], declaration_expansion: DeclarationExpansionContext, + sql_lexical_syntax: SqlLexicalSyntax, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, ) -> tuple[CompileSqlScenarioInput, ...]: """Build compile-time scenario inputs from discovered SQL-native scenario files.""" @@ -590,14 +619,17 @@ def build_scenario_inputs( scenario_ctes: CompileSqlScenarioCtes = extract_sql_scenario_ctes( sql=expanded_sql_body, file_label=str(scenario_file.relative_path), + syntax=sql_lexical_syntax, ) _validate_scenario_source_references( scenario_ctes=scenario_ctes, scenario_file=scenario_file, known_source_names=known_source_names, + syntax=sql_lexical_syntax, ) assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in scenario_ctes.assertion_ctes) + assertion_sql=tuple(cte.sql_body for cte in scenario_ctes.assertion_ctes), + syntax=sql_lexical_syntax, ) scenario_inputs.append( CompileSqlScenarioInput( @@ -629,10 +661,13 @@ def _validate_scenario_source_references( scenario_ctes: CompileSqlScenarioCtes, scenario_file: DiscoveredSqlScenarioFile, known_source_names: set[str], + syntax: SqlLexicalSyntax, ) -> None: cte: CompileSqlScenarioCte for cte in (*scenario_ctes.expected_ctes, *scenario_ctes.assertion_ctes): - references: tuple[CompileSqlReference, ...] = extract_sql_references(cte.sql_body) + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=cte.sql_body, syntax=syntax + ) reference: CompileSqlReference for reference in references: if reference.ref_kind != SqlReferenceKind.SOURCE: @@ -645,7 +680,7 @@ def _validate_scenario_source_references( ) for cte in scenario_ctes.authored_ctes: - references = extract_sql_references(cte.sql_body) + references = extract_sql_references(sql=cte.sql_body, syntax=syntax) for reference in references: if reference.ref_kind != SqlReferenceKind.SOURCE: continue @@ -666,6 +701,7 @@ def validate_test_ctes( known_source_names: set[str], known_table_function_names: set[str], loaded_macros: dict[str, LoadedMacro], + assertion_target_model_names: tuple[str, ...], ) -> None: """Validate SQL-native test CTE targets against discovered inputs.""" @@ -714,9 +750,7 @@ def validate_test_ctes( ) assertion_target_name: str - for assertion_target_name in extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in model_payload.assertion_ctes) - ): + for assertion_target_name in assertion_target_model_names: if assertion_target_name not in known_model_names: raise CompileInputError( f"SQL test file {test_file.relative_path} assertion references unknown model " diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index 8020d1a704..b8eae7c29b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -7,6 +7,10 @@ from decimal import Decimal, InvalidOperation from typing import cast +from sqlbuild.compiler.authored_values.main._change_policy_header_help import ( + change_policy_header_help, +) +from sqlbuild.compiler.authored_values.main._change_policy_problem import change_policy_problem from sqlbuild.compiler.compile.constants import ( MICROBATCH_LIMIT_ACTION_KEY, MICROBATCH_LIMIT_MAX_BATCHES_KEY, @@ -160,7 +164,6 @@ class _IncrementalConfigValues: max_microbatches: object | None microbatch_limit: object | None cursor_grain: str | None - replay_on_change: object | None merge_exclude_columns: object | None full_refresh: object | None @@ -198,7 +201,7 @@ def validate_incremental_config( return values: _IncrementalConfigValues = _resolve_incremental_config_values(config=config) - _validate_incremental_core(model_name=model_name, values=values) + _validate_incremental_core(config=config, model_name=model_name, values=values) _validate_incremental_batching( model_name=model_name, ref_count=ref_count, @@ -218,9 +221,18 @@ def validate_incremental_config( ) -def _validate_incremental_core(*, model_name: str, values: _IncrementalConfigValues) -> None: - if values.replay_on_change is not None and not isinstance(values.replay_on_change, str): - raise CompileInputError(f"model '{model_name}': replay_on_change must be a string") +def _validate_incremental_core( + *, config: CompileModelConfig, model_name: str, values: _IncrementalConfigValues +) -> None: + key: str + for key in ("on_schema_change", "replay_on_change"): + value: object | None = config.values.get(key) + problem: str | None = None if value is None else change_policy_problem(key=key, value=value) + if problem is not None: + raise CompileInputError( + f"model '{model_name}': {problem}", + help=change_policy_header_help(key=key), + ) if values.strategy is None: raise CompileInputError( f"model '{model_name}': incremental materialization requires incremental_strategy" @@ -405,7 +417,6 @@ def _resolve_incremental_config_values(*, config: CompileModelConfig) -> _Increm max_microbatches=config.values.get("max_microbatches"), microbatch_limit=config.values.get("microbatch_limit"), cursor_grain=get_config_str(values=config.values, key="cursor_grain"), - replay_on_change=config.values.get("replay_on_change"), merge_exclude_columns=config.values.get("merge_exclude_columns"), full_refresh=config.values.get("full_refresh"), ) diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py index dfa069eb6c..6965afd73c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py @@ -22,6 +22,7 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -49,6 +50,8 @@ rf"{_RELATION_PLACEHOLDER_PREFIX}(\d+)__" ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + def relation_placeholder_text(index: int) -> str: """Return the in-expansion placeholder text for the typed reference at ``index``.""" @@ -80,7 +83,9 @@ def call_site_sql_references( if not refs: return () - return extract_sql_references(" ".join(reference_call_text(ref) for ref in refs)) + return extract_sql_references( + sql=" ".join(reference_call_text(ref) for ref in refs), syntax=_GENERIC_SQL_SYNTAX + ) def resource_references( @@ -165,7 +170,9 @@ def reject_macro_generated_references( def _generated_references(sql: str) -> tuple[SqlResourceRef, ...]: try: - references: tuple[CompileSqlReference, ...] = extract_sql_references(sql) + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=sql, syntax=_GENERIC_SQL_SYNTAX + ) except CompileInputError: return () generated: dict[SqlResourceRef, None] = {} diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/cache.py b/src/sqlbuild/compiler/compile/_helpers/refs/cache.py index 29c10e8a3f..c6241d5e72 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/cache.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/cache.py @@ -16,6 +16,7 @@ from sqlbuild.compiler.compile.exceptions import AnalysisCacheEntryError from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _REFERENCE_CACHE_VERSION: int = 2 _CACHE_DATABASE_NAME: str = "sql-references.sqlite3" @@ -33,8 +34,10 @@ class _SqlReferenceCache: """Reuse exact SQL reference facts while preserving safe scanner fallback.""" - def __init__(self, *, root: Path | None) -> None: + def __init__(self, *, root: Path | None, syntax: SqlLexicalSyntax) -> None: self._root: Path | None = root + self._syntax: SqlLexicalSyntax = syntax + self._syntax_key: str = syntax.cache_key self._database_path: Path | None = None self._connection: sqlite3.Connection | None = None self._pending_contents_by_key: dict[str, str] = {} @@ -83,7 +86,7 @@ def __exit__( def references(self, sql: str) -> tuple[CompileSqlReference, ...]: """Return cached references or scan and record this exact expanded SQL.""" - cache_key: str = _reference_cache_key(sql) + cache_key: str = _reference_cache_key(sql=sql, syntax_key=self._syntax_key) pending_contents: str | None = self._pending_contents_by_key.get(cache_key) if pending_contents is not None: pending_references: tuple[CompileSqlReference, ...] | None = _references_from_contents( @@ -109,7 +112,9 @@ def references(self, sql: str) -> tuple[CompileSqlReference, ...]: self._disable() connection = None - references: tuple[CompileSqlReference, ...] = extract_sql_references(sql) + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=sql, syntax=self._syntax + ) if self._database_path is not None: try: contents: str = _reference_contents( @@ -150,8 +155,11 @@ def _disable(self) -> None: pass -def _reference_cache_key(sql: str) -> str: - return hashlib.sha256(sql.encode()).hexdigest() +def _reference_cache_key(*, sql: str, syntax_key: str) -> str: + digest: Any = hashlib.sha256(syntax_key.encode()) + digest.update(b"\0") + digest.update(sql.encode()) + return digest.hexdigest() def _reference_contents(*, cache_key: str, references: tuple[CompileSqlReference, ...]) -> str: @@ -247,9 +255,9 @@ def _cache_entry_digest(*, cache_key: str, serialized_payload: str) -> str: @contextmanager def cached_sql_reference_extractor( - *, root: Path | None + *, root: Path | None, syntax: SqlLexicalSyntax ) -> Iterator[Callable[[str], tuple[CompileSqlReference, ...]]]: - """Yield an exact cached reference extractor for one compile invocation.""" + """Yield an exact cached reference extractor for one compile invocation and dialect.""" - with _SqlReferenceCache(root=root) as cache: + with _SqlReferenceCache(root=root, syntax=syntax) as cache: yield cache.references diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/references.py b/src/sqlbuild/compiler/compile/_helpers/refs/references.py index 255f6b19c9..9671450f92 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/references.py @@ -16,9 +16,8 @@ from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main._find_matching_paren import find_matching_paren -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import skip_quoted_text +from sqlbuild.compiler.sql_analysis.main._skip_dialect_non_code import dialect_non_code_end +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _CONTEXT: str = "SQL reference" _DOUBLE_QUOTE_TOKEN: str = '"' @@ -34,12 +33,16 @@ _REFERENCE_PREFIX_BY_KIND: dict[SqlReferenceKind, str] = { ref_kind: prefix[:-1] for prefix, ref_kind in _REFERENCE_PREFIXES } -_REFERENCE_SCAN_PATTERN: re.Pattern[str] = re.compile(r"__|--|/\*|'|\"|`|\$") +_REFERENCE_SCAN_PATTERN: re.Pattern[str] = re.compile(r"__|--|/\*|//|#|'|\"|`|\$") -def extract_sql_references(sql: str) -> tuple[CompileSqlReference, ...]: - """Return logical SQL refs found outside comments and quoted text.""" +def extract_sql_references( + *, sql: str, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlReference, ...]: + """Return logical SQL refs found outside comments and quoted text under the dialect's rules.""" + if syntax.reads_differently_from_generic(sql): + return _extract_sql_references_with_python(sql=sql, syntax=syntax) native_references: list[tuple[str, str, str | None, int | None]] | None = ( _native.extract_static_sql_references(sql) ) @@ -53,10 +56,12 @@ def extract_sql_references(sql: str) -> tuple[CompileSqlReference, ...]: ) for kind, name, package, call_argument_count in native_references ) - return _extract_sql_references_with_python(sql) + return _extract_sql_references_with_python(sql=sql, syntax=syntax) -def _extract_sql_references_with_python(sql: str) -> tuple[CompileSqlReference, ...]: +def _extract_sql_references_with_python( + *, sql: str, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlReference, ...]: """Return logical SQL refs through the authoritative general scanner.""" references: list[CompileSqlReference] = [] @@ -66,19 +71,17 @@ def _extract_sql_references_with_python(sql: str) -> tuple[CompileSqlReference, index = _next_reference_scan_position(sql=sql, start=index) if index >= length: break - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context=_CONTEXT) - continue - if sql[index] in SQL_QUOTE_TOKENS: - index = skip_quoted_text(sql=sql, start=index, context=_CONTEXT) + non_code_end: int | None = dialect_non_code_end( + sql=sql, start=index, syntax=syntax, context=_CONTEXT + ) + if non_code_end is not None: + index = non_code_end continue parsed_reference: tuple[CompileSqlReference, int] | None = _parse_reference_at( sql=sql, start=index, + syntax=syntax, ) if parsed_reference is None: index += 1 @@ -93,7 +96,9 @@ def _next_reference_scan_position(*, sql: str, start: int) -> int: return match.start() if match is not None else len(sql) -def _parse_reference_at(*, sql: str, start: int) -> tuple[CompileSqlReference, int] | None: +def _parse_reference_at( + *, sql: str, start: int, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlReference, int] | None: ref_kind: SqlReferenceKind | None = None prefix: str for prefix, candidate_kind in _REFERENCE_PREFIXES: @@ -105,10 +110,12 @@ def _parse_reference_at(*, sql: str, start: int) -> tuple[CompileSqlReference, i open_paren_index: int = start + len(_REFERENCE_PREFIX_BY_KIND[ref_kind]) closing_paren_index: int = find_matching_paren( - sql=sql, open_paren_index=open_paren_index, context=_CONTEXT + sql=sql, open_paren_index=open_paren_index, context=_CONTEXT, syntax=syntax ) raw_arguments: str = sql[open_paren_index + 1 : closing_paren_index] - argument_values: tuple[str, ...] = _split_top_level_arguments(raw_arguments) + argument_values: tuple[str, ...] = _split_top_level_arguments( + raw_arguments=raw_arguments, syntax=syntax + ) two_argument_count: int = 2 if ref_kind == SqlReferenceKind.DBT_REF: if len(argument_values) not in {1, two_argument_count}: @@ -137,9 +144,12 @@ def _parse_reference_at(*, sql: str, start: int) -> tuple[CompileSqlReference, i sql=sql, open_paren_index=call_suffix_start, context="SQL table function call", + syntax=syntax, ) call_argument_count = len( - _split_top_level_arguments(sql[call_suffix_start + 1 : call_suffix_end]) + _split_top_level_arguments( + raw_arguments=sql[call_suffix_start + 1 : call_suffix_end], syntax=syntax + ) ) return ( CompileSqlReference( @@ -163,27 +173,25 @@ def _skip_whitespace(*, sql: str, start: int) -> int: return index -def _split_top_level_arguments(raw_arguments: str) -> tuple[str, ...]: +def _split_top_level_arguments(*, raw_arguments: str, syntax: SqlLexicalSyntax) -> tuple[str, ...]: arguments: list[str] = [] current: list[str] = [] depth: int = 0 index: int = 0 saw_separator: bool = False while index < len(raw_arguments): - if raw_arguments.startswith("--", index): - index = skip_line_comment(sql=raw_arguments, start=index) - current.append(" ") - continue - if raw_arguments.startswith("/*", index): - index = skip_block_comment(sql=raw_arguments, start=index, context=_CONTEXT) - current.append(" ") + non_code_end: int | None = dialect_non_code_end( + sql=raw_arguments, start=index, syntax=syntax, context=_CONTEXT + ) + if non_code_end is not None: + current.append( + raw_arguments[index:non_code_end] + if raw_arguments[index] in SQL_QUOTE_TOKENS + else " " + ) + index = non_code_end continue character: str = raw_arguments[index] - if character in SQL_QUOTE_TOKENS: - quoted_end: int = skip_quoted_text(sql=raw_arguments, start=index, context=_CONTEXT) - current.append(raw_arguments[index:quoted_end]) - index = quoted_end - continue if character == SQL_OPEN_PAREN_TOKEN: depth += 1 elif character == SQL_CLOSE_PAREN_TOKEN: diff --git a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py index 111313f063..144cb43faf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py @@ -27,6 +27,7 @@ CompileSqlScenarioCte, CompileSqlScenarioCtes, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _CONTEXT: str = "SQL scenario" _WITH_REQUIREMENT: str = ( @@ -34,7 +35,9 @@ ) -def extract_sql_scenario_ctes(*, sql: str, file_label: str) -> CompileSqlScenarioCtes: +def extract_sql_scenario_ctes( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax +) -> CompileSqlScenarioCtes: """Extract top-level SQL-native scenario fixture, expected, and assertion CTEs.""" try: @@ -44,6 +47,7 @@ def extract_sql_scenario_ctes(*, sql: str, file_label: str) -> CompileSqlScenari context_label=_CONTEXT, with_requirement=_WITH_REQUIREMENT, cte_type=CompileSqlScenarioCte, + syntax=syntax, ) except CompileInputError as scanner_error: cte_values: tuple[tuple[str, str], ...] | None = extract_top_level_ctes_with_sql_analysis( @@ -54,13 +58,15 @@ def extract_sql_scenario_ctes(*, sql: str, file_label: str) -> CompileSqlScenari if cte_values is None: raise scanner_error from None ctes = tuple(CompileSqlScenarioCte(name=name, sql_body=body) for name, body in cte_values) - return _classify_sql_scenario_ctes(ctes=ctes, file_label=file_label) + return _classify_sql_scenario_ctes(ctes=ctes, file_label=file_label, syntax=syntax) -def extract_sql_scenario_expected_model_names(*, sql: str, file_label: str) -> tuple[str, ...]: +def extract_sql_scenario_expected_model_names( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax +) -> tuple[str, ...]: """Extract explicit expected-model relationships without inspecting CTE bodies.""" - start: int = _skip_ignorable(sql=sql, start=0, context_label=_CONTEXT) + start: int = _skip_ignorable(sql=sql, start=0, context_label=_CONTEXT, syntax=syntax) if _try_consume_keyword(sql=sql, start=start, keyword=SQL_WITH_KEYWORD) is None: return () ctes: tuple[CompileSqlScenarioCte, ...] = extract_top_level_ctes_with_scanner( @@ -69,6 +75,7 @@ def extract_sql_scenario_expected_model_names(*, sql: str, file_label: str) -> t context_label=_CONTEXT, with_requirement=_WITH_REQUIREMENT, cte_type=CompileSqlScenarioCte, + syntax=syntax, ) return tuple( _require_prefixed_name( @@ -84,7 +91,7 @@ def extract_sql_scenario_expected_model_names(*, sql: str, file_label: str) -> t def _classify_sql_scenario_ctes( - *, ctes: tuple[CompileSqlScenarioCte, ...], file_label: str + *, ctes: tuple[CompileSqlScenarioCte, ...], file_label: str, syntax: SqlLexicalSyntax ) -> CompileSqlScenarioCtes: validate_independent_expected_and_assertion_ctes( ctes=ctes, @@ -92,6 +99,7 @@ def _classify_sql_scenario_ctes( assertion_prefix=ASSERT_SCENARIO_CTE_PREFIX, file_label=file_label, context_label="SQL scenario", + syntax=syntax, ) authored_ctes: list[CompileSqlScenarioCte] = [] expected_ctes: list[CompileSqlScenarioCte] = [] diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index 687955d01b..045bc918f2 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -31,6 +31,7 @@ SQL_ARGUMENT_SEPARATOR_TOKEN, SQL_CEREMONIAL_SELECT_VALUE, SQL_OPEN_PAREN_TOKEN, + SQL_QUOTE_TOKENS, SQL_SINGLE_QUOTE_TOKEN, SQL_STATEMENT_TERMINATOR_TOKEN, SQL_WILDCARD_TOKEN, @@ -58,21 +59,17 @@ ) from sqlbuild.compiler.sql_analysis.main._is_identifier_start import is_identifier_start from sqlbuild.compiler.sql_analysis.main._iter_code_positions import iter_code_positions -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import ( - skip_quoted_text, -) +from sqlbuild.compiler.sql_analysis.main._skip_dialect_non_code import dialect_non_code_end from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( split_set_operation_branches, ) from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _CONTEXT: str = "SQL test" _SQL_TEST_WITH_REQUIREMENT: str = "mock CTEs and one __expected__ CTE" _DIRECT_DEPENDENCY_PATH_LENGTH: int = 2 _SQL_IDENTIFIER_QUOTE_TOKENS: frozenset[str] = frozenset({'"', "`"}) -_SQL_STRING_QUOTE_TOKENS: frozenset[str] = frozenset({SQL_SINGLE_QUOTE_TOKEN, "$"}) @dataclass(frozen=True) @@ -121,19 +118,22 @@ class _DirectLogicModeSpec: def extract_sql_test_ctes( - *, sql: str, file_label: str, mode: SqlTestMode = DEFAULT_SQL_TEST_MODE + *, + sql: str, + file_label: str, + syntax: SqlLexicalSyntax, + mode: SqlTestMode = DEFAULT_SQL_TEST_MODE, ) -> CompileSqlTestCtes: """Extract top-level SQL-native test mock and expected CTEs.""" ctes: tuple[CompileSqlTestCte, ...] = extract_unclassified_sql_test_ctes( - sql=sql, - file_label=file_label, + sql=sql, file_label=file_label, syntax=syntax ) - return classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode) + return classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode, syntax=syntax) def extract_unclassified_sql_test_ctes( - *, sql: str, file_label: str + *, sql: str, file_label: str, syntax: SqlLexicalSyntax ) -> tuple[CompileSqlTestCte, ...]: """Extract raw top-level CTEs before mode-specific classification.""" @@ -144,6 +144,7 @@ def extract_unclassified_sql_test_ctes( context_label=_CONTEXT, with_requirement=_SQL_TEST_WITH_REQUIREMENT, cte_type=CompileSqlTestCte, + syntax=syntax, ) except CompileInputError as scanner_error: cte_values: tuple[tuple[str, str], ...] | None = extract_top_level_ctes_with_sql_analysis( @@ -158,21 +159,29 @@ def extract_unclassified_sql_test_ctes( def classify_sql_test_ctes( - *, ctes: tuple[CompileSqlTestCte, ...], file_label: str, mode: SqlTestMode + *, + ctes: tuple[CompileSqlTestCte, ...], + file_label: str, + mode: SqlTestMode, + syntax: SqlLexicalSyntax, ) -> CompileSqlTestCtes: """Apply mode-specific validation to extracted SQL test CTEs.""" - return _classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode) + return _classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode, syntax=syntax) def extract_sql_test_expected_model_names( - *, sql: str, file_label: str, mode: SqlTestMode = DEFAULT_SQL_TEST_MODE + *, + sql: str, + file_label: str, + syntax: SqlLexicalSyntax, + mode: SqlTestMode = DEFAULT_SQL_TEST_MODE, ) -> tuple[str, ...]: """Extract explicit expected-model relationships without inspecting CTE bodies.""" if mode is not SqlTestMode.MODEL: return () - start: int = _skip_ignorable(sql=sql, start=0) + start: int = _skip_ignorable(sql=sql, start=0, syntax=syntax) if _try_consume_keyword(sql=sql, start=start, keyword=SQL_WITH_KEYWORD) is None: return () ctes: tuple[CompileSqlTestCte, ...] = extract_top_level_ctes_with_scanner( @@ -181,6 +190,7 @@ def extract_sql_test_expected_model_names( context_label=_CONTEXT, with_requirement=_SQL_TEST_WITH_REQUIREMENT, cte_type=CompileSqlTestCte, + syntax=syntax, ) return tuple( _require_prefixed_name( @@ -194,14 +204,16 @@ def extract_sql_test_expected_model_names( ) -def extract_assertion_target_model_names(*, assertion_sql: tuple[str, ...]) -> tuple[str, ...]: +def extract_assertion_target_model_names( + *, assertion_sql: tuple[str, ...], syntax: SqlLexicalSyntax +) -> tuple[str, ...]: """Extract assertion model targets in authored order using canonical references.""" targets: list[str] = [] for sql in assertion_sql: targets.extend( reference.ref_name - for reference in extract_sql_references(sql) + for reference in extract_sql_references(sql=sql, syntax=syntax) if reference.ref_kind == SqlReferenceKind.REF ) return tuple(dict.fromkeys(targets)) @@ -215,22 +227,27 @@ def extract_top_level_ctes_with_scanner[CteT]( context_label: str, with_requirement: str, cte_type: Callable[..., CteT], + syntax: SqlLexicalSyntax, ) -> tuple[CteT, ...]: """Scan top-level `WITH` CTEs followed by the ceremonial `SELECT 1` of a test file.""" with_end: int | None = _try_consume_keyword( sql=sql, - start=_skip_ignorable(sql=sql, start=0, context_label=context_label), + start=_skip_ignorable(sql=sql, start=0, context_label=context_label, syntax=syntax), keyword=SQL_WITH_KEYWORD, ) if with_end is None: raise CompileInputError( f"{context_label} '{file_label}' must declare {with_requirement} before `SELECT 1`" ) - index: int = _skip_ignorable(sql=sql, start=with_end, context_label=context_label) + index: int = _skip_ignorable( + sql=sql, start=with_end, context_label=context_label, syntax=syntax + ) recursive_end: int | None = _try_consume_keyword(sql=sql, start=index, keyword="RECURSIVE") if recursive_end is not None: - index = _skip_ignorable(sql=sql, start=recursive_end, context_label=context_label) + index = _skip_ignorable( + sql=sql, start=recursive_end, context_label=context_label, syntax=syntax + ) ctes: list[CteT] = [] seen_cte_names: set[str] = set() @@ -244,10 +261,17 @@ def extract_top_level_ctes_with_scanner[CteT]( ) seen_cte_names.add(cte_name) - index = _skip_ignorable(sql=sql, start=index, context_label=context_label) + index = _skip_ignorable(sql=sql, start=index, context_label=context_label, syntax=syntax) if index < len(sql) and sql[index] == SQL_OPEN_PAREN_TOKEN: - index = find_matching_paren(sql=sql, open_paren_index=index, context=context_label) + 1 - index = _skip_ignorable(sql=sql, start=index, context_label=context_label) + index = ( + find_matching_paren( + sql=sql, open_paren_index=index, context=context_label, syntax=syntax + ) + + 1 + ) + index = _skip_ignorable( + sql=sql, start=index, context_label=context_label, syntax=syntax + ) index = _consume_keyword( sql=sql, start=index, @@ -255,19 +279,23 @@ def extract_top_level_ctes_with_scanner[CteT]( file_label=file_label, context_label=context_label, ) - index = _skip_ignorable(sql=sql, start=index, context_label=context_label) + index = _skip_ignorable(sql=sql, start=index, context_label=context_label, syntax=syntax) if index >= len(sql) or sql[index] != SQL_OPEN_PAREN_TOKEN: raise CompileInputError( f"{context_label} '{file_label}' CTE '{cte_name}' must use AS (...)" ) cte_body_start: int = index + 1 cte_body_end: int = find_matching_paren( - sql=sql, open_paren_index=index, context=context_label + sql=sql, open_paren_index=index, context=context_label, syntax=syntax ) ctes.append(cte_type(name=cte_name, sql_body=sql[cte_body_start:cte_body_end].strip())) - index = _skip_ignorable(sql=sql, start=cte_body_end + 1, context_label=context_label) + index = _skip_ignorable( + sql=sql, start=cte_body_end + 1, context_label=context_label, syntax=syntax + ) if index < len(sql) and sql[index] == SQL_ARGUMENT_SEPARATOR_TOKEN: - index = _skip_ignorable(sql=sql, start=index + 1, context_label=context_label) + index = _skip_ignorable( + sql=sql, start=index + 1, context_label=context_label, syntax=syntax + ) continue break @@ -276,26 +304,38 @@ def extract_top_level_ctes_with_scanner[CteT]( start=index, file_label=file_label, context_label=context_label, + syntax=syntax, ) return tuple(ctes) def _classify_sql_test_ctes( - *, ctes: tuple[CompileSqlTestCte, ...], file_label: str, mode: SqlTestMode + *, + ctes: tuple[CompileSqlTestCte, ...], + file_label: str, + mode: SqlTestMode, + syntax: SqlLexicalSyntax, ) -> CompileSqlTestCtes: match mode: case SqlTestMode.MODEL: - return _classify_model_sql_test_ctes(ctes=ctes, file_label=file_label) + return _classify_model_sql_test_ctes(ctes=ctes, file_label=file_label, syntax=syntax) case SqlTestMode.MACRO | SqlTestMode.UDF | SqlTestMode.TABLE_FN: return _classify_direct_logic_sql_test_ctes( - ctes=ctes, file_label=file_label, spec=_DIRECT_LOGIC_MODE_SPECS[mode] + ctes=ctes, + file_label=file_label, + spec=_DIRECT_LOGIC_MODE_SPECS[mode], + syntax=syntax, ) case _: raise CompileInputError(f"SQL test '{file_label}' has unsupported mode '{mode}'") def _classify_direct_logic_sql_test_ctes( - *, ctes: tuple[CompileSqlTestCte, ...], file_label: str, spec: _DirectLogicModeSpec + *, + ctes: tuple[CompileSqlTestCte, ...], + file_label: str, + spec: _DirectLogicModeSpec, + syntax: SqlLexicalSyntax, ) -> CompileSqlTestCtes: mode: str = spec.mode.value authored_ctes: list[CompileSqlTestCte] = [] @@ -318,7 +358,9 @@ def _classify_direct_logic_sql_test_ctes( f"SQL test '{file_label}' mode '{mode}' must define exactly one " f"{spec.expected_cte_name} CTE" ) - _validate_expected_cte_query(cte=cte, file_label=file_label, label=cte.name) + _validate_expected_cte_query( + cte=cte, file_label=file_label, label=cte.name, syntax=syntax + ) expected_cte = cte continue if _is_model_mode_cte(cte.name): @@ -356,6 +398,7 @@ def _classify_direct_logic_sql_test_ctes( file_label=file_label, mode=spec.mode, actual_cte_name=spec.actual_cte_name, + syntax=syntax, ) return CompileSqlTestCtes( mode=spec.mode, @@ -369,7 +412,7 @@ def _classify_direct_logic_sql_test_ctes( def _classify_model_sql_test_ctes( - *, ctes: tuple[CompileSqlTestCte, ...], file_label: str + *, ctes: tuple[CompileSqlTestCte, ...], file_label: str, syntax: SqlLexicalSyntax ) -> CompileSqlTestCtes: validate_independent_expected_and_assertion_ctes( ctes=ctes, @@ -377,6 +420,7 @@ def _classify_model_sql_test_ctes( assertion_prefix=ASSERT_TEST_CTE_PREFIX, file_label=file_label, context_label="SQL test", + syntax=syntax, ) authored_ctes: list[CompileSqlTestCte] = [] macro_mocks: dict[str, str] = {} @@ -414,7 +458,9 @@ def _classify_model_sql_test_ctes( label="__macro__", file_label=file_label, ) - macro_mocks[macro_name] = _extract_macro_mock_value(cte=cte, file_label=file_label) + macro_mocks[macro_name] = _extract_macro_mock_value( + cte=cte, file_label=file_label, syntax=syntax + ) continue if cte.name.startswith(REF_TEST_CTE_PREFIX): mock_model_names.append( @@ -484,6 +530,7 @@ def _classify_model_sql_test_ctes( cte=cte, file_label=file_label, label=cte.name, + syntax=syntax, allow_empty_fixture=True, ) expected_ctes.append(cte) @@ -583,6 +630,7 @@ def _validate_call_free_direct_logic_bodies( file_label: str, mode: SqlTestMode, actual_cte_name: str, + syntax: SqlLexicalSyntax, ) -> None: helper_cte: CompileSqlTestCte for helper_cte in helper_ctes: @@ -592,6 +640,7 @@ def _validate_call_free_direct_logic_bodies( mode=mode, cte_label=f"helper CTE '{helper_cte.name}'", allowed_location=actual_cte_name, + syntax=syntax, ) _validate_no_direct_logic_calls( sql=expected_cte.sql_body, @@ -599,18 +648,25 @@ def _validate_call_free_direct_logic_bodies( mode=mode, cte_label=f"CTE {expected_cte.name}", allowed_location=actual_cte_name, + syntax=syntax, ) def _validate_no_direct_logic_calls( - *, sql: str, file_label: str, mode: SqlTestMode, cte_label: str, allowed_location: str + *, + sql: str, + file_label: str, + mode: SqlTestMode, + cte_label: str, + allowed_location: str, + syntax: SqlLexicalSyntax, ) -> None: macro_names: tuple[str, ...] = find_macro_call_names(sql) if macro_names: raise CompileInputError( f"SQL test '{file_label}' mode '{mode.value}' {cte_label} must not call macros" ) - references: tuple[CompileSqlReference, ...] = extract_sql_references(sql) + references: tuple[CompileSqlReference, ...] = extract_sql_references(sql=sql, syntax=syntax) reference: CompileSqlReference | None = next( ( item @@ -627,13 +683,15 @@ def _validate_no_direct_logic_calls( ) -def _extract_macro_mock_value(*, cte: CompileSqlTestCte, file_label: str) -> str: +def _extract_macro_mock_value( + *, cte: CompileSqlTestCte, file_label: str, syntax: SqlLexicalSyntax +) -> str: """Extract the single SQL string literal value from a __macro__ CTE.""" body: str = cte.sql_body.strip() - index: int = _skip_ignorable(sql=body, start=0) + index: int = _skip_ignorable(sql=body, start=0, syntax=syntax) index = _consume_keyword(sql=body, start=index, keyword="SELECT", file_label=file_label) - index = _skip_ignorable(sql=body, start=index) + index = _skip_ignorable(sql=body, start=index, syntax=syntax) if index >= len(body) or body[index] != SQL_SINGLE_QUOTE_TOKEN: raise CompileInputError( f"SQL test '{file_label}' macro mock '{cte.name}' must be a single SELECT string " @@ -641,9 +699,9 @@ def _extract_macro_mock_value(*, cte: CompileSqlTestCte, file_label: str) -> str ) value: str value, index = _read_sql_string_literal(sql=body, start=index) - index = _skip_ignorable(sql=body, start=index) + index = _skip_ignorable(sql=body, start=index, syntax=syntax) if index < len(body) and body[index] == SQL_STATEMENT_TERMINATOR_TOKEN: - index = _skip_ignorable(sql=body, start=index + 1) + index = _skip_ignorable(sql=body, start=index + 1, syntax=syntax) if index != len(body): raise CompileInputError( f"SQL test '{file_label}' macro mock '{cte.name}' must be a single SELECT string " @@ -674,17 +732,19 @@ def _validate_expected_cte_query( *, cte: CompileSqlTestCte, file_label: str, + syntax: SqlLexicalSyntax, label: str = "__expected__", allow_empty_fixture: bool = False, ) -> None: if allow_empty_fixture and is_empty_fixture_query(cte.sql_body): return - if _contains_select_star(cte.sql_body): + if _contains_select_star(sql=cte.sql_body, syntax=syntax): raise CompileInputError(f"SQL test '{file_label}' must not use SELECT * in {label} CTEs") branch_column_names: tuple[tuple[str, ...], ...] = _extract_expected_branch_column_names( sql=cte.sql_body, file_label=file_label, label=label, + syntax=syntax, ) first_branch_column_names: tuple[str, ...] = branch_column_names[0] branch_index: int @@ -698,7 +758,7 @@ def _validate_expected_cte_query( def _extract_expected_branch_column_names( - *, sql: str, file_label: str, label: str + *, sql: str, file_label: str, label: str, syntax: SqlLexicalSyntax ) -> tuple[tuple[str, ...], ...]: sql_analysis_column_names: tuple[tuple[str, ...], ...] | None = ( extract_expected_branch_column_names_with_sql_analysis( @@ -709,38 +769,42 @@ def _extract_expected_branch_column_names( return sql_analysis_column_names branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_CONTEXT) return tuple( - _extract_expected_select_column_names(branch_sql=branch, file_label=file_label, label=label) + _extract_expected_select_column_names( + branch_sql=branch, file_label=file_label, label=label, syntax=syntax + ) for branch in branches ) def _extract_expected_select_column_names( - *, branch_sql: str, file_label: str, label: str + *, branch_sql: str, file_label: str, label: str, syntax: SqlLexicalSyntax ) -> tuple[str, ...]: - index: int = _skip_ignorable(sql=branch_sql, start=0) + index: int = _skip_ignorable(sql=branch_sql, start=0, syntax=syntax) select_end: int | None = _try_consume_keyword(sql=branch_sql, start=index, keyword="SELECT") if select_end is None: raise CompileInputError( f"SQL test '{file_label}' must define each {label} set-operation " "branch as a SELECT query" ) - select_list_end: int = _find_select_list_end(sql=branch_sql, start=select_end) + select_list_end: int = _find_select_list_end(sql=branch_sql, start=select_end, syntax=syntax) raw_select_list: str = branch_sql[select_end:select_list_end] - expressions: tuple[str, ...] = _split_top_level_commas(raw_select_list) + expressions: tuple[str, ...] = _split_top_level_commas(raw_value=raw_select_list, syntax=syntax) if not expressions: raise CompileInputError( f"SQL test '{file_label}' must project at least one column in {label}" ) return tuple( - _extract_expected_projection_name(expression=expression, file_label=file_label, label=label) + _extract_expected_projection_name( + expression=expression, file_label=file_label, label=label, syntax=syntax + ) for expression in expressions ) -def _find_select_list_end(*, sql: str, start: int) -> int: +def _find_select_list_end(*, sql: str, start: int, syntax: SqlLexicalSyntax) -> int: index: int depth: int - for index, depth in iter_code_positions(sql=sql[start:], context=_CONTEXT): + for index, depth in iter_code_positions(sql=sql[start:], context=_CONTEXT, syntax=syntax): if ( depth == 0 and _try_consume_keyword(sql=sql, start=start + index, keyword="FROM") is not None @@ -749,12 +813,12 @@ def _find_select_list_end(*, sql: str, start: int) -> int: return len(sql) -def _split_top_level_commas(raw_value: str) -> tuple[str, ...]: +def _split_top_level_commas(*, raw_value: str, syntax: SqlLexicalSyntax) -> tuple[str, ...]: values: list[str] = [] value_start: int = 0 index: int depth: int - for index, depth in iter_code_positions(sql=raw_value, context=_CONTEXT): + for index, depth in iter_code_positions(sql=raw_value, context=_CONTEXT, syntax=syntax): if depth == 0 and raw_value[index] == SQL_ARGUMENT_SEPARATOR_TOKEN: item: str = raw_value[value_start:index].strip() if item: @@ -767,8 +831,10 @@ def _split_top_level_commas(raw_value: str) -> tuple[str, ...]: return tuple(values) -def _extract_expected_projection_name(*, expression: str, file_label: str, label: str) -> str: - alias_name: str | None = _extract_as_alias(expression) +def _extract_expected_projection_name( + *, expression: str, file_label: str, label: str, syntax: SqlLexicalSyntax +) -> str: + alias_name: str | None = _extract_as_alias(expression=expression, syntax=syntax) if alias_name is not None: return alias_name stripped_expression: str = expression.strip() @@ -779,17 +845,17 @@ def _extract_expected_projection_name(*, expression: str, file_label: str, label ) -def _extract_as_alias(expression: str) -> str | None: +def _extract_as_alias(*, expression: str, syntax: SqlLexicalSyntax) -> str | None: last_alias_name: str | None = None index: int depth: int - for index, depth in iter_code_positions(sql=expression, context=_CONTEXT): + for index, depth in iter_code_positions(sql=expression, context=_CONTEXT, syntax=syntax): if depth != 0: continue as_end: int | None = _try_consume_keyword(sql=expression, start=index, keyword="AS") if as_end is None: continue - alias_index: int = _skip_ignorable(sql=expression, start=as_end) + alias_index: int = _skip_ignorable(sql=expression, start=as_end, syntax=syntax) if alias_index < len(expression) and is_identifier_start(expression[alias_index]): alias_name, alias_end = _read_identifier( sql=expression, @@ -807,13 +873,13 @@ def _is_simple_identifier(value: str) -> bool: return all(is_identifier_character(character) for character in value[1:]) -def _contains_select_star(sql: str) -> bool: +def _contains_select_star(*, sql: str, syntax: SqlLexicalSyntax) -> bool: index: int - for index, _depth in iter_code_positions(sql=sql, context=_CONTEXT): + for index, _depth in iter_code_positions(sql=sql, context=_CONTEXT, syntax=syntax): select_end: int | None = _try_consume_keyword(sql=sql, start=index, keyword="SELECT") if select_end is None: continue - value_index: int = _skip_ignorable(sql=sql, start=select_end) + value_index: int = _skip_ignorable(sql=sql, start=select_end, syntax=syntax) if value_index < len(sql) and sql[value_index] == SQL_WILDCARD_TOKEN: return True return False @@ -833,9 +899,12 @@ def _validate_ceremonial_select( sql: str, start: int, file_label: str, + syntax: SqlLexicalSyntax, context_label: str = _CONTEXT, ) -> None: - if _is_ceremonial_select_statement(sql=sql, start=start, context_label=context_label): + if _is_ceremonial_select_statement( + sql=sql, start=start, context_label=context_label, syntax=syntax + ): return raise CompileInputError( f"{context_label} '{file_label}' must end with a ceremonial top-level `SELECT 1` " @@ -843,22 +912,28 @@ def _validate_ceremonial_select( ) -def _is_ceremonial_select_statement(*, sql: str, start: int, context_label: str = _CONTEXT) -> bool: - index: int = _skip_ignorable(sql=sql, start=start, context_label=context_label) +def _is_ceremonial_select_statement( + *, sql: str, start: int, syntax: SqlLexicalSyntax, context_label: str = _CONTEXT +) -> bool: + index: int = _skip_ignorable(sql=sql, start=start, context_label=context_label, syntax=syntax) select_end: int | None = _try_consume_keyword(sql=sql, start=index, keyword="SELECT") if select_end is None: return False - index = _skip_ignorable(sql=sql, start=select_end, context_label=context_label) + index = _skip_ignorable(sql=sql, start=select_end, context_label=context_label, syntax=syntax) if index >= len(sql) or sql[index] != SQL_CEREMONIAL_SELECT_VALUE: return False - index = _skip_ignorable(sql=sql, start=index + 1, context_label=context_label) - return _is_statement_end(sql=sql, start=index, context_label=context_label) + index = _skip_ignorable(sql=sql, start=index + 1, context_label=context_label, syntax=syntax) + return _is_statement_end(sql=sql, start=index, context_label=context_label, syntax=syntax) -def _is_statement_end(*, sql: str, start: int, context_label: str = _CONTEXT) -> bool: +def _is_statement_end( + *, sql: str, start: int, syntax: SqlLexicalSyntax, context_label: str = _CONTEXT +) -> bool: index: int = start if index < len(sql) and sql[index] == SQL_STATEMENT_TERMINATOR_TOKEN: - index = _skip_ignorable(sql=sql, start=index + 1, context_label=context_label) + index = _skip_ignorable( + sql=sql, start=index + 1, context_label=context_label, syntax=syntax + ) return index == len(sql) @@ -869,6 +944,7 @@ def validate_independent_expected_and_assertion_ctes( assertion_prefix: str, file_label: str, context_label: str, + syntax: SqlLexicalSyntax, ) -> None: """Reject direct and transitive dependencies between expected and assertion checks.""" @@ -915,6 +991,7 @@ def validate_independent_expected_and_assertion_ctes( cte.name.casefold(): _known_cte_references( sql=cte.sql_body, names_by_key=names_by_key, + syntax=syntax, ) for cte in ctes } @@ -974,7 +1051,9 @@ def collect(value: Any) -> tuple[str, ...]: return collect(parsed.to_dict()) -def _known_cte_references(*, sql: str, names_by_key: dict[str, str]) -> tuple[str, ...]: +def _known_cte_references( + *, sql: str, names_by_key: dict[str, str], syntax: SqlLexicalSyntax +) -> tuple[str, ...]: folded_sql: str = sql.casefold() if not any(name in folded_sql for name in names_by_key): return () @@ -982,7 +1061,7 @@ def _known_cte_references(*, sql: str, names_by_key: dict[str, str]) -> tuple[st try: parsed: Any = polyglot_module.parse_one(sql, dialect="generic") except polyglot_module.PolyglotError: - return _known_cte_identifier_references(sql=sql, names_by_key=names_by_key) + return _known_cte_identifier_references(sql=sql, names_by_key=names_by_key, syntax=syntax) references: list[str] = [] for table in parsed.find_all("table"): key: str = str(getattr(table, "name", "") or "").casefold() @@ -991,22 +1070,21 @@ def _known_cte_references(*, sql: str, names_by_key: dict[str, str]) -> tuple[st return tuple(references) -def _known_cte_identifier_references(*, sql: str, names_by_key: dict[str, str]) -> tuple[str, ...]: +def _known_cte_identifier_references( + *, sql: str, names_by_key: dict[str, str], syntax: SqlLexicalSyntax +) -> tuple[str, ...]: references: list[str] = [] index: int = 0 while index < len(sql): - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context=_CONTEXT) - continue - if sql[index] in _SQL_STRING_QUOTE_TOKENS: - index = skip_quoted_text(sql=sql, start=index, context=_CONTEXT) + non_code_end: int | None = dialect_non_code_end( + sql=sql, start=index, syntax=syntax, context=_CONTEXT + ) + if non_code_end is not None and sql[index] not in _SQL_IDENTIFIER_QUOTE_TOKENS: + index = non_code_end continue - if sql[index] in _SQL_IDENTIFIER_QUOTE_TOKENS: + if non_code_end is not None: quote: str = sql[index] - end: int = skip_quoted_text(sql=sql, start=index, context=_CONTEXT) + end: int = non_code_end name: str = sql[index + 1 : end - 1].replace(quote * 2, quote) key: str = name.casefold() if key in names_by_key and key not in references: @@ -1075,17 +1153,20 @@ def _read_identifier( return sql[start:index], index -def _skip_ignorable(*, sql: str, start: int, context_label: str = _CONTEXT) -> int: +def _skip_ignorable( + *, sql: str, start: int, syntax: SqlLexicalSyntax, context_label: str = _CONTEXT +) -> int: index: int = start while index < len(sql): if sql[index].isspace(): index += 1 continue - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context=context_label) - continue + if sql[index] not in SQL_QUOTE_TOKENS: + comment_end: int | None = dialect_non_code_end( + sql=sql, start=index, syntax=syntax, context=context_label + ) + if comment_end is not None: + index = comment_end + continue return index return index diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py index 648fb30043..8792025f14 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py @@ -2,12 +2,14 @@ from __future__ import annotations +from collections.abc import Iterator from pathlib import Path from typing import Protocol, cast import orjson import sqlbuild._native as _native +from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_sql_test_ctes from sqlbuild.compiler.compile.constants import ( SQL_TEST_FACT_CACHE_ALGORITHM, SQL_TEST_FACT_CACHE_NAMESPACE, @@ -21,6 +23,7 @@ ) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class _NativeSqlTestModule(Protocol): @@ -36,6 +39,7 @@ def extract_expanded_sql_tests_cached( *, tests: tuple[tuple[str, str, SqlTestMode], ...], cache_root: Path | None, + syntax: SqlLexicalSyntax, ) -> tuple[CompileSqlTestCtes, ...]: """Reuse exact per-file extraction facts and extract only tests of changed files natively.""" @@ -45,13 +49,15 @@ def extract_expanded_sql_tests_cached( algorithm=SQL_TEST_FACT_CACHE_ALGORITHM, ) as fact_cache: if not fact_cache.enabled: - return extract_expanded_sql_tests(tests) + return extract_expanded_sql_tests(tests=tests, syntax=syntax) indexes_by_file: dict[str, list[int]] = {} for index, (_sql, file_label, _mode) in enumerate(tests): indexes_by_file.setdefault(file_label, []).append(index) keys_by_file: dict[str, str] = { file_label: fact_cache.key( - file_label, *_file_test_key_parts(tests=tests, indexes=indexes) + file_label, + syntax.cache_key, + *_file_test_key_parts(tests=tests, indexes=indexes), ) for file_label, indexes in indexes_by_file.items() } @@ -73,7 +79,7 @@ def extract_expanded_sql_tests_cached( missing_indexes.extend(indexes_by_file[file_label]) if missing_indexes: extracted: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - tuple(tests[index] for index in missing_indexes) + tests=tuple(tests[index] for index in missing_indexes), syntax=syntax ) for index, test_ctes in zip(missing_indexes, extracted, strict=True): results[index] = test_ctes @@ -108,10 +114,37 @@ def _cached_file_tests( def extract_expanded_sql_tests( + *, tests: tuple[tuple[str, str, SqlTestMode], ...], + syntax: SqlLexicalSyntax, ) -> tuple[CompileSqlTestCtes, ...]: - """Extract and classify expanded tests in one authoritative native call.""" + """Extract and classify expanded tests, natively unless the dialect reads a test differently.""" + + dialect_indexes: frozenset[int] = frozenset( + index + for index, (sql, _file_label, _mode) in enumerate(tests) + if syntax.reads_differently_from_generic(sql) + ) + if not dialect_indexes: + return _extract_expanded_sql_tests_natively(tests) + native_results: Iterator[CompileSqlTestCtes] = iter( + _extract_expanded_sql_tests_natively( + tuple(test for index, test in enumerate(tests) if index not in dialect_indexes) + ) + ) + return tuple( + extract_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax, mode=mode) + if index in dialect_indexes + else next(native_results) + for index, (sql, file_label, mode) in enumerate(tests) + ) + +def _extract_expanded_sql_tests_natively( + tests: tuple[tuple[str, str, SqlTestMode], ...], +) -> tuple[CompileSqlTestCtes, ...]: + if not tests: + return () request_json: str = orjson.dumps( { "tests": [ diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py index 53edda5aa0..8211248490 100644 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py +++ b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py @@ -6,13 +6,16 @@ extract_unclassified_sql_test_ctes, ) from sqlbuild.compiler.compile.models import CompileSqlTestCte +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class SqlTestCteExtractor: """Expose authored SQL-test CTE parsing to compiler consumers.""" @staticmethod - def extract(*, sql: str, file_label: str) -> tuple[CompileSqlTestCte, ...]: + def extract( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax + ) -> tuple[CompileSqlTestCte, ...]: """Extract top-level SQL-test CTEs without classifying their roles.""" - return extract_unclassified_sql_test_ctes(sql=sql, file_label=file_label) + return extract_unclassified_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax) diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index a40fe5f8c1..534c86793f 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -127,6 +127,7 @@ def build_compile_inputs( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) model_context: ModelInputBuildContext = ModelInputBuildContext( effective_vars=effective_vars, @@ -138,6 +139,7 @@ def build_compile_inputs( loaded_macros=declaration_scope.loaded_macros, value_renderer=adapter_context.value_renderer, collection_rendering=adapter_context.collection_rendering, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, declaration_resolver=declaration_scope.resolver, ) model_build: ModelInputScopeBuild = _build_models_with_declarations( @@ -178,6 +180,7 @@ def build_compile_inputs( external_sql_reference_resolver=external_sql_reference_resolver, sql_function_inputs=sql_function_inputs, compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( discovered_inputs=discovered_inputs, @@ -186,6 +189,7 @@ def build_compile_inputs( loaded_macros=declaration_scope.loaded_macros, declaration_expansion=model_context.declaration_expansion, external_sql_reference_resolver=external_sql_reference_resolver, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) project_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] = ( index_generic_audit_definitions(discovered_inputs.audit_files) @@ -198,13 +202,9 @@ def build_compile_inputs( diagnostics = (*diagnostics, *model_build.diagnostics) audit_inputs: tuple[CompileAuditInput, ...] = build_audit_inputs( discovered_inputs=discovered_inputs, - effective_settings=effective_settings, + context=model_context, model_inputs=model_build.inputs, source_inputs=source_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, generic_audit_definitions=generic_audit_definitions, seed_inputs=seed_inputs, ) @@ -212,6 +212,7 @@ def build_compile_inputs( project_config=discovered_inputs.project_config, local_config=discovered_inputs.local_config, discovered_inputs=discovered_inputs, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, run_id=resolved_run_id, effective_target_name=effective_target_name, effective_target=effective_target, @@ -261,12 +262,12 @@ def _build_sql_functions( effective_vars=context.effective_vars, effective_settings=context.effective_settings, target_config=context.target_config, - adapter_name=context.macro_context.adapter_name, macro_context=context.macro_context, loaded_macros=context.loaded_macros, declaration_expansion=declaration_expansion, no_sql_validation=no_sql_validation, python_functions_inherit_default_namespace=python_functions_inherit_default_namespace, + sql_lexical_syntax=context.sql_lexical_syntax, ) validate_table_function_call_arities( model_inputs=model_inputs, diff --git a/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py b/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py index 69d12227a7..f6d7255467 100644 --- a/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py +++ b/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py @@ -8,11 +8,19 @@ from sqlbuild.compiler.compile.models import ScopeRelationshipBuild from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax def build_scope_relationship_grants( - *, discovered_inputs: DiscoveredProjectInputs, index: ScopeIndex + *, + discovered_inputs: DiscoveredProjectInputs, + index: ScopeIndex, + sql_lexical_syntax: SqlLexicalSyntax, ) -> ScopeRelationshipBuild: """Return expected-model grants while retaining independent extraction faults.""" - return _build_scope_relationship_grants(discovered_inputs=discovered_inputs, index=index) + return _build_scope_relationship_grants( + discovered_inputs=discovered_inputs, + index=index, + sql_lexical_syntax=sql_lexical_syntax, + ) diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index dfc0042d0d..06e375ed70 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -26,6 +26,7 @@ from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) @@ -57,7 +58,9 @@ def build_sql_expansion_context( cli_vars={} if cli_vars is None else cli_vars, ) scope: DeclarationScopeBuild = declaration_scope or _build_declaration_scope( - discovered_inputs=effective_discovered_inputs, static_scope=static_declaration_scope + discovered_inputs=effective_discovered_inputs, + static_scope=static_declaration_scope, + sql_lexical_syntax=value_renderer.sql_lexical_syntax, ) loaded_macros: dict[str, LoadedMacro] = scope.loaded_macros enums: dict[str, EnumDeclaration] @@ -107,7 +110,10 @@ def build_sql_expansion_context( def _build_declaration_scope( - *, discovered_inputs: DiscoveredProjectInputs, static_scope: DeclarationScopeBuild | None + *, + discovered_inputs: DiscoveredProjectInputs, + static_scope: DeclarationScopeBuild | None, + sql_lexical_syntax: SqlLexicalSyntax, ) -> DeclarationScopeBuild: loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) rebound: DeclarationScopeBuild | None = ( @@ -118,5 +124,7 @@ def _build_declaration_scope( ) ) return rebound or build_declaration_scope( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros + discovered_inputs=discovered_inputs, + loaded_macros=loaded_macros, + sql_lexical_syntax=sql_lexical_syntax, ) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index dcd3fbfef1..ca50211cf0 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -61,7 +61,7 @@ UsageRecord, VisibilityRecord, ) -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( DefaultsConfig, LocalConfig, @@ -439,6 +439,7 @@ class CompileAdapterContext: value_renderer: TypedSqlValueRenderer collection_rendering: CollectionRendering python_functions_inherit_default_namespace: bool + sql_lexical_syntax: SqlLexicalSyntax @dataclass(frozen=True) @@ -454,6 +455,7 @@ class ModelInputBuildContext: loaded_macros: dict[str, LoadedMacro] value_renderer: TypedSqlValueRenderer collection_rendering: CollectionRendering + sql_lexical_syntax: SqlLexicalSyntax public_enums: dict[str, EnumDeclaration] = field(default_factory=dict) public_constants: dict[str, ConstantDeclaration] = field(default_factory=dict) public_model_schemas: dict[str, ModelSchemaDeclaration] = field(default_factory=dict) @@ -846,6 +848,7 @@ class CompileProjectInputs: project_config: ProjectConfig local_config: LocalConfig discovered_inputs: DiscoveredProjectInputs + sql_lexical_syntax: SqlLexicalSyntax run_id: str = "" effective_target_name: str | None = None effective_target: TargetConfig | None = None @@ -1048,6 +1051,7 @@ class CompiledProject: effective_target_database: str | None = None effective_target_schema: str | None = None sql_analysis_dialect: str | None = None + sql_lexical_syntax: SqlLexicalSyntax = field(default_factory=SqlLexicalSyntax) compile_cache_dir: Path | None = None settings: SettingsConfig = field(default_factory=SettingsConfig) scenario: ScenarioConfig = field(default_factory=ScenarioConfig) @@ -1133,6 +1137,7 @@ class CompileModelSqlTestInputPayload: expected_model_names: tuple[str, ...] = field(default_factory=tuple) assertion_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) assertion_names: tuple[str, ...] = field(default_factory=tuple) + assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index ce19323a59..9820257fd2 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -9,12 +9,14 @@ if TYPE_CHECKING: from sqlbuild.compiler.compile.models import CompactBatchPreparation, CompileProjectInputs + from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class TypedSqlValueRenderer(Protocol): """Adapter rendering operations consumed by authored SQL expansion.""" adapter_name: str + sql_lexical_syntax: SqlLexicalSyntax def render_typed_scalar(self, *, value: SqlValue) -> str: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py index 57ff142659..c6f1d2e12b 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py +++ b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py @@ -46,6 +46,17 @@ "unique_key", } ) +_DLT_SOURCE_GROUP_KEYS: frozenset[str] = frozenset( + {"config", "destination", "resources", "schema", "type"} +) +_DLT_RESOURCE_COMMON_KEYS: frozenset[str] = frozenset( + {"incremental", "merge_key", "name", "primary_key", "schema", "write_disposition"} +) +_DLT_RESOURCE_TYPE_KEYS: dict[str, frozenset[str]] = { + DLT_SOURCE_TYPE_FILESYSTEM: frozenset({"reader"}), + DLT_SOURCE_TYPE_REST_API: frozenset({"endpoint"}), + DLT_SOURCE_TYPE_SQL_DATABASE: frozenset({"table"}), +} def integration_loader_name(*, kind: str, source_name: str) -> str: @@ -201,6 +212,13 @@ def _optional_ingestr_string_tuple( def _parse_dlt_source_group( *, group: dict[str, object], file_path: Path, group_index: int ) -> tuple[SourceEntry, ...]: + reject_unknown_mapping_keys( + mapping=group, + allowed=_DLT_SOURCE_GROUP_KEYS, + file_path=file_path, + label="dlt source", + error_class=SourceParseError, + ) source_type: str = require_non_empty_string( entry=group, key="type", @@ -275,6 +293,13 @@ def _parse_dlt_resource_entry( raise SourceParseError( f"{file_path} dlt resource '{name}' must use dlt write_disposition, not write_strategy" ) + reject_unknown_mapping_keys( + mapping=resource, + allowed=_DLT_RESOURCE_COMMON_KEYS | _DLT_RESOURCE_TYPE_KEYS[source_type], + file_path=file_path, + label=f"dlt {source_type} resource '{name}'", + error_class=SourceParseError, + ) dlt_name: str = _dlt_resource_name( source_type=source_type, resource=resource, file_path=file_path ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index 4aad18539e..f05cf7df1f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -14,7 +14,11 @@ from yaml import YAMLError from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.constants import MAX_MICROBATCHES_CONFIG_KEY +from sqlbuild.compiler.authored_values.main._change_policy_problem import change_policy_problem +from sqlbuild.compiler.authored_values.main._change_policy_toml_help import ( + change_policy_toml_help, +) +from sqlbuild.compiler.compile.constants import MAX_MICROBATCHES_CONFIG_KEY, TEMPLATE_OPEN_TOKEN from sqlbuild.compiler.discovery._helpers.validation.supported_keys import unsupported_keys_help from sqlbuild.compiler.discovery.constants import ( CONFIG_CONCURRENCY_KEY, @@ -932,6 +936,20 @@ def _validate_allowed_keys( ) +def _validate_change_policies(*, mapping: dict[str, object], section: str, file_path: Path) -> None: + key: str + for key in ("on_schema_change", "replay_on_change"): + value: object | None = mapping.get(key) + if value is None or (isinstance(value, str) and TEMPLATE_OPEN_TOKEN in value): + continue + problem: str | None = change_policy_problem(key=key, value=value) + if problem is not None: + raise ProjectConfigError( + f"{file_path} [{section}] {problem}", + help=change_policy_toml_help(key=key, file_name=file_path.name, section=section), + ) + + def _normalize_path_default_key(*, path_key: str, file_path: Path) -> str: normalized_key: str = path_key.strip() if not normalized_key: @@ -986,6 +1004,7 @@ def _load_defaults(*, payload: object, file_path: Path) -> DefaultsConfig: _validate_allowed_keys( mapping=mapping, allowed_keys=_DEFAULTS_KEYS, label="[defaults]", file_path=file_path ) + _validate_change_policies(mapping=mapping, section="defaults", file_path=file_path) row_diff_exclude_columns: tuple[str, ...] = tuple( _load_string_sequence( payload=mapping.get("row_diff_exclude_columns"), @@ -1085,6 +1104,11 @@ def _load_path_defaults(*, payload: object, file_path: Path) -> dict[str, dict[s label=f"path_defaults['{path_key}']", file_path=file_path, ) + _validate_change_policies( + mapping=path_dict, + section=f'path_defaults."{path_key}"', + file_path=file_path, + ) _validate_path_default_tags(path_dict=path_dict, path_key=path_key, file_path=file_path) normalized_path_key: str = _normalize_path_default_key( path_key=path_key, file_path=file_path diff --git a/src/sqlbuild/compiler/pipeline/main/compiled_project.py b/src/sqlbuild/compiler/pipeline/main/compiled_project.py index 84d343e16e..499eda8df0 100644 --- a/src/sqlbuild/compiler/pipeline/main/compiled_project.py +++ b/src/sqlbuild/compiler/pipeline/main/compiled_project.py @@ -77,6 +77,7 @@ def build_compiled_project( python_functions_inherit_default_namespace=( adapter.python_functions_inherit_default_namespace() ), + sql_lexical_syntax=adapter.sql_lexical_syntax, ), selected_target=selected_target, no_sql_validation=no_sql_validation, diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/policy.py b/src/sqlbuild/compiler/planner/_helpers/changes/policy.py index 419fcab0f6..2195b5de55 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/policy.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/policy.py @@ -4,6 +4,7 @@ import re +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import BackfillResult from sqlbuild.compiler.planner.types import BackfillAction @@ -44,4 +45,4 @@ def _resolve_replay_value(raw: str | None) -> BackfillResult: if match is not None: duration: str = match.group(1).strip() return BackfillResult(action=BackfillAction.BOUNDED, duration=duration) - return BackfillResult(action=BackfillAction.FORWARD_ONLY) + raise PlannerInputError(f"unknown replay_on_change '{raw}'") diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 1a0602f3ae..b91db81652 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -1027,12 +1027,11 @@ def _get_on_schema_change(model: CompiledModel) -> OnSchemaChange | None: """Extract on_schema_change from model config.""" raw: object | None = model.config.values.get("on_schema_change") - if isinstance(raw, str): - try: - return OnSchemaChange(raw) - except ValueError: - pass - return None + if raw is None: + return None + if isinstance(raw, str) and raw in set(OnSchemaChange): + return OnSchemaChange(raw) + raise PlannerInputError(f"model '{model.name}': unknown on_schema_change '{raw}'") def extract_seed_columns(seed: CompiledSeed) -> tuple[ColumnInfo, ...]: diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py index f2f591c8e4..64dacfb4f4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py @@ -28,6 +28,7 @@ FixtureRelationMetadata, ) from sqlbuild.compiler.planner.types import ContractPolicy, FixtureKey +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SourceEntry _TYPED_NULL_LINE_PATTERN: re.Pattern[str] = re.compile( @@ -72,6 +73,7 @@ def format_redundant_fixture_nulls( files: dict[Path, str], project_dir: Path, discovered_inputs: DiscoveredProjectInputs, + sql_lexical_syntax: SqlLexicalSyntax, ) -> dict[Path, str]: """Remove typed-null fixture projections that planner completion can reproduce.""" @@ -100,6 +102,7 @@ def format_redundant_fixture_nulls( block=block, file_label=str(test_file.relative_path), relations=relations, + syntax=sql_lexical_syntax, ) if fixed_block == block.sql_body: continue @@ -117,6 +120,7 @@ def _format_test_block( block: DiscoveredSqlTestBlock, file_label: str, relations: dict[FixtureKey, FixtureRelationMetadata], + syntax: SqlLexicalSyntax, ) -> str: if ( block.mode is not SqlTestMode.MODEL @@ -128,6 +132,7 @@ def _format_test_block( authored_ctes: tuple[CompileSqlTestCte, ...] = SqlTestCteExtractor.extract( sql=block.sql_body, file_label=file_label, + syntax=syntax, ) except CompileInputError: return block.sql_body diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py index d47bbfb665..87c199d9ab 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py @@ -35,6 +35,7 @@ from sqlbuild.compiler.planner.types import WarningSeverity from sqlbuild.compiler.profiling.classes.context import CompileTimingContext from sqlbuild.compiler.profiling.models import CompileTimingCollector +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule _CALL_SUFFIX_SENTINEL: str = "__SQLBUILD_CALL_SUFFIX__" @@ -103,6 +104,7 @@ def plan_sql_tests_natively( "sqlAnalysisDialect": adapter.sql_analysis_dialect(), "setDifferenceOperator": adapter.render_set_difference_operator(), "requiresDerivedTableAliases": adapter.requires_derived_table_aliases(), + "lexicalSyntax": _lexical_syntax_request(project.sql_lexical_syntax), "workers": _NATIVE_WORKERS, "renderSql": render_sql, "includePlan": include_plan, @@ -159,6 +161,7 @@ def resolve_sql_test_model_chains( request: dict[str, object] = { "models": _chain_model_requests(project=project), "tests": [_test_request(test=test) for test in tests], + "lexicalSyntax": _lexical_syntax_request(project.sql_lexical_syntax), } try: native_response: str = cast( @@ -457,3 +460,14 @@ def _render_call_template(*, rendered: str) -> tuple[str, str]: if not separator or _CALL_SUFFIX_SENTINEL in suffix: raise NativeSqlTestPlanningError("adapter SQL function call template is not deterministic") return prefix, suffix + + +def _lexical_syntax_request(syntax: SqlLexicalSyntax) -> dict[str, object]: + return { + "backslashEscapeQuotes": sorted(syntax.backslash_escape_quotes), + "escapeStringPrefix": syntax.escape_string_prefix, + "rawStringPrefix": syntax.raw_string_prefix, + "tripleQuotedStrings": syntax.triple_quoted_strings, + "nestedBlockComments": syntax.nested_block_comments, + "lineCommentPrefixes": sorted(syntax.line_comment_prefixes), + } diff --git a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py index 97cf4178ec..16cfc725de 100644 --- a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py +++ b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py @@ -8,6 +8,7 @@ from sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting import ( format_redundant_fixture_nulls, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class FixtureNullAutofix: @@ -19,6 +20,7 @@ def apply( files: dict[Path, str], project_dir: Path, discovered_inputs: DiscoveredProjectInputs, + sql_lexical_syntax: SqlLexicalSyntax, ) -> dict[Path, str]: """Return contract-safe fixture source rewrites.""" @@ -26,4 +28,5 @@ def apply( files=files, project_dir=project_dir, discovered_inputs=discovered_inputs, + sql_lexical_syntax=sql_lexical_syntax, ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/builder.py b/src/sqlbuild/compiler/scopes/_helpers/builder.py index 1da3d2e6ce..95a8235ef7 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/builder.py +++ b/src/sqlbuild/compiler/scopes/_helpers/builder.py @@ -61,6 +61,7 @@ ScopeKind, UsageKind, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.sql_values.types import SqlValueKind _ROOTS: dict[ResourceKind, str] = { @@ -254,7 +255,9 @@ def _macro_dependency_usages(*, declarations: list[DeclarationRecord]) -> tuple[ return tuple(dict.fromkeys(usages)) -def build_tolerant_scope_index(*, project_dir: Path) -> ScopeIndex: +def build_tolerant_scope_index( + *, project_dir: Path, sql_lexical_syntax: SqlLexicalSyntax +) -> ScopeIndex: """Build a partial static index from published tolerant compiler facts.""" snapshot: TolerantScopeDiscovery = discover_scope_snapshot(project_dir=project_dir) @@ -269,6 +272,7 @@ def build_tolerant_scope_index(*, project_dir: Path) -> ScopeIndex: relationships: ScopeRelationshipBuild = build_scope_relationship_grants( discovered_inputs=snapshot.discovered_inputs, index=index, + sql_lexical_syntax=sql_lexical_syntax, ) diagnostics: tuple[ScopeDiagnostic, ...] = _tolerant_diagnostics( snapshot=snapshot, diff --git a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py index f25bbfe920..2bf34d2a19 100644 --- a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py @@ -20,6 +20,7 @@ write_cached_scope_index, ) from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) @@ -44,6 +45,7 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S if cached is not None: return cached + sql_lexical_syntax: SqlLexicalSyntax = SqlLexicalSyntax() try: discovered: DiscoveredProjectInputs = discover_project_inputs( project_dir=resolved_project_dir, @@ -54,6 +56,7 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S local_config=discovered.local_config, ) adapter: BaseAdapter = resolve_adapter(adapter_name=adapter_name) + sql_lexical_syntax = adapter.sql_lexical_syntax compile_inputs: CompileProjectInputs = build_compile_inputs( discovered_inputs=discovered, adapter_context=CompileAdapterContext( @@ -65,6 +68,7 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S python_functions_inherit_default_namespace=( adapter.python_functions_inherit_default_namespace() ), + sql_lexical_syntax=adapter.sql_lexical_syntax, ), resolved_connection={}, no_sql_validation=True, @@ -73,7 +77,9 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S ) index: ScopeIndex = scope_index_with_compile_usages(inputs=compile_inputs) except (OSError, UnicodeError, ValueError, ImportError): - index = build_tolerant_scope_index(project_dir=resolved_project_dir) + index = build_tolerant_scope_index( + project_dir=resolved_project_dir, sql_lexical_syntax=sql_lexical_syntax + ) if not no_cache and fingerprint is not None: write_cached_scope_index( diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py b/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py index 9352b90638..ee018e4914 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py @@ -25,7 +25,7 @@ ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -_PAREN_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"[-/'\"`$()]") +_PAREN_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"[-/#'\"`$()]") _DOLLAR_QUOTE_DELIMITER: re.Pattern[str] = re.compile(r"\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$") @@ -175,7 +175,13 @@ def skip_block_comment_impl(*, sql: str, start: int, context: str = "SQL") -> in return closing_index + 2 -def find_matching_paren_impl(*, sql: str, open_paren_index: int, context: str = "SQL") -> int: +def find_matching_paren_impl( + *, + sql: str, + open_paren_index: int, + context: str = "SQL", + syntax: SqlLexicalSyntax | None = None, +) -> int: """Find the closing parenthesis matching an opening parenthesis.""" depth: int = 1 @@ -185,7 +191,9 @@ def find_matching_paren_impl(*, sql: str, open_paren_index: int, context: str = if special is None: break index = special.start() - non_code_end: int | None = _non_code_end(sql=sql, index=index, context=context) + non_code_end: int | None = _non_code_end( + sql=sql, index=index, context=context, syntax=syntax + ) if non_code_end is not None: index = non_code_end continue @@ -199,13 +207,17 @@ def find_matching_paren_impl(*, sql: str, open_paren_index: int, context: str = raise CompileInputError(f"{context} contains an unclosed parenthesis") -def iter_code_positions_impl(*, sql: str, context: str = "SQL") -> Iterator[tuple[int, int]]: +def iter_code_positions_impl( + *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None +) -> Iterator[tuple[int, int]]: """Yield code offsets outside comments, quotes, and parentheses with their nesting depth.""" depth: int = 0 index: int = 0 while index < len(sql): - non_code_end: int | None = _non_code_end(sql=sql, index=index, context=context) + non_code_end: int | None = _non_code_end( + sql=sql, index=index, context=context, syntax=syntax + ) if non_code_end is not None: index = non_code_end continue @@ -219,7 +231,11 @@ def iter_code_positions_impl(*, sql: str, context: str = "SQL") -> Iterator[tupl index += 1 -def _non_code_end(*, sql: str, index: int, context: str) -> int | None: +def _non_code_end( + *, sql: str, index: int, context: str, syntax: SqlLexicalSyntax | None +) -> int | None: + if syntax is not None: + return dialect_non_code_end_impl(sql=sql, start=index, syntax=syntax, context=context) if sql.startswith("--", index): return skip_line_comment_impl(sql=sql, start=index) if sql.startswith("/*", index): diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index bec3ddd0bf..cfeba4649c 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -16,6 +16,7 @@ SQL_STRING_PREFIX_CHARACTERS: frozenset[str] = frozenset("bBeErR") SQL_STRING_PREFIX_MAX_LENGTH: int = 2 SQL_TRIPLE_QUOTE_LENGTH: int = 3 +SQL_LINE_COMMENT_PREFIX: str = "--" SQL_BLOCK_COMMENT_OPEN: str = "/*" SQL_BLOCK_COMMENT_CLOSE: str = "*/" SQL_DOLLAR_QUOTE_DELIMITER: str = "$$" diff --git a/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py b/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py index 8599e4b19a..a5b31ae45c 100644 --- a/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py +++ b/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py @@ -1,13 +1,21 @@ """SQL parenthesis matching entrypoint.""" from sqlbuild.compiler.sql_analysis._helpers.scanning import find_matching_paren_impl +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -def find_matching_paren(*, sql: str, open_paren_index: int, context: str = "SQL") -> int: - """Find the closing parenthesis matching an opening parenthesis.""" +def find_matching_paren( + *, + sql: str, + open_paren_index: int, + context: str = "SQL", + syntax: SqlLexicalSyntax | None = None, +) -> int: + """Find the closing parenthesis, ending quoted text and comments per the optional dialect.""" return find_matching_paren_impl( sql=sql, open_paren_index=open_paren_index, context=context, + syntax=syntax, ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py b/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py index b2a491333d..649dbd11d2 100644 --- a/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py +++ b/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py @@ -3,9 +3,12 @@ from collections.abc import Iterator from sqlbuild.compiler.sql_analysis._helpers.scanning import iter_code_positions_impl +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -def iter_code_positions(*, sql: str, context: str = "SQL") -> Iterator[tuple[int, int]]: - """Yield each code character offset outside comments and quotes with its parenthesis depth.""" +def iter_code_positions( + *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None +) -> Iterator[tuple[int, int]]: + """Yield each code offset outside comments and quotes with its parenthesis depth.""" - return iter_code_positions_impl(sql=sql, context=context) + return iter_code_positions_impl(sql=sql, context=context, syntax=syntax) diff --git a/src/sqlbuild/compiler/sql_analysis/models.py b/src/sqlbuild/compiler/sql_analysis/models.py index 2a2a74dafd..7ddbd859e5 100644 --- a/src/sqlbuild/compiler/sql_analysis/models.py +++ b/src/sqlbuild/compiler/sql_analysis/models.py @@ -6,6 +6,14 @@ from dataclasses import dataclass, field from typing import Any +from sqlbuild.compiler.sql_analysis.constants import ( + SQL_BLOCK_COMMENT_OPEN, + SQL_ESCAPABLE_QUOTE_CHARACTERS, + SQL_ESCAPE_CHARACTER, + SQL_LINE_COMMENT_PREFIX, + SQL_TRIPLE_QUOTE_LENGTH, +) + @dataclass(frozen=True) class SqlBindingDiagnostic: @@ -49,4 +57,38 @@ class SqlLexicalSyntax: raw_string_prefix: bool = False triple_quoted_strings: bool = False nested_block_comments: bool = False - line_comment_prefixes: frozenset[str] = frozenset({"--"}) + line_comment_prefixes: frozenset[str] = frozenset({SQL_LINE_COMMENT_PREFIX}) + + @property + def cache_key(self) -> str: + """Return a process-stable text key for these lexical rules.""" + + return "|".join( + ( + ",".join(sorted(self.backslash_escape_quotes)), + str(self.escape_string_prefix), + str(self.raw_string_prefix), + str(self.triple_quoted_strings), + str(self.nested_block_comments), + ",".join(sorted(self.line_comment_prefixes)), + ) + ) + + def reads_differently_from_generic(self, sql: str) -> bool: + """Return whether these rules can read the SQL differently from generic SQL.""" + + if SQL_ESCAPE_CHARACTER in sql and ( + self.backslash_escape_quotes or self.escape_string_prefix + ): + return True + if self.triple_quoted_strings and any( + quote * SQL_TRIPLE_QUOTE_LENGTH in sql for quote in SQL_ESCAPABLE_QUOTE_CHARACTERS + ): + return True + if self.nested_block_comments and sql.count(SQL_BLOCK_COMMENT_OPEN) > 1: + return True + return any( + prefix in sql + for prefix in self.line_comment_prefixes + if prefix != SQL_LINE_COMMENT_PREFIX + ) diff --git a/src/sqlbuild/lint/main/run_format.py b/src/sqlbuild/lint/main/run_format.py index ea8a98be5e..91075f5b48 100644 --- a/src/sqlbuild/lint/main/run_format.py +++ b/src/sqlbuild/lint/main/run_format.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.classes.fixture_null_autofix import FixtureNullAutofix +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.lint._helpers.fixes import finalize_fix_reports, persist_changes, plan_rule_fixes from sqlbuild.lint._helpers.headers import scan_headers from sqlbuild.lint._helpers.native import ( @@ -20,6 +21,7 @@ from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions from sqlbuild.lint.constants import VIOLATION_SEVERITY_WARNING +from sqlbuild.lint.exceptions import LintError from sqlbuild.lint.models import ( FormatChange, HeaderSpan, @@ -42,6 +44,8 @@ def run_format( ) -> LintRunResult: """Format all DSL files in place and report the violations that remain.""" + if discovered_inputs is not None and value_renderer is None: + raise LintError("fixture-null formatting requires the project adapter") files: dict[Path, str] = collect_project_files( project_dir=project_dir, selected_paths=selected_paths ) @@ -58,7 +62,11 @@ def run_format( files=safe_files, config=config, project_dir=project_dir, - discovered_inputs=discovered_inputs, + fixture_inputs=( + None + if discovered_inputs is None or value_renderer is None + else (discovered_inputs, value_renderer.sql_lexical_syntax) + ), fixtures_only=fixtures_only, ) format_faults.extend(rule_faults) @@ -105,7 +113,7 @@ def _apply_fixes( files: dict[Path, str], config: LintConfig, project_dir: Path, - discovered_inputs: DiscoveredProjectInputs | None, + fixture_inputs: tuple[DiscoveredProjectInputs, SqlLexicalSyntax] | None, fixtures_only: bool, ) -> tuple[dict[Path, str], list[LintViolation]]: """Return contents after native header and supported SQL body fixes.""" @@ -114,9 +122,10 @@ def _apply_fixes( FixtureNullAutofix.apply( files=files, project_dir=project_dir, - discovered_inputs=discovered_inputs, + discovered_inputs=fixture_inputs[0], + sql_lexical_syntax=fixture_inputs[1], ) - if discovered_inputs is not None + if fixture_inputs is not None else {} ) if fixtures_only: @@ -160,9 +169,10 @@ def _apply_fixes( FixtureNullAutofix.apply( files=post_native_files, project_dir=project_dir, - discovered_inputs=discovered_inputs, + discovered_inputs=fixture_inputs[0], + sql_lexical_syntax=fixture_inputs[1], ) - if discovered_inputs is not None + if fixture_inputs is not None else {} ) if fixture_formatted: diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 962190784c..4022e48b90 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -8,10 +8,10 @@ import time from concurrent.futures import Executor, Future, ThreadPoolExecutor from dataclasses import asdict, dataclass, replace -from importlib.metadata import version from pathlib import Path from typing import Any +import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -71,7 +71,7 @@ ) _SQL_RULE_CACHE_VERSION: str = "sql-rules-v5" -_SQLBUILD_VERSION: str = version("sqlbuild") +_RULES_BUILD_IDENTITY: str = _native.BUILD_IDENTITY _SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" @@ -697,7 +697,7 @@ def _sql_model_rule_identities( def _sql_rule_identity(*, model: CompiledModel, codes: tuple[str, ...], dialect: str) -> str: digest: Any = hashlib.sha256() digest.update(_SQL_RULE_CACHE_VERSION.encode()) - digest.update(_SQLBUILD_VERSION.encode()) + digest.update(_RULES_BUILD_IDENTITY.encode()) digest.update(dialect.encode()) digest.update("\0".join(codes).encode()) digest.update(model.authored_sql.encode()) @@ -735,7 +735,7 @@ def _sql_file_rule_identity( for value in ( _SQL_RULE_CACHE_VERSION, "file", - _SQLBUILD_VERSION, + _RULES_BUILD_IDENTITY, dialect, "\0".join(codes), relative_path, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 259876f0bf..095d87af2d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -98,6 +98,15 @@ class CappedMicrobatchScenarioE2ETestCase: expected_exit_code: int +@dataclass(frozen=True) +class CappedFilterJoinE2ETestCase: + """Expected rows from a consumer that filters a capped producer and joins its source table.""" + + description: str + limit_action: str + expected_ids: tuple[int, ...] + + @dataclass(frozen=True) class CappedWatermarkRejectionE2ETestCase: """Expected static rejection for a capped watermark producer.""" @@ -767,6 +776,17 @@ class AuditFailureBuildE2ETestCase: expected_retained_relation_fragment: str +@dataclass(frozen=True) +class CompileJsonRaisedErrorE2ETestCase: + """Expected JSON report for a compile that stops on a raised error.""" + + description: str + repo_files: dict[str, str] + expected_code: str + expected_message_fragment: str + expected_help_fragment: str | None + + @dataclass(frozen=True) class CompileJsonBuildE2ETestCase: """Test case for compile JSON behavior.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 2f47746609..60c5479ca0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -131,6 +131,39 @@ def capped_watermark_consumer_project_files(*, limit_action: str) -> dict[str, s return repo_files +def capped_filter_join_consumer_project_files(*, limit_action: str) -> dict[str, str]: + """Build a project whose microbatch consumer filters a capped producer and joins its source.""" + + repo_files: dict[str, str] = capped_microbatch_project_files(limit_action=limit_action) + repo_files["models/downstream_events.sql"] = ( + dedent( + """ + MODEL ( + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor event_time, + cursor_type timestamp, + cursor_grain day, + cursor_start '2026-01-01', + cursor_watermark_mode all, + cursor_inputs ( + capped_events (column event_time, roles [filter]), + raw_events (column event_time, roles [watermark]), + ), + batch_size 1d, + ); + SELECT capped.id, capped.event_time + FROM __ref("capped_events") AS capped + JOIN __source("raw_events") AS raw_events USING (id) + """ + ).strip() + + "\n" + ) + return repo_files + + def prepare_defer_clone_project( *, tmp_path: Path, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py index a56879249b..766986e419 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py @@ -8,11 +8,13 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( CalendarGrainCursorBoundaryE2ETestCase, + CappedFilterJoinE2ETestCase, CappedMicrobatchBuildE2ETestCase, CappedMicrobatchScenarioE2ETestCase, CappedWatermarkRejectionE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + capped_filter_join_consumer_project_files, capped_microbatch_project_files, capped_watermark_consumer_project_files, ) @@ -531,5 +533,51 @@ def test_given_capped_producer_as_watermark_when_building_then_plan_rejects_befo assert target_names.isdisjoint(test_case.expected_absent_relations) +@pytest.mark.parametrize( + "test_case", + ( + CappedFilterJoinE2ETestCase( + description="join using the shared id column of a real source table", + limit_action="cap_from_end", + expected_ids=(3, 4, 5), + ), + ), + ids=lambda case: case.description, +) +def test_given_capped_filter_consumer_joining_source_table_using_id_when_building_then_join_binds( + tmp_path: Path, test_case: CappedFilterJoinE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="capped_filter_join", + repo_files=capped_filter_join_consumer_project_files(limit_action=test_case.limit_action), + ) + db_path: Path = project_dir / "regression.duckdb" + + import duckdb + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(db_path)) + connection.execute("CREATE TABLE raw_events (id INTEGER, event_time TIMESTAMP)") + connection.execute( + "INSERT INTO raw_events VALUES " + "(1, '2026-01-01 12:00:00'), (2, '2026-01-02 12:00:00'), " + "(3, '2026-01-03 12:00:00'), (4, '2026-01-04 12:00:00'), " + "(5, '2026-01-05 12:00:00')" + ) + connection.close() + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "B002" not in result.stdout + result.stderr + rows: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql="SELECT id FROM main.downstream_events ORDER BY id", + ) + assert tuple(row[0] for row in rows) == test_case.expected_ids + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 736251b3b9..2cfca30829 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -11,6 +11,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( CompileJsonBuildE2ETestCase, + CompileJsonRaisedErrorE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( prepare_inline_project, @@ -221,3 +222,63 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m assert hooks_by_name["notify"]["relative_path"] == "hooks/python/notify.py" assert hooks_by_name["notify"]["definition_hash"] assert hooks_by_name["notify"]["version_hash"] + + +@pytest.mark.parametrize( + "test_case", + [ + CompileJsonRaisedErrorE2ETestCase( + description="unknown materialization stops compile", + repo_files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (materialized tablex);\n\nSELECT 1 AS order_id\n", + }, + expected_code="P001", + expected_message_fragment="unknown materialization 'tablex'", + expected_help_fragment=None, + ), + CompileJsonRaisedErrorE2ETestCase( + description="unknown on_schema_change stops compile with help", + repo_files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": ( + "MODEL (\n materialized incremental,\n incremental_strategy append,\n" + " on_schema_change append_columns,\n);\n\nSELECT 1 AS order_id\n" + ), + }, + expected_code="P001", + expected_message_fragment="unknown on_schema_change 'append_columns'", + expected_help_fragment="on_schema_change append_new_columns,", + ), + ], + ids=lambda case: case.description, +) +def test_given_raised_compile_error_when_running_compile_json_then_stdout_holds_one_diagnostic_report( + test_case: CompileJsonRaisedErrorE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="orders", repo_files=test_case.repo_files + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("compile", "--json"), project_dir=project_dir + ) + + assert result.returncode == 1, result.stdout + result.stderr + payload: dict[str, object] = json.loads(result.stdout) + assert payload["command"] == "compile" + assert payload["has_errors"] is True + assert payload["stopped"] is True + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], payload["diagnostics"]) + assert len(diagnostics) == 1 + diagnostic: dict[str, object] = diagnostics[0] + assert diagnostic["severity"] == "error" + assert diagnostic["code"] == test_case.expected_code + message: str = cast(str, diagnostic["message"]) + assert test_case.expected_message_fragment in message + assert f"error[{test_case.expected_code}]: {message}" in result.stderr + help_text: str = cast(str, diagnostic.get("help", "")) + assert ("help" in diagnostic) is (test_case.expected_help_fragment is not None) + assert (test_case.expected_help_fragment or "") in help_text + assert all(line.strip() in result.stderr for line in help_text.splitlines()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py index ec17edfe84..0d71df9d74 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py @@ -59,7 +59,7 @@ def test_given_serial_microbatch_when_building_then_state_table_is_not_used( "models/orders.sql": timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=1, - replay_policy="forward_only", + replay_policy="forward", ), }, ) @@ -217,7 +217,7 @@ def test_given_concurrent_ceiling_when_first_run_and_full_refresh_then_target_bo "models/orders.sql": timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=3, - replay_policy="forward_only", + replay_policy="forward", ), }, ) @@ -303,7 +303,7 @@ def test_given_recreated_target_when_building_then_old_completion_generation_is_ "models/orders.sql": timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=2, - replay_policy="forward_only", + replay_policy="forward", ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py index f124475940..757f96b935 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py @@ -59,7 +59,7 @@ def test_given_completion_write_failure_during_rebuild_when_retried_then_live_ta "models/orders.sql": timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=3, - replay_policy="forward_only", + replay_policy="forward", ), "adapters/failing_microbatch_state_duckdb.py": ( "import os\n" @@ -166,7 +166,7 @@ def test_given_concurrent_delta_audit_failure_when_fixed_then_rejected_partition "models/orders.sql": timestamp_microbatch_model_sql( value_expression="CAST(payload AS INTEGER)", batch_concurrency=3, - replay_policy="forward_only", + replay_policy="forward", extra_config=( "audits [expression_is_true (" 'name "positive_value", expression "value > 0", severity error, ' @@ -255,7 +255,7 @@ def test_given_concurrent_batches_when_audit_and_hooks_wrap_model_then_aggregate model_sql: str = timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=3, - replay_policy="forward_only", + replay_policy="forward", extra_config=( "pre_hooks [inline_sql(\"INSERT INTO main.hook_log VALUES ('pre')\")], " "post_hooks [inline_sql(\"INSERT INTO main.hook_log VALUES ('post')\")]," diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py index 1ce0fbeb69..d6ec40454b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py @@ -37,7 +37,7 @@ def test_given_ahead_of_frontier_partial_backfill_when_normal_run_resumes_then_l tmp_path=tmp_path, project_name="microbatch_future_backfill", database_name="future_backfill.duckdb", - replay_policy="forward_only", + replay_policy="forward", ) execute_duckdb( db_path=db_path, @@ -120,7 +120,7 @@ def test_given_failed_backfill_tail_when_source_tail_disappears_then_no_durable_ tmp_path=tmp_path, project_name="microbatch_backfill_tail", database_name="backfill_tail.duckdb", - replay_policy="forward_only", + replay_policy="forward", ) execute_duckdb( db_path=db_path, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 8c798b12bb..8d1d5a5a68 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -443,5 +443,5 @@ class RequireSqlAnalysisCase: repo_files: tuple[tuple[str, str], ...] command: tuple[str, ...] expected_returncode: int - expected_diagnostics: tuple[tuple[str, str, int], ...] + expected_diagnostics: tuple[tuple[str, str | None, int | None], ...] expected_text_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index bd7b20fb3c..a178d3dfa3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2584,8 +2584,11 @@ def resource_sql_project_files(files: tuple[tuple[str, str], ...]) -> dict[str, def require_sql_analysis_output( result: subprocess.CompletedProcess[str], -) -> tuple[tuple[tuple[str, str, int], ...], str]: - """Return `(code, path, line)` per JSON diagnostic and all JSON help, note and stderr text.""" +) -> tuple[tuple[tuple[str, str | None, int | None], ...], str]: + """Return `(code, path, line)` per JSON diagnostic and all JSON help, note and stderr text. + + A compile that stops on a raised error reports it without a path or line. + """ payload: dict[str, Any] = json.loads(result.stdout or '{"diagnostics": []}') diagnostics: list[dict[str, Any]] = payload["diagnostics"] @@ -2595,6 +2598,6 @@ def require_sql_analysis_output( parts.extend(item.get("notes", ())) text: str = "\n".join(parts) return ( - tuple((item["code"], item["path"], item["line"]) for item in diagnostics), + tuple((item["code"], item.get("path"), item.get("line")) for item in diagnostics), text, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py index c24177f4b4..c34877a32f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py @@ -104,7 +104,7 @@ ), command=_COMPILE, expected_returncode=1, - expected_diagnostics=(), + expected_diagnostics=(("D001", None, None),), expected_text_fragments=( "error[D001]:", "turns SQL analysis off with [settings] sql_analysis = false, which this project " @@ -123,7 +123,7 @@ ), command=_COMPILE, expected_returncode=1, - expected_diagnostics=(), + expected_diagnostics=(("D001", None, None),), expected_text_fragments=( "cannot set [settings] require_sql_analysis; it is a shared project policy that " "only sqlbuild_project.toml can set", @@ -141,7 +141,7 @@ ), command=_COMPILE, expected_returncode=1, - expected_diagnostics=(), + expected_diagnostics=(("P001", None, None),), expected_text_fragments=( "error[P001]: SQL syntax error in model 'order_lookup'", "= help: if this SQL is valid for your warehouse, skip SQL analysis for this " diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py index 52d9bcc933..3c267d704f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py @@ -1,4 +1,4 @@ -"""E2E coverage for rejecting unknown MODEL header, model config layer, and source keys.""" +"""E2E coverage for rejecting unknown MODEL header, model config layer, and source keys and values.""" from __future__ import annotations @@ -66,6 +66,58 @@ "did you mean 'type'?", ), ), + ModelHeaderKeyCompileCase( + description="unknown on_schema_change in MODEL header", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/fct_orders.sql": ( + "MODEL (\n materialized incremental,\n incremental_strategy append,\n" + " on_schema_change append_columns,\n);\n\nSELECT 1 AS order_id\n" + ), + }, + expected_fragments=( + "error[P001]", + "unknown on_schema_change 'append_columns'; valid values: append_new_columns, " + "fail, ignore, sync_all_columns", + "on_schema_change append_new_columns,", + ), + ), + ModelHeaderKeyCompileCase( + description="unknown replay_on_change in project defaults", + repo_files={ + "sqlbuild_project.toml": ( + _PROJECT_TOML + '\n[defaults]\nreplay_on_change = "everything"\n' + ), + "models/fct_orders.sql": ( + "MODEL (materialized incremental, incremental_strategy append);\n\n" + "SELECT 1 AS order_id\n" + ), + }, + expected_fragments=( + "error[D001]", + "[defaults] unknown replay_on_change 'everything'; valid values: forward, full, " + "bounded-", + 'replay_on_change = "bounded-14d"', + ), + ), + ModelHeaderKeyCompileCase( + description="misspelled dlt resource key", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": ( + "dlt_sources:\n - type: sql_database\n config:\n" + " credentials: duckdb:///inputs.duckdb\n resources:\n" + " - name: raw_orders\n table: orders\n" + " primary_keys: order_id\n" + ), + "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + }, + expected_fragments=( + "error[D006]", + "dlt sql_database resource 'raw_orders' has unknown keys: primary_keys", + "did you mean 'primary_key'?", + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py index 6d734d3b2a..7c5aa935ef 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py @@ -63,6 +63,24 @@ ), expected_stdout_fragment="PASS=1 FAIL=0", ), + ScenarioAuthoredCheckSqlE2ETestCase( + description="DuckDB escape string before the first relation marker", + scenario_sql=( + "SCENARIO ();\n\n" + "WITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS id, 10 AS amount\n" + " UNION ALL\n" + " SELECT 2 AS id, 5 AS amount\n" + "),\n" + "__assert__no_named_totals AS (\n" + " SELECT E'O\\'Brien' AS customer_name, total_amount\n" + ' FROM __ref("order_totals") WHERE total_amount <> 15\n' + ")\n" + "SELECT 1\n" + ), + expected_stdout_fragment="PASS=1 FAIL=0", + ), ], ids=lambda case: case.description, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py new file mode 100644 index 0000000000..994276ecfa --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py @@ -0,0 +1,65 @@ +"""E2E coverage that compile-step SQL scanning follows DuckDB escape-string rules.""" + +from __future__ import annotations + +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import SqlTestE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "escape_strings"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "escape_strings.duckdb"\n' + ), + "models/stg_orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + "models/order_names.sql": ( + "MODEL (materialized table);\n\n" + "SELECT E'O\\'Brien' AS customer_name, order_id\n" + 'FROM __ref("stg_orders")\n' + ), + "tests/unit/test_order_names.sql": ( + "TEST ();\n\n" + "WITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id),\n" + "__assert__named_orders AS (\n" + " SELECT E'O\\'Brien), (' AS note, customer_name\n" + ' FROM __ref("order_names")\n' + " WHERE customer_name <> 'O''Brien'\n" + ")\n" + "SELECT 1\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + ( + SqlTestE2ETestCase( + description="escape strings before model references and test markers", + expected_exit_code=0, + expected_stdout_fragment="PASS=1 FAIL=0", + ), + ), + ids=lambda case: case.description, +) +def test_given_duckdb_escape_strings_before_references_when_building_and_testing_then_succeeds( + test_case: SqlTestE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="escape_strings", repo_files=_PROJECT_FILES + ) + + built: CompletedProcess[str] = run_sqb(command=("--no-color", "build"), project_dir=project_dir) + tested: CompletedProcess[str] = run_sqb(command=("--no-color", "test"), project_dir=project_dir) + + assert built.returncode == test_case.expected_exit_code, built.stdout + built.stderr + assert tested.returncode == test_case.expected_exit_code, tested.stdout + tested.stderr + assert test_case.expected_stdout_fragment in tested.stdout, tested.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index c4773e46ec..b6c5142b08 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -19,6 +19,7 @@ from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.models import CompiledProject, DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.rule_engine.models import Rule from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( DynamicPivotRulesIntegrationTestCase, @@ -645,6 +646,52 @@ def test_given_selected_sql_rule_when_compiling_then_authored_diagnostic_blocks_ assert (tmp_path / "target" / "manifest.json").is_file() +@pytest.mark.parametrize( + "test_case", + [RulesIntegrationTestCase("SQL rule cache follows the native build", 1, "SQBRSQL004")], + ids=lambda case: case.description, +) +def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rules_rerun( + test_case: RulesIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', + encoding="utf-8", + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text( + 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n', + encoding="utf-8", + ) + command: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] + + cold_exit: int = main(command) + cold: dict[str, object] = json.loads(capsys.readouterr().out) + monkeypatch.setattr( + rules_module, + "_RULES_BUILD_IDENTITY", + f"{rules_module._RULES_BUILD_IDENTITY}-rebuilt", + ) + rebuilt_exit: int = main(command) + rebuilt: dict[str, object] = json.loads(capsys.readouterr().out) + warm_exit: int = main(command) + warm: dict[str, object] = json.loads(capsys.readouterr().out) + + assert cold_exit == rebuilt_exit == warm_exit == test_case.expected_exit_code + rebuilt_timings: dict[str, int] = cast(dict[str, int], rebuilt["compile_timings"]) + warm_timings: dict[str, int] = cast(dict[str, int], warm["compile_timings"]) + assert (rebuilt_timings["rule_cache_hits"], rebuilt_timings["rule_cache_misses"]) == (0, 1) + assert (warm_timings["rule_cache_hits"], warm_timings["rule_cache_misses"]) == (1, 0) + assert [item["code"] for item in cast(list[dict[str, object]], rebuilt["diagnostics"])] == [ + test_case.expected_code + ] + assert rebuilt["diagnostics"] == cold["diagnostics"] + + @pytest.mark.parametrize( "test_case", [RulesIntegrationTestCase("warm SQL rules track non-model file edits", 1, "SQBRSQL004")], @@ -735,12 +782,14 @@ def counting_build( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: scope_builds.append(tmp_path) return original_build( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, + sql_lexical_syntax=sql_lexical_syntax, compile_cache_dir=compile_cache_dir, ) @@ -795,12 +844,14 @@ def counting_build( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: scope_builds.append(tmp_path) return original_build( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, + sql_lexical_syntax=sql_lexical_syntax, compile_cache_dir=compile_cache_dir, ) @@ -1461,7 +1512,10 @@ def test_given_assertion_expected_dependency_when_compiling_then_compiler_reject assert exit_code == test_case.expected_exit_code captured: CaptureResult[str] = capsys.readouterr() - assert captured.out == "" + payload: dict[str, object] = json.loads(captured.out) + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], payload["diagnostics"]) + assert len(diagnostics) == 1 + assert test_case.expected_code in str(diagnostics[0]["message"]) assert test_case.expected_code in captured.err diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 8edeea1fde..debd19f9d3 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -42,6 +42,7 @@ value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, ) diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py index 9e10f3c489..98193b4a7e 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py @@ -3,7 +3,6 @@ from __future__ import annotations import threading -import time from pathlib import Path from textwrap import dedent from typing import Any @@ -22,10 +21,16 @@ run_concurrent_build, ) +_OVERLAP_TIMEOUT_SECONDS: float = 30.0 + class _TrackingDuckDbAdapter(DuckDbAdapter): - def __init__(self) -> None: + def __init__(self, *, overlap_target: int) -> None: self.track_delta_staging = False + self.overlap_target = overlap_target + self.overlap_wait_timed_out = False + self._overlap_reached = threading.Event() + self._serial_first_batch_models: set[str] = set() self.active_delta_staging = 0 self.max_active_delta_staging = 0 self.active_delta_models: dict[str, int] = {} @@ -53,7 +58,15 @@ def _execute(self, *, connection: Any, sql: str) -> Any: self.max_active_delta_models, len(self.active_delta_models), ) - time.sleep(0.05) + if self.active_delta_staging >= self.overlap_target: + self._overlap_reached.set() + # Each model stages its first batch serially before fanning out. + gated: bool = model_name in self._serial_first_batch_models + self._serial_first_batch_models.add(model_name) + if gated and not self._overlap_reached.wait(timeout=_OVERLAP_TIMEOUT_SECONDS): + with self._tracking_lock: + self.overlap_wait_timed_out = True + self._overlap_reached.set() try: return super()._execute(connection=connection, sql=sql) finally: @@ -167,7 +180,9 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(contents, encoding="utf-8") - adapter: _TrackingDuckDbAdapter = _TrackingDuckDbAdapter() + adapter: _TrackingDuckDbAdapter = _TrackingDuckDbAdapter( + overlap_target=test_case.expected_max_active_batches + ) db_path: Path = tmp_path / "test.duckdb" initial_project_files: dict[str, str] = { path: contents.replace("batch_concurrency 3", "batch_concurrency 1") @@ -232,6 +247,10 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( adapter=adapter, ) + assert not adapter.overlap_wait_timed_out, ( + f"only {adapter.max_active_delta_staging} delta stagings were ever active together; " + f"expected {test_case.expected_max_active_batches} within {_OVERLAP_TIMEOUT_SECONDS}s" + ) assert incremental_result.status == test_case.expected_status assert adapter.max_active_delta_staging == test_case.expected_max_active_batches assert adapter.max_active_delta_models == test_case.expected_max_active_models diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index d1e949528a..05e2bb5231 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -197,6 +197,29 @@ def capped_microbatch_intermediary_sql(*, input_role: str) -> str: ) +def capped_dependency_warehouse_files(*, project_dir: Path) -> dict[str, str]: + """Create the capped graph's raw source table and return project files that read it.""" + + database_path: Path = project_dir / "warehouse.duckdb" + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(database_path)) + try: + connection.execute( + "CREATE TABLE main.raw_events AS " + "SELECT 1 AS id, TIMESTAMP '2026-01-01 00:00:00' AS event_time" + ) + finally: + connection.close() + return { + "sqlbuild_project.toml": ( + 'name = "capped_dependency"\nadapter = "duckdb"\n\n' + f"[connection]\ndatabase = {json.dumps(str(database_path))}\n" + ), + "sources/raw.yml": ( + "sources:\n - name: raw_events\n schema: main\n table: raw_events\n" + ), + } + + def compile_capped_microbatch_intermediary_project( *, project_dir: Path, adapter: DuckDbAdapter, input_role: str ) -> CompilePipelineResult: @@ -205,11 +228,7 @@ def compile_capped_microbatch_intermediary_project( write_build_project_files( project_dir=project_dir, project_files={ - "sqlbuild_project.toml": 'name = "capped_dependency"\nadapter = "duckdb"\n', - "sources/raw.yml": ( - "sources:\n - name: raw_events\n expression: " - "SELECT 1 AS id, TIMESTAMP '2026-01-01 00:00:00' AS event_time\n" - ), + **capped_dependency_warehouse_files(project_dir=project_dir), "models/capped_events.sql": capped_dependency_producer_sql( action=MicrobatchLimitAction.CAP_FROM_END ), @@ -239,11 +258,7 @@ def compile_capped_dependency_project( """Compile a capped-producer project with optional intermediary views.""" project_files: dict[str, str] = { - "sqlbuild_project.toml": 'name = "capped_dependency"\nadapter = "duckdb"\n', - "sources/raw.yml": ( - "sources:\n - name: raw_events\n expression: " - "SELECT 1 AS id, TIMESTAMP '2026-01-01 00:00:00' AS event_time\n" - ), + **capped_dependency_warehouse_files(project_dir=project_dir), "models/capped_events.sql": capped_dependency_producer_sql(action=action), "models/downstream_events.sql": consumer_sql, } diff --git a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py index fdf66c1377..95931ffd37 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py @@ -14,6 +14,7 @@ value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 68593ec23b..eb7202473b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -109,6 +109,15 @@ class MainTestCase: expected_dbt_init_skip_dbt_debug: bool = False +@dataclass(frozen=True) +class CompileJsonErrorRenderingTestCase: + description: str + argv: list[str] + error_factory: Callable[[], Exception] + expected_stdout_diagnostics: tuple[dict[str, object], ...] + expected_stderr_fragment: str + + @dataclass(frozen=True) class MainErrorRenderingTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 2c0e47f625..98ea5b9414 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import sys from collections.abc import Callable from pathlib import Path @@ -43,6 +44,7 @@ from sqlbuild.spec.contracts.exceptions import SpecConfigError from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( AuditAggregateRenderingTestCase, + CompileJsonErrorRenderingTestCase, MainErrorRenderingTestCase, MainTestCase, SkillFreshnessNoticeTestCase, @@ -2336,6 +2338,68 @@ def run_query( assert rendered_stderr.endswith("\n\n") +@pytest.mark.parametrize( + "test_case", + [ + CompileJsonErrorRenderingTestCase( + description="compile input error with help", + argv=["--project-dir", "/tmp/demo", "compile", "--json"], + error_factory=lambda: CompileInputError( + "model 'orders': unknown on_schema_change 'append_columns'", + help="use a valid on_schema_change", + ), + expected_stdout_diagnostics=( + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "model 'orders': unknown on_schema_change 'append_columns'", + "help": "use a valid on_schema_change", + }, + ), + expected_stderr_fragment=( + "error[P001]: model 'orders': unknown on_schema_change 'append_columns'" + ), + ), + CompileJsonErrorRenderingTestCase( + description="plain value error uses the fallback code", + argv=["--project-dir", "/tmp/demo", "compile", "--json"], + error_factory=lambda: ValueError("invalid compile request"), + expected_stdout_diagnostics=( + { + "phase": "compile", + "severity": "error", + "code": "E001", + "message": "invalid compile request", + }, + ), + expected_stderr_fragment="error[E001]: invalid compile request", + ), + ], + ids=lambda case: case.description, +) +def test_given_raised_compile_error_when_running_main_then_json_mode_reports_one_diagnostic_document( + test_case: CompileJsonErrorRenderingTestCase, + capsys: pytest.CaptureFixture[str], +) -> None: + def run_compile(request: CompileCommandRequest) -> int: + del request + raise test_case.error_factory() + + exit_code: int = _main_with_dependencies( + argv=test_case.argv, handlers=build_handlers(run_compile=run_compile) + ) + captured: CaptureResult[str] = capsys.readouterr() + payload: dict[str, object] = json.loads(captured.out) + + assert exit_code == 1 + assert test_case.expected_stderr_fragment in captured.err + assert payload["command"] == "compile" + assert payload["has_errors"] is True + assert payload["stopped"] is True + assert tuple(payload["diagnostics"]) == test_case.expected_stdout_diagnostics + + @pytest.mark.parametrize( "test_case", ( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 6378766475..cae1745351 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -18,6 +18,7 @@ ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import UsageRecord +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.sql_values.models import SqlValue @@ -35,6 +36,23 @@ class SqlReferenceExtractionTestCase: expected_references: tuple[tuple[SqlReferenceKind, str, str | None], ...] +@dataclass(frozen=True) +class DialectCteScanTestCase: + description: str + syntax: SqlLexicalSyntax + sql: str + expected_cte_names: tuple[str, ...] + expected_assertion_targets: tuple[str, ...] + + +@dataclass(frozen=True) +class DialectSqlScanTestCase: + description: str + syntax: SqlLexicalSyntax + sql: str + expected_names: tuple[str, ...] + + @dataclass(frozen=True) class SqlReferenceExtractionErrorTestCase: description: str @@ -608,6 +626,14 @@ class IncrementalConfigErrorTestCase: expected_error_fragment: str +@dataclass(frozen=True) +class ChangePolicyHelpTestCase: + description: str + key: str + value: str + expected_help: str + + @dataclass(frozen=True) class ContractConfigValidTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index e6a967255d..52e442f5dd 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -69,11 +69,13 @@ value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, ) DUCKDB_ARRAY_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DUCKDB_COMPILE_ADAPTER_CONTEXT.value_renderer, collection_rendering=CollectionRendering.ARRAY, python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, ) DUCKDB_DECLARATION_EXPANSION_CONTEXT: DeclarationExpansionContext = DeclarationExpansionContext( declarations=DeclarationResolutionContext(), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py index 78559b4c50..ed93ad2b24 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py @@ -8,10 +8,13 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlTestCte from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ClassifyDirectLogicSqlTestCtesErrorTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + _FILE: str = "tests/unit/orders.sql" @@ -281,6 +284,8 @@ def test_given_invalid_direct_logic_ctes_when_classifying_then_raises_mode_speci ) with pytest.raises(CompileInputError) as error_info: - _ = classify_sql_test_ctes(ctes=ctes, file_label=_FILE, mode=test_case.mode) + _ = classify_sql_test_ctes( + ctes=ctes, file_label=_FILE, mode=test_case.mode, syntax=_GENERIC_SQL_SYNTAX + ) assert str(error_info.value) == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py index 1377c29cfb..6daaf4791a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py @@ -8,10 +8,13 @@ from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( split_set_operation_branches, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedProjectionScanTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -112,11 +115,17 @@ def test_given_expected_cte_sql_when_scanning_top_level_then_results_are_charact == test_case.expected_branches ) assert ( - sql_test_core._find_select_list_end(sql=test_case.sql, start=0) + sql_test_core._find_select_list_end(sql=test_case.sql, start=0, syntax=_GENERIC_SQL_SYNTAX) == test_case.expected_select_list_end ) - assert sql_test_core._split_top_level_commas(test_case.sql) == test_case.expected_commas - assert sql_test_core._extract_as_alias(test_case.sql) == test_case.expected_alias - assert sql_test_core._contains_select_star(test_case.sql) is ( + assert ( + sql_test_core._split_top_level_commas(raw_value=test_case.sql, syntax=_GENERIC_SQL_SYNTAX) + == test_case.expected_commas + ) + assert ( + sql_test_core._extract_as_alias(expression=test_case.sql, syntax=_GENERIC_SQL_SYNTAX) + == test_case.expected_alias + ) + assert sql_test_core._contains_select_star(sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX) is ( test_case.expected_contains_select_star ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py index 178450d3c5..ef3f2a1c28 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py @@ -21,6 +21,7 @@ from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.spec.contracts.types import TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + ChangePolicyHelpTestCase, ContractConfigErrorTestCase, ContractConfigValidTestCase, CustomMaterializationConfigErrorTestCase, @@ -37,6 +38,8 @@ SnapshotConfigValidTestCase, ) +_SNIPPET_INDENT: str = " " * 12 + @pytest.mark.parametrize( "test_case", @@ -354,7 +357,43 @@ def test_given_valid_config_when_validating_then_passes( "replay_on_change": {"add_column": "bounded-30d"}, }, ref_count=1, - expected_error_fragment="replay_on_change must be a string", + expected_error_fragment="replay_on_change must be a string; valid values: forward", + ), + IncrementalConfigErrorTestCase( + description="unknown replay_on_change raises with valid values", + config_values={ + "materialized": "incremental", + "incremental_strategy": "append", + "replay_on_change": "bounded", + }, + ref_count=1, + expected_error_fragment=( + "unknown replay_on_change 'bounded'; valid values: forward, full, " + "bounded-" + ), + ), + IncrementalConfigErrorTestCase( + description="bounded replay_on_change without valid duration raises", + config_values={ + "materialized": "incremental", + "incremental_strategy": "append", + "replay_on_change": "bounded-two-weeks", + }, + ref_count=1, + expected_error_fragment="has an invalid duration 'two-weeks'", + ), + IncrementalConfigErrorTestCase( + description="unknown on_schema_change raises with valid values", + config_values={ + "materialized": "incremental", + "incremental_strategy": "append", + "on_schema_change": "append_columns", + }, + ref_count=1, + expected_error_fragment=( + "unknown on_schema_change 'append_columns'; valid values: append_new_columns, " + "fail, ignore, sync_all_columns" + ), ), IncrementalConfigErrorTestCase( description="cursor without cursor_type raises", @@ -780,6 +819,59 @@ def test_given_invalid_config_when_validating_then_raises( ) +@pytest.mark.parametrize( + "test_case", + [ + ChangePolicyHelpTestCase( + description="on_schema_change", + key="on_schema_change", + value="append", + expected_help=( + "use a valid on_schema_change, add this to the MODEL header:\n" + f"{_SNIPPET_INDENT}MODEL (\n" + f"{_SNIPPET_INDENT} on_schema_change append_new_columns,\n" + f"{_SNIPPET_INDENT} ...\n" + f"{_SNIPPET_INDENT});" + ), + ), + ChangePolicyHelpTestCase( + description="replay_on_change", + key="replay_on_change", + value="everything", + expected_help=( + "use a valid replay_on_change, add this to the MODEL header:\n" + f"{_SNIPPET_INDENT}MODEL (\n" + f"{_SNIPPET_INDENT} replay_on_change bounded-14d,\n" + f"{_SNIPPET_INDENT} ...\n" + f"{_SNIPPET_INDENT});" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_change_policy_when_validating_then_help_shows_valid_header_line( + test_case: ChangePolicyHelpTestCase, +) -> None: + config: CompileModelConfig = CompileModelConfig( + values={ + "materialized": "incremental", + "incremental_strategy": "append", + test_case.key: test_case.value, + } + ) + + with pytest.raises(CompileInputError) as error_info: + validate_incremental_config( + config=config, + model_name="test_model", + ref_count=1, + known_input_names=frozenset({"orders"}), + ) + + assert f"unknown {test_case.key} '{test_case.value}'" in str(error_info.value) + assert error_info.value.help == test_case.expected_help + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py index 85ef3116ac..9a0672b680 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py @@ -2,15 +2,27 @@ import pytest -from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_sql_test_ctes +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.compiler.compile._helpers.sql_tests.core import ( + extract_assertion_target_model_names, + extract_sql_test_ctes, +) from sqlbuild.compiler.compile._helpers.sql_tests.native import extract_expanded_sql_tests -from sqlbuild.compiler.compile.models import CompileSqlTestCtes +from sqlbuild.compiler.compile.models import CompileModelSqlTestCtes, CompileSqlTestCtes from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + DialectCteScanTestCase, ExpectedBooleanTestCase, NativeSqlTestExtractionParityTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -89,13 +101,15 @@ def test_given_expanded_sql_tests_when_native_batch_extracting_then_matches_refe test_case: NativeSqlTestExtractionParityTestCase, ) -> None: expected: CompileSqlTestCtes = extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/unit/example.sql", mode=test_case.mode, ) actual: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - ((test_case.sql, "tests/unit/example.sql", test_case.mode),) + tests=((test_case.sql, "tests/unit/example.sql", test_case.mode),), + syntax=_GENERIC_SQL_SYNTAX, ) assert (actual == (expected,)) is test_case.expected_matches @@ -117,13 +131,17 @@ def test_given_cross_check_dependency_when_native_batch_extracting_then_matches_ ) with pytest.raises(ValueError) as reference_error: extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=sql, file_label="tests/unit/example.sql", mode=SqlTestMode.MODEL, ) with pytest.raises(ValueError) as native_error: - extract_expanded_sql_tests(((sql, "tests/unit/example.sql", SqlTestMode.MODEL),)) + extract_expanded_sql_tests( + tests=((sql, "tests/unit/example.sql", SqlTestMode.MODEL),), + syntax=_GENERIC_SQL_SYNTAX, + ) assert (str(native_error.value) == str(reference_error.value)) is test_case.expected_result @@ -150,12 +168,129 @@ def test_given_comma_separated_cross_check_dependency_when_extracting_then_nativ ) with pytest.raises(ValueError) as reference_error: extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=sql, file_label="tests/unit/example.sql", mode=SqlTestMode.MODEL, ) with pytest.raises(ValueError) as native_error: - extract_expanded_sql_tests(((sql, "tests/unit/example.sql", SqlTestMode.MODEL),)) + extract_expanded_sql_tests( + tests=((sql, "tests/unit/example.sql", SqlTestMode.MODEL),), + syntax=_GENERIC_SQL_SYNTAX, + ) assert (str(native_error.value) == str(reference_error.value)) is test_case.expected_result + + +@pytest.mark.parametrize( + "test_case", + [ + DialectCteScanTestCase( + description="snowflake backslash-escaped quote before a marker", + syntax=SnowflakeAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="bigquery hash comment and triple quotes around markers", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT '''it's ), (''' AS note), # ), (\n" + "__assert__named AS (\n" + ' SELECT * FROM __ref("order_totals") # __ref("ignored")\n' + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="databricks backslash-escaped quote before a marker", + syntax=DatabricksAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="duckdb escape string and nested comment before a marker", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" + "/* outer /* inner */ ), ( */\n" + "__assert__named AS (\n" + " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="postgres escape string before a marker", + syntax=PostgresAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_sql_test_when_batch_extracting_then_follows_dialect_lexical_rules( + test_case: DialectCteScanTestCase, +) -> None: + extracted: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( + tests=( + (test_case.sql, "tests/unit/orders.sql", SqlTestMode.MODEL), + ( + "WITH __ref__a AS (SELECT 1 AS id), __expected__b AS (SELECT 1 AS id) SELECT 1", + "tests/unit/plain.sql", + SqlTestMode.MODEL, + ), + ), + syntax=test_case.syntax, + ) + + payload: object = extracted[0].payload + assert isinstance(payload, CompileModelSqlTestCtes) + assert ( + tuple(cte.name for cte in (*payload.authored_ctes, *payload.assertion_ctes)) + == test_case.expected_cte_names + ) + assert ( + extract_assertion_target_model_names( + assertion_sql=tuple(cte.sql_body for cte in payload.assertion_ctes), + syntax=test_case.syntax, + ) + == test_case.expected_assertion_targets + ) + assert isinstance(extracted[1].payload, CompileModelSqlTestCtes) + assert extracted[1].payload.expected_model_names == ("b",) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 3c31f88dd1..ff1fe2a463 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -17,6 +17,7 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.models import DeclarationIdentity from sqlbuild.compiler.scopes.types import DeclarationKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( RebindDeclarationScopeTestCase, ) @@ -24,6 +25,8 @@ write_scoped_macro_orders_project, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -44,6 +47,7 @@ def test_given_same_discovery_when_rebinding_private_macros_then_index_matches_a write_scoped_macro_orders_project(tmp_path) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) shared: DeclarationScopeBuild = build_declaration_scope( + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=discovered, loaded_macros=load_project_macros(discovered.macro_files), ) @@ -56,7 +60,9 @@ def test_given_same_discovery_when_rebinding_private_macros_then_index_matches_a assert (rebound is not None) is test_case.expected_rebound assert rebound is not None fresh: DeclarationScopeBuild = build_declaration_scope( - discovered_inputs=discovered, loaded_macros=private_macros + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, + discovered_inputs=discovered, + loaded_macros=private_macros, ) identity: DeclarationIdentity = DeclarationIdentity(DeclarationKind.MACRO, test_case.macro_name) private_value: object = rebound.resolver.projection.declarations[identity] @@ -88,6 +94,7 @@ def test_given_different_private_macro_metadata_when_rebinding_then_full_build_i write_scoped_macro_orders_project(tmp_path) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) shared: DeclarationScopeBuild = build_declaration_scope( + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=discovered, loaded_macros=load_project_macros(discovered.macro_files), ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py index 3b191b7ba2..f24b85846a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py @@ -4,11 +4,17 @@ import pytest +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile._helpers.attachment.sql_tests import build_scenario_inputs from sqlbuild.compiler.compile._helpers.scenarios.core import ( extract_sql_scenario_ctes, extract_sql_scenario_expected_model_names, ) +from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_assertion_target_model_names from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileSqlScenarioCtes, @@ -20,11 +26,13 @@ DiscoveredSourceFile, DiscoveredSqlScenarioFile, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( BuildScenarioInputsErrorTestCase, BuildScenarioInputsTestCase, CteScannerMessageTestCase, + DialectCteScanTestCase, ExtractSqlScenarioCtesErrorTestCase, ExtractSqlScenarioCtesTestCase, ) @@ -32,6 +40,8 @@ DUCKDB_DECLARATION_EXPANSION_CONTEXT, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -145,6 +155,7 @@ def test_given_sql_scenario_cte_variants_when_extracting_then_it_returns_expecte test_case: ExtractSqlScenarioCtesTestCase, ) -> None: extracted_ctes: CompileSqlScenarioCtes = extract_sql_scenario_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/scenarios/revenue__customer_refund.sql", ) @@ -275,6 +286,7 @@ def test_given_invalid_sql_scenario_ctes_when_extracting_then_it_raises_clear_er ) -> None: with pytest.raises(ValueError, match=test_case.expected_error_fragment): extract_sql_scenario_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/scenarios/revenue__customer_refund.sql", ) @@ -362,6 +374,7 @@ def test_given_discovered_scenario_when_building_scenario_inputs_then_it_attache ) scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=discovered_inputs, effective_vars=test_case.effective_vars, macro_context=MacroContext( @@ -436,6 +449,7 @@ def test_given_invalid_scenario_source_refs_when_building_inputs_then_it_raises_ with pytest.raises(ValueError, match=test_case.expected_error_fragment): build_scenario_inputs( + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=discovered_inputs, effective_vars={}, macro_context=MacroContext( @@ -502,7 +516,9 @@ def test_given_malformed_scenario_ctes_when_extracting_then_messages_name_the_sc test_case: CteScannerMessageTestCase, ) -> None: with pytest.raises(CompileInputError) as error_info: - _ = extract_sql_scenario_ctes(sql=test_case.sql, file_label="tests/scenarios/orders.sql") + _ = extract_sql_scenario_ctes( + sql=test_case.sql, file_label="tests/scenarios/orders.sql", syntax=_GENERIC_SQL_SYNTAX + ) assert str(error_info.value) == test_case.expected_message @@ -544,7 +560,110 @@ def test_given_malformed_scenario_ctes_when_scanning_expected_models_then_names_ ) -> None: with pytest.raises(CompileInputError) as error_info: _ = extract_sql_scenario_expected_model_names( - sql=test_case.sql, file_label="tests/scenarios/orders.sql" + sql=test_case.sql, + file_label="tests/scenarios/orders.sql", + syntax=_GENERIC_SQL_SYNTAX, ) assert str(error_info.value) == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + DialectCteScanTestCase( + description="snowflake backslash-escaped quote before a marker", + syntax=SnowflakeAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="bigquery hash comment and triple quotes around markers", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT '''it's ), (''' AS note), # ), (\n" + "__assert__named AS (\n" + ' SELECT * FROM __ref("order_totals") # __ref("ignored")\n' + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="databricks backslash-escaped quote before a marker", + syntax=DatabricksAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="duckdb escape string and nested comment before a marker", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" + "/* outer /* inner */ ), ( */\n" + "__assert__named AS (\n" + " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="postgres escape string before a marker", + syntax=PostgresAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_scenario_sql_when_extracting_ctes_then_follows_dialect_lexical_rules( + test_case: DialectCteScanTestCase, +) -> None: + extracted: CompileSqlScenarioCtes = extract_sql_scenario_ctes( + sql=test_case.sql, file_label="tests/scenarios/orders.sql", syntax=test_case.syntax + ) + + assert ( + tuple(cte.name for cte in (*extracted.authored_ctes, *extracted.assertion_ctes)) + == test_case.expected_cte_names + ) + assert ( + extract_assertion_target_model_names( + assertion_sql=tuple(cte.sql_body for cte in extracted.assertion_ctes), + syntax=test_case.syntax, + ) + == test_case.expected_assertion_targets + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py index 123f8a652d..d81aede976 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py @@ -10,11 +10,14 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlTestCte, CompileSqlTestCtes from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( SetOperationExpectedKind, SetOperationExpectedTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + _FILE: str = "tests/unit/orders.sql" _MISMATCH: str = ( "SQL test '{file}' must use the same {cte} projection names and order in every " @@ -99,7 +102,7 @@ def test_given_set_operation_expected_cte_when_classifying_then_it_is_accepted( ) classified: CompileSqlTestCtes = classify_sql_test_ctes( - ctes=ctes, file_label=_FILE, mode=test_case.kind.mode + ctes=ctes, file_label=_FILE, mode=test_case.kind.mode, syntax=_GENERIC_SQL_SYNTAX ) assert type(classified.payload).__name__ == test_case.expected_payload_type @@ -130,7 +133,9 @@ def test_given_invalid_set_operation_branch_when_classifying_then_it_names_the_e ) with pytest.raises(CompileInputError) as error_info: - _ = classify_sql_test_ctes(ctes=ctes, file_label=_FILE, mode=test_case.kind.mode) + _ = classify_sql_test_ctes( + ctes=ctes, file_label=_FILE, mode=test_case.kind.mode, syntax=_GENERIC_SQL_SYNTAX + ) assert str(error_info.value) == test_case.expected_error_template.format( file=_FILE, cte=test_case.kind.expected_cte_name diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index 4d13a8b42b..4148fbc475 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -4,15 +4,24 @@ import pytest +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + DialectSqlScanTestCase, SqlReferenceExtractionErrorTestCase, SqlReferenceExtractionTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -51,7 +60,9 @@ def test_given_simple_references_when_extracting_then_returns_authored_order( test_case: SqlReferenceExtractionTestCase, ) -> None: - references: tuple[CompileSqlReference, ...] = extract_sql_references(test_case.sql) + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX + ) assert ( tuple( @@ -97,4 +108,90 @@ def test_given_unsupported_reference_sql_when_extracting_then_preserves_python_d test_case: SqlReferenceExtractionErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error): - extract_sql_references(test_case.sql) + extract_sql_references(sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX) + + +@pytest.mark.parametrize( + "test_case", + [ + DialectSqlScanTestCase( + description="snowflake backslash-escaped quote before a reference", + syntax=SnowflakeAdapter.sql_lexical_syntax, + sql="SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="bigquery backslash-escaped quote before a reference", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql="SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="databricks backslash-escaped quote before a reference", + syntax=DatabricksAdapter.sql_lexical_syntax, + sql="SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="duckdb escape string before a reference", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql="SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="postgres escape string before a reference", + syntax=PostgresAdapter.sql_lexical_syntax, + sql="SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="duckdb plain string keeps a literal backslash", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql="SELECT 'C:\\' AS folder, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="databricks raw string keeps a literal backslash", + syntax=DatabricksAdapter.sql_lexical_syntax, + sql="SELECT r'C:\\' AS folder, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="bigquery triple-quoted string hides a reference", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql="SELECT '''it's __ref(\"ignored\")''' AS note FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="bigquery hash comment hides a reference", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql='SELECT 1 # __ref("ignored")\nFROM __ref("orders")', + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="snowflake slash comment hides a reference", + syntax=SnowflakeAdapter.sql_lexical_syntax, + sql='SELECT 1 // __ref("ignored")\nFROM __ref("orders")', + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="duckdb nested block comment hides a reference", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql='SELECT 1 /* outer /* inner */ __ref("ignored") */ FROM __ref("orders")', + expected_names=("orders",), + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_sql_when_extracting_references_then_follows_dialect_lexical_rules( + test_case: DialectSqlScanTestCase, +) -> None: + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=test_case.sql, syntax=test_case.syntax + ) + + assert tuple(reference.ref_name for reference in references) == test_case.expected_names + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py index 4f2e2d17ba..8071264867 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py @@ -13,12 +13,15 @@ CompileModelSqlTestCtes, ) from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CteScannerMessageTestCase, ExtractSqlTestCtesErrorTestCase, ExtractSqlTestCtesTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -336,6 +339,7 @@ def test_given_model_sql_test_cte_variants_when_extracting_then_it_returns_expec test_case: ExtractSqlTestCtesTestCase, ) -> None: extracted_ctes: CompileSqlTestCtes = extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode, @@ -422,6 +426,7 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return test_case: ExtractSqlTestCtesTestCase, ) -> None: extracted_ctes: CompileSqlTestCtes = extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode, @@ -832,6 +837,7 @@ def test_given_invalid_sql_test_cte_variants_when_extracting_then_it_raises_clea ) -> None: with pytest.raises(ValueError, match=test_case.expected_error_fragment): extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode, @@ -873,7 +879,9 @@ def test_given_malformed_sql_test_ctes_when_extracting_then_messages_name_the_sq test_case: CteScannerMessageTestCase, ) -> None: with pytest.raises(CompileInputError) as error_info: - _ = extract_sql_test_ctes(sql=test_case.sql, file_label="tests/unit/orders.sql") + _ = extract_sql_test_ctes( + sql=test_case.sql, file_label="tests/unit/orders.sql", syntax=_GENERIC_SQL_SYNTAX + ) assert str(error_info.value) == test_case.expected_message @@ -904,7 +912,7 @@ def test_given_malformed_sql_test_ctes_when_scanning_expected_models_then_names_ ) -> None: with pytest.raises(CompileInputError) as error_info: _ = extract_sql_test_expected_model_names( - sql=test_case.sql, file_label="tests/unit/orders.sql" + sql=test_case.sql, file_label="tests/unit/orders.sql", syntax=_GENERIC_SQL_SYNTAX ) assert str(error_info.value) == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 91fe3a1b76..b8a1d47041 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -329,6 +329,14 @@ class LoadProjectConfigErrorTestCase: expected_error_fragment: str +@dataclass(frozen=True) +class LoadProjectConfigHelpTestCase: + description: str + project_file_contents: str + expected_error_fragment: str + expected_help: str + + @dataclass(frozen=True) class LoadProjectCostConfigTestCase: description: str @@ -722,6 +730,14 @@ class ParseSourcesYamlErrorTestCase: expected_error_fragment: str +@dataclass(frozen=True) +class ParseSourcesYamlHelpTestCase: + description: str + contents: str + expected_error_fragment: str + expected_help: str + + @dataclass(frozen=True) class ParseSourcesYamlIngestrTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index fca7e0c412..9e1cd99b0d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -31,6 +31,7 @@ LoadLocalConfigTestCase, LoadNamedConnectionsTestCase, LoadProjectConfigErrorTestCase, + LoadProjectConfigHelpTestCase, LoadProjectConfigTestCase, LoadProjectConstantsConfigErrorTestCase, LoadProjectConstantsConfigTestCase, @@ -1766,6 +1767,34 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed( """.strip(), expected_error_fragment=r"option\(s\) were removed: replay_on_change", ), + LoadProjectConfigErrorTestCase( + description="raises for unknown replay_on_change in defaults", + project_file_contents=""" +name = "demo" +adapter = "duckdb" + +[defaults] +replay_on_change = "always" +""".strip(), + expected_error_fragment=( + r"\[defaults\] unknown replay_on_change 'always'; valid values: forward, full, " + r"bounded-" + ), + ), + LoadProjectConfigErrorTestCase( + description="raises for unknown on_schema_change in path defaults", + project_file_contents=""" +name = "demo" +adapter = "duckdb" + +[path_defaults.marts] +on_schema_change = "sync" +""".strip(), + expected_error_fragment=( + r"\[path_defaults.\"marts\"\] unknown on_schema_change 'sync'; valid values: " + r"append_new_columns, fail, ignore, sync_all_columns" + ), + ), LoadProjectConfigErrorTestCase( description="raises targeted error for removed dbt production_ref reuse config", project_file_contents=""" @@ -1802,6 +1831,37 @@ def test_given_invalid_project_config_file_when_loading_project_config_then_it_r load_project_config(project_dir=tmp_path) +@pytest.mark.parametrize( + "test_case", + [ + LoadProjectConfigHelpTestCase( + description="bounded replay without a valid duration in defaults", + project_file_contents=( + 'name = "demo"\nadapter = "duckdb"\n\n[defaults]\nreplay_on_change = "bounded-x"\n' + ), + expected_error_fragment="replay_on_change 'bounded-x' has an invalid duration 'x'", + expected_help=( + "use a valid replay_on_change, set this in sqlbuild_project.toml:\n" + " [defaults]\n" + ' replay_on_change = "bounded-14d"' + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_change_policy_in_defaults_when_loading_then_help_shows_toml_line( + test_case: LoadProjectConfigHelpTestCase, tmp_path: Path +) -> None: + project_file: Path = tmp_path / "sqlbuild_project.toml" + project_file.write_text(test_case.project_file_contents, encoding="utf-8") + + with pytest.raises(ProjectConfigError) as error_info: + load_project_config(project_dir=tmp_path) + + assert test_case.expected_error_fragment in str(error_info.value) + assert error_info.value.help == test_case.expected_help + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index 45d68ea8ce..8ff38e7023 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -6,6 +6,7 @@ import pytest from sqlbuild.compiler.discovery._helpers.yml.sources import parse_sources_yml +from sqlbuild.compiler.discovery.exceptions import SourceParseError from sqlbuild.integrations.dlt.models import DltSourceConfig from sqlbuild.integrations.ingestr.models import IngestrSourceConfig from sqlbuild.spec.contracts.models import SourceEntry @@ -13,6 +14,7 @@ from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSourcesYamlDltTestCase, ParseSourcesYamlErrorTestCase, + ParseSourcesYamlHelpTestCase, ParseSourcesYamlIngestrTestCase, ParseSourcesYamlTestCase, ) @@ -787,6 +789,50 @@ def test_given_dlt_sources_yaml_when_parsing_then_expands_managed_sources( """, expected_error_fragment="dlt source destination cannot define 'credentials'", ), + ParseSourcesYamlErrorTestCase( + description="raises when dlt source group has an unknown key", + contents=""" + dlt_sources: + - type: sql_database + confg: + credentials: duckdb:///source.duckdb + resources: + - name: raw_orders + table: orders + """, + expected_error_fragment="dlt source has unknown keys: confg", + ), + ParseSourcesYamlErrorTestCase( + description="raises when dlt resource has an unknown key", + contents=""" + dlt_sources: + - type: sql_database + config: + credentials: duckdb:///source.duckdb + resources: + - name: raw_orders + table: orders + primary_keys: id + """, + expected_error_fragment=( + "dlt sql_database resource 'raw_orders' has unknown keys: primary_keys" + ), + ), + ParseSourcesYamlErrorTestCase( + description="raises when dlt resource uses a key of another source type", + contents=""" + dlt_sources: + - type: sql_database + config: + credentials: duckdb:///source.duckdb + resources: + - name: raw_orders + table: orders + endpoint: + path: orders + """, + expected_error_fragment="dlt sql_database resource 'raw_orders' has unknown keys: endpoint", + ), ParseSourcesYamlErrorTestCase( description="raises when write strategy is unknown", contents=""" @@ -1290,3 +1336,38 @@ def test_given_invalid_sources_yaml_when_parsing_then_it_raises_clear_errors( ) -> None: with pytest.raises(ValueError, match=test_case.expected_error_fragment): parse_sources_yml(contents=test_case.contents, file_path=Path("sources/raw.yml")) + + +@pytest.mark.parametrize( + "test_case", + [ + ParseSourcesYamlHelpTestCase( + description="misspelled rest_api resource key", + contents=""" +dlt_sources: + - type: rest_api + config: + client: + base_url: https://api.example.com/ + resources: + - name: raw_orders + write_dispositon: append + endpoint: + path: orders +""", + expected_error_fragment=( + "dlt rest_api resource 'raw_orders' has unknown keys: write_dispositon" + ), + expected_help="did you mean 'write_disposition'?", + ), + ], + ids=lambda case: case.description, +) +def test_given_misspelled_dlt_resource_key_when_parsing_then_help_names_nearest_key( + test_case: ParseSourcesYamlHelpTestCase, +) -> None: + with pytest.raises(SourceParseError) as error_info: + parse_sources_yml(contents=test_case.contents, file_path=Path("sources/raw.yml")) + + assert test_case.expected_error_fragment in str(error_info.value) + assert error_info.value.help == test_case.expected_help diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 099e4f5d7c..67848cdf49 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -89,6 +89,13 @@ class ResolveBackfillTestCase: expected_result: BackfillResult +@dataclass(frozen=True) +class ResolveBackfillErrorTestCase: + description: str + raw_value: str + expected_error_fragment: str + + @dataclass(frozen=True) class DetectRenamedModelTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py index 95f8a09ea5..2b12491a2d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py @@ -3,9 +3,11 @@ import pytest from sqlbuild.compiler.planner._helpers.changes.policy import resolve_replay_on_change +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import BackfillResult from sqlbuild.compiler.planner.types import BackfillAction from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( + ResolveBackfillErrorTestCase, ResolveBackfillTestCase, ) @@ -28,11 +30,6 @@ raw_value=None, expected_result=BackfillResult(action=BackfillAction.FORWARD_ONLY), ), - ResolveBackfillTestCase( - description="returns forward only for unrecognized policy value", - raw_value="unknown", - expected_result=BackfillResult(action=BackfillAction.FORWARD_ONLY), - ), ], ids=lambda case: case.description, ) @@ -44,3 +41,21 @@ def test_given_policy_when_resolving_replay_on_change_then_returns_expected( ) assert result == test_case.expected_result + + +@pytest.mark.parametrize( + "test_case", + [ + ResolveBackfillErrorTestCase( + description="raises for an unvalidated policy value", + raw_value="unknown", + expected_error_fragment="unknown replay_on_change 'unknown'", + ), + ], + ids=lambda case: case.description, +) +def test_given_unvalidated_policy_when_resolving_replay_on_change_then_raises( + test_case: ResolveBackfillErrorTestCase, +) -> None: + with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): + resolve_replay_on_change(replay_on_change=test_case.raw_value) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 78881f8001..bfc3e44d74 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -72,6 +72,7 @@ GraphResourceKind, ) from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( SchemaColumn, SchemaModelEntry, @@ -94,6 +95,8 @@ SourceCursorInputColumnsTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + def microbatch_model_with_cursor_end( test_case: MicrobatchCursorEndPlanTestCase, @@ -586,7 +589,7 @@ def build_scenario_from_test_case( for model_name in test_case.expected_model_names ) assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=test_case.assertion_sql_bodies + assertion_sql=test_case.assertion_sql_bodies, syntax=_GENERIC_SQL_SYNTAX ) return CompiledSqlScenario( key=scenario_key, diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py index a7db4a553d..9303c99533 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py @@ -72,3 +72,23 @@ class DialectNonCodeErrorTestCase: syntax: SqlLexicalSyntax sql: str expected_error: str + + +@dataclass(frozen=True) +class RequiresDialectScanTestCase: + """Detection of SQL a dialect lexes differently from generic SQL.""" + + description: str + syntax: SqlLexicalSyntax + sql: str + expected_reads_differently: bool + + +@dataclass(frozen=True) +class DialectMatchingParenTestCase: + """Dialect-aware parenthesis matching case.""" + + description: str + syntax: SqlLexicalSyntax | None + sql: str + expected_index: int diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py index abe21d87b2..2d45c8472c 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py @@ -6,16 +6,19 @@ from sqlbuild.compiler.sql_analysis._helpers.scanning import ( dialect_non_code_end_impl, dialect_quoted_text_end_impl, + find_matching_paren_impl, iter_code_positions_impl, skip_quoted_text_impl, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers._test_types import ( + DialectMatchingParenTestCase, DialectNonCodeErrorTestCase, DialectNonCodeSuccessTestCase, DialectQuotedTextErrorTestCase, DialectQuotedTextSuccessTestCase, IterCodePositionsTestCase, + RequiresDialectScanTestCase, SkipQuotedTextErrorTestCase, SkipQuotedTextSuccessTestCase, ) @@ -359,3 +362,98 @@ def test_given_sql_when_iterating_code_positions_then_yields_code_offsets_with_d test_case: IterCodePositionsTestCase, ) -> None: assert tuple(iter_code_positions_impl(sql=test_case.sql)) == test_case.expected_positions + + +@pytest.mark.parametrize( + "test_case", + ( + RequiresDialectScanTestCase( + description="backslash under backslash escapes", + syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX, + sql="SELECT 'O\\'Brien'", + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="backslash under escape-string prefixes", + syntax=ESCAPE_PREFIX_SYNTAX, + sql="SELECT E'O\\'Brien'", + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="backslash under generic sql", + syntax=ANSI_SYNTAX, + sql="SELECT 'C:\\'", + expected_reads_differently=False, + ), + RequiresDialectScanTestCase( + description="triple quotes", + syntax=TRIPLE_QUOTE_SYNTAX, + sql='SELECT """note"""', + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="second block comment opener under nested comments", + syntax=NESTED_COMMENT_SYNTAX, + sql="/* a /* b */ */ SELECT 1", + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="single block comment under nested comments", + syntax=NESTED_COMMENT_SYNTAX, + sql="/* a */ SELECT 1", + expected_reads_differently=False, + ), + RequiresDialectScanTestCase( + description="dialect line comment prefix", + syntax=HASH_COMMENT_SYNTAX, + sql="SELECT 1 # note", + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="plain sql under a dialect", + syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX, + sql="SELECT 'customer''s order' -- note", + expected_reads_differently=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_sql_when_checking_dialect_sensitivity_then_reports_differences( + test_case: RequiresDialectScanTestCase, +) -> None: + reads_differently: bool = test_case.syntax.reads_differently_from_generic(test_case.sql) + + assert reads_differently is test_case.expected_reads_differently + + +@pytest.mark.parametrize( + "test_case", + ( + DialectMatchingParenTestCase( + description="backslash-escaped quote hides a parenthesis", + syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX, + sql="('O\\') ' AS name) tail", + expected_index=len("('O\\') ' AS name"), + ), + DialectMatchingParenTestCase( + description="hash comment hides a parenthesis", + syntax=HASH_COMMENT_SYNTAX, + sql="(1 # )\n) tail", + expected_index=len("(1 # )\n"), + ), + DialectMatchingParenTestCase( + description="generic scanning keeps the backslash literal", + syntax=None, + sql="('C:\\') tail", + expected_index=len("('C:\\'"), + ), + ), + ids=lambda case: case.description, +) +def test_given_dialect_sql_when_matching_parenthesis_then_follows_dialect_rules( + test_case: DialectMatchingParenTestCase, +) -> None: + assert ( + find_matching_paren_impl(sql=test_case.sql, open_paren_index=0, syntax=test_case.syntax) + == test_case.expected_index + ) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py index b50c6ad60d..159458e9b0 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py @@ -18,6 +18,14 @@ class SchemaValidationScopeTestCase: expected_diagnostic_count: int +@dataclass(frozen=True) +class ExactColumnCatalogValidationTestCase: + description: str + query_sql: str + schema: dict[str, dict[str, str]] + expected_messages: tuple[str, ...] + + @dataclass(frozen=True) class PolyglotSqlNormalizationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py index d24d314a51..a2b52ef8f6 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py @@ -2,9 +2,11 @@ import pytest +from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( + ExactColumnCatalogValidationTestCase, SchemaValidationScopeTestCase, ) @@ -200,3 +202,70 @@ def test_given_nested_query_scopes_when_validating_complete_schema_then_resolves assert len(results) == len(test_case.dialects) assert sum(len(result.diagnostics) for result in results) == test_case.expected_diagnostic_count + + +@pytest.mark.parametrize( + "test_case", + [ + ExactColumnCatalogValidationTestCase( + description="join using a column that both exact-name relations hold", + query_sql=( + "SELECT orders.order_id FROM orders JOIN raw_orders AS raw_orders USING (order_id)" + ), + schema={ + "orders": {"order_id": "INTEGER"}, + "raw_orders": {'"order_id"': "INTEGER", '"ordered_at"': "TIMESTAMP"}, + }, + expected_messages=(), + ), + ExactColumnCatalogValidationTestCase( + description="unqualified join using across two exact-name relations", + query_sql="SELECT order_id FROM orders JOIN raw_orders USING (order_id)", + schema={ + "orders": {'"order_id"': "INTEGER"}, + "raw_orders": {'"order_id"': "INTEGER"}, + }, + expected_messages=(), + ), + ExactColumnCatalogValidationTestCase( + description="join using a column one exact-name relation lacks", + query_sql="SELECT orders.order_id FROM orders JOIN raw_orders USING (customer_id)", + schema={ + "orders": {"order_id": "INTEGER", "customer_id": "INTEGER"}, + "raw_orders": {'"order_id"': "INTEGER"}, + }, + expected_messages=( + "JOIN USING column 'customer_id' must exist on both sides (context: JOIN USING)", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_exact_name_relations_when_validating_join_using_then_matches_authored_columns( + test_case: ExactColumnCatalogValidationTestCase, +) -> None: + known_types: tuple[str, ...] = ("INTEGER", "TIMESTAMP") + catalog: BindingCatalog = BindingCatalog( + dialect="duckdb", + quoted_ignore_case=False, + known_functions=(), + known_types=known_types, + relations={}, + ) + + results: tuple[SqlBindingResult, ...] = get_schema_validations( + requests=( + SqlSchemaValidationRequest( + sql=test_case.query_sql, + dialect="duckdb", + schema=test_case.schema, + known_types=known_types, + catalog=catalog, + ), + ) + ) + + assert ( + tuple(diagnostic.message for diagnostic in results[0].diagnostics) + == test_case.expected_messages + ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index de504d1df0..efcfe9c8f6 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -29,6 +29,7 @@ PlanAction, PlanReason, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv from sqlbuild.integrations.dbt._helpers.graph.core import ( dbt_model_graph_key, @@ -48,6 +49,8 @@ ) from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + def _build_present_field(name: str, value: object) -> dict[str, object]: return {name: value} @@ -435,13 +438,14 @@ def build_compiled_project_with_model_specs( ) ), query_sql=sql, - references=extract_sql_references(sql), + references=extract_sql_references(sql=sql, syntax=_GENERIC_SQL_SYNTAX), ) ) return assemble_compiled_project( inputs=CompileProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index 478a55d411..ebd928eea2 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -4,6 +4,8 @@ from dataclasses import dataclass, field +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + @dataclass(frozen=True) class LintBehaviorTestCase: @@ -116,3 +118,11 @@ class ReservedKeywordFoldTestCase: dialect: str keyword: str expected_same_fingerprint: bool + + +@dataclass(frozen=True) +class PassedAdapterSyntaxTestCase: + """A caller-held lexical syntax that SQL scanning must use without rediscovery.""" + + description: str + expected_syntax: SqlLexicalSyntax diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py new file mode 100644 index 0000000000..6c42dc005f --- /dev/null +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -0,0 +1,78 @@ +"""Caller-held adapter lexical syntax reaches lint and format SQL scanning.""" + +from __future__ import annotations + +from pathlib import Path +from typing import ClassVar +from unittest.mock import Mock + +import pytest + +import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module +import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from tests.unit.src.sqlbuild.lint._test_types import PassedAdapterSyntaxTestCase +from tests.unit.src.sqlbuild.lint.helpers import write_fixture_format_project + +_CUSTOM_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax( + triple_quoted_strings=True, line_comment_prefixes=frozenset({"--", "#"}) +) + + +class _CustomSyntaxDuckDbAdapter(DuckDbAdapter): + sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = _CUSTOM_SYNTAX + + +@pytest.mark.parametrize( + "test_case", + [PassedAdapterSyntaxTestCase("fixture formatting uses the passed syntax", _CUSTOM_SYNTAX)], + ids=lambda case: case.description, +) +def test_given_passed_syntax_when_formatting_fixture_nulls_then_scanner_uses_it( + test_case: PassedAdapterSyntaxTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + test_file: Path = write_fixture_format_project( + tmp_path=tmp_path, fixture_sql="SELECT 1 AS id, CAST(NULL AS BOOLEAN) AS flag" + ) + inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + extract: Mock = Mock(wraps=fixture_formatting_module.SqlTestCteExtractor.extract) + monkeypatch.setattr(fixture_formatting_module.SqlTestCteExtractor, "extract", extract) + + _ = fixture_formatting_module.format_redundant_fixture_nulls( + files={test_file.resolve(): test_file.read_text(encoding="utf-8")}, + project_dir=tmp_path, + discovered_inputs=inputs, + sql_lexical_syntax=test_case.expected_syntax, + ) + + assert extract.call_args.kwargs["syntax"] is test_case.expected_syntax + + +@pytest.mark.parametrize( + "test_case", + [PassedAdapterSyntaxTestCase("expansion scope uses the renderer syntax", _CUSTOM_SYNTAX)], + ids=lambda case: case.description, +) +def test_given_value_renderer_when_building_expansion_context_then_scope_uses_its_syntax( + test_case: PassedAdapterSyntaxTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _ = write_fixture_format_project(tmp_path=tmp_path, fixture_sql="SELECT 1 AS id, TRUE AS flag") + build_scope: Mock = Mock(wraps=expansion_context_module.build_declaration_scope) + monkeypatch.setattr(expansion_context_module, "build_declaration_scope", build_scope) + + _ = expansion_context_module.build_sql_expansion_context( + project_dir=tmp_path, value_renderer=_CustomSyntaxDuckDbAdapter() + ) + + assert build_scope.call_args.kwargs["sql_lexical_syntax"] is test_case.expected_syntax + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index d5d553bbfc..a7dd474af3 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -140,6 +140,11 @@ Returns a structured report including: - `lineage_mode` - which lineage mode was used - `artifacts` - paths to written files +If compile stops on an error before it can build the report, such as a model header that does not +parse, `--json` still prints one JSON document. It has `command`, `has_errors: true`, +`stopped: true`, and a `diagnostics` list with that error's `code`, `message`, and `help`, the same +text the error shows on stderr. The exit code is still 1, and progress stays on stderr. + ## Column lineage modes The `--lineage-mode` flag controls how column lineage is computed during compile: diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 51b24356ae..010e0a66d8 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -137,7 +137,7 @@ Table promotion mode is a project setting rather than a `MODEL()` field. Staged See [Incremental](/docs/concepts/incremental) for full semantics. -Current compatibility behavior treats an unrecognized `on_schema_change` value as the default `append_new_columns` policy and an unrecognized `replay_on_change` value as `forward`. Use the documented values exactly; future validation may reject unknown values instead of applying these fallbacks. +Unknown `on_schema_change` or `replay_on_change` values fail compilation, in `MODEL()` headers, `[defaults]` and `[path_defaults]`. The error names the value and lists the valid ones. `bounded-` needs a positive duration such as `bounded-14d`. ## Snapshot fields diff --git a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx index 0d180f6d43..dcb0247ba5 100644 --- a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx +++ b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx @@ -25,7 +25,9 @@ normal compiler before treating the complete project as valid. Rules cache under `target/rules-cache`. A cache identity includes the Rule code, implementation and imported helpers, options, invocation subject, accessed compiler facts, dialect, tracked project -observations, and compatibility versions. +observations, and compatibility versions. The SQLBuild version in that identity also carries a hash of +the native engine's source, so a local development build never reuses results from a different build +of the same version. Model-subject Rules support fine-grained reuse. Editing one model invalidates affected model subjects rather than every Rule invocation. Project-subject Rules run once and are appropriate for diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 4c3c64626a..74c2470ec1 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -77,6 +77,13 @@ The trailing `SELECT 1` is required as a ceremonial closing statement. Any CTE without one of these prefixes is treated as a helper CTE, available to mock, model, `__expected__`, and `__assert__` SQL in the test. +SQLBuild splits test files into CTEs and finds `__ref`, `__source`, `__seed`, `__dbt_ref`, +`__udf` and `__table_fn` calls outside comments and quoted text, in tests, scenarios, models, +functions, audits and hooks alike. Quoted text and comments follow the project adapter's rules: +backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, +nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, +and `//` line comments on Snowflake. + An `__expected__` CTE compares only the columns it lists, matched by name, so column order does not matter and unlisted model columns are ignored. Row counts are always compared. Listing a column the model does not output is a test error. When the expected CTE's columns are not explicit diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx index 7341376c11..2f8e03e7fa 100644 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ b/website/src/content/docs/docs/integrations/dlt.mdx @@ -82,6 +82,7 @@ dlt_sources: bucket_url: "s3://my-bucket/events/" resources: - name: raw_events + reader: csv write_disposition: append ``` @@ -94,11 +95,14 @@ dlt_sources: | `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | | `table` | Source table to replicate (`sql_database` only). Required for that type. | | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | +| `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | | `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | | `primary_key` / `merge_key` | dlt keys for `merge`. | | `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | | `schema` | Per-resource schema override (falls back to the group `schema`). | +A `dlt_sources` entry accepts only `type`, `config`, `destination`, `schema`, and `resources`, and each resource accepts only the keys above for its source type. Any other key fails compilation with the nearest supported key as a hint. + Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_insert` is not available declaratively; use a Python loader for that. Config values support SQLBuild's [interpolation](/docs/concepts/interpolation): `${name}` for a project variable, `${ENV:NAME}` for an environment variable, and `${CTX:...}` for context, resolved at load time, so credentials stay out of the YAML. ### Destination