diff --git a/Cargo.lock b/Cargo.lock index 39606b27fd..94c815788e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -710,6 +710,7 @@ dependencies = [ name = "sqlbuild-attachments" version = "0.139.0" dependencies = [ + "sqlbuild-core", "sqlbuild-sqltext", ] diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs index 78c5e19658..031df7a8c9 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/extraction.rs @@ -58,7 +58,7 @@ struct TestRequest { } #[derive(Clone, Debug, PartialEq, Eq, Serialize)] -struct Cte(String, String); +pub(crate) struct Cte(pub(crate) String, pub(crate) String); #[derive(Debug, PartialEq, Eq, Serialize)] #[serde(tag = "kind", rename_all = "snake_case")] @@ -105,6 +105,12 @@ pub(crate) fn extract_batch_json(request_json: &str) -> Result { } fn extract_ctes(sql: &str, file: &str) -> Result, String> { + extract_ctes_with_quoting(sql, file).map(|(ctes, _)| ctes) +} + +/// The top-level CTEs and whether any CTE name was a quoted identifier. +pub(crate) fn extract_ctes_with_quoting(sql: &str, file: &str) -> Result<(Vec, bool), String> { + let mut quoted = false; let mut index = skip_ignorable(sql, 0)?; index = consume_keyword(sql, index, "WITH").ok_or_else(|| { format!("SQL test '{file}' must declare mock CTEs and one __expected__ CTE in a top-level WITH clause") @@ -116,6 +122,7 @@ fn extract_ctes(sql: &str, file: &str) -> Result, String> { let mut ctes: Vec = Vec::new(); let mut seen: HashSet = HashSet::new(); loop { + quoted |= matches!(byte_at(sql, index), Some(b'"' | b'`')); let (name, end) = read_identifier(sql, index) .ok_or_else(|| format!("SQL test '{file}' expected a CTE name"))?; if !seen.insert(name.clone()) { @@ -145,7 +152,7 @@ fn extract_ctes(sql: &str, file: &str) -> Result, String> { "SQL test '{file}' must end after its CTEs; only an optional ceremonial top-level `SELECT 1` may follow them" )); } - Ok(ctes) + Ok((ctes, quoted)) } fn classify(ctes: Vec, file: &str, mode: &str) -> Result { @@ -568,7 +575,7 @@ fn implicit_projection_alias(expression: &str) -> Option { (end == expression.len()).then_some(alias) } -fn validate_independence(ctes: &[Cte], file: &str) -> Result<(), String> { +pub(crate) fn validate_independence(ctes: &[Cte], file: &str) -> Result<(), String> { let names: HashMap = ctes .iter() .map(|cte| (cte.0.to_lowercase(), cte.0.clone())) diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs index b89dd43537..cb1217cfcd 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs @@ -9,3 +9,4 @@ pub mod markers; pub mod planning; pub mod relation_markers; pub mod rendering; +pub mod scenario_extraction; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs new file mode 100644 index 0000000000..fa12bf8f6d --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs @@ -0,0 +1,101 @@ +//! Python's `extract_sql_scenario_ctes` for scenarios its top-level scanner reads exactly. + +use serde::Serialize; + +use crate::compiler::_helpers::sql_tests::extraction::{ + Cte, extract_ctes_with_quoting, validate_independence, +}; + +/// Characters whose Python case mapping reaches ASCII, which Python's keyword match would accept. +const CASE_MAPPED_TO_ASCII: [char; 20] = [ + '\u{df}', '\u{130}', '\u{131}', '\u{149}', '\u{17f}', '\u{1f0}', '\u{1e96}', '\u{1e97}', + '\u{1e98}', '\u{1e99}', '\u{1e9a}', '\u{1e9e}', '\u{212a}', '\u{fb00}', '\u{fb01}', '\u{fb02}', + '\u{fb03}', '\u{fb04}', '\u{fb05}', '\u{fb06}', +]; + +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct ClassifiedScenario { + authored: Vec, + expected: Vec, + assertions: Vec, + source_fixtures: Vec, + ref_fixtures: Vec, + seed_fixtures: Vec, + dbt_ref_fixtures: Vec, + expected_models: Vec, + assertion_names: Vec, +} + +pub(crate) fn extract_scenario_json(sql: &str, file: &str) -> Result, String> { + if sql.chars().any(|character| { + matches!(character, '\u{1c}'..='\u{1f}') || CASE_MAPPED_TO_ASCII.contains(&character) + }) { + return Ok(None); + } + let Ok((ctes, quoted)) = extract_ctes_with_quoting(sql, file) else { + return Ok(None); + }; + if quoted || !ctes.iter().all(|cte| is_plain_identifier(&cte.0)) { + return Ok(None); + } + if validate_independence(&ctes, file).is_err() { + return Ok(None); + } + let Some(scenario) = classify(ctes) else { + return Ok(None); + }; + serde_json::to_string(&scenario) + .map(Some) + .map_err(|error| error.to_string()) +} + +/// Python's `_read_identifier` reads only `[A-Za-z_][A-Za-z0-9_]*` the same way. +fn is_plain_identifier(name: &str) -> bool { + let mut bytes = name.bytes(); + bytes + .next() + .is_some_and(|first| first.is_ascii_alphabetic() || first == b'_') + && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +} + +fn classify(ctes: Vec) -> Option { + let mut scenario: ClassifiedScenario = ClassifiedScenario::default(); + for cte in ctes { + let name: String = cte.0.clone(); + if let Some(value) = name.strip_prefix("__source__") { + scenario.source_fixtures.push(required(value)?); + scenario.authored.push(cte); + } else if let Some(value) = name.strip_prefix("__ref__") { + scenario.ref_fixtures.push(required(value)?); + scenario.authored.push(cte); + } else if let Some(value) = name.strip_prefix("__seed__") { + scenario.seed_fixtures.push(required(value)?); + scenario.authored.push(cte); + } else if let Some(value) = name.strip_prefix("__dbt_ref__") { + scenario.dbt_ref_fixtures.push(required(value)?); + scenario.authored.push(cte); + } else if let Some(value) = name.strip_prefix("__expected__") { + scenario.expected_models.push(required(value)?); + scenario.expected.push(cte); + } else if let Some(value) = name.strip_prefix("__assert__") { + scenario.assertion_names.push(required(value)?); + scenario.assertions.push(cte); + } else if name.starts_with("__macro__") { + return None; + } else { + scenario.authored.push(cte); + } + } + let has_fixture: bool = !(scenario.source_fixtures.is_empty() + && scenario.ref_fixtures.is_empty() + && scenario.seed_fixtures.is_empty() + && scenario.dbt_ref_fixtures.is_empty()); + let has_check: bool = + !(scenario.expected_models.is_empty() && scenario.assertion_names.is_empty()); + (has_fixture && has_check).then_some(scenario) +} + +fn required(value: &str) -> Option { + (!value.is_empty()).then(|| value.to_owned()) +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/mod.rs b/crates/sqlbuild-analysis/src/compiler/main/mod.rs index b45f3bacd3..0267d8f76c 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/mod.rs @@ -1,3 +1,4 @@ +pub mod sql_scenario_extraction; pub mod sql_test_chain_resolution; pub mod sql_test_difference_sampling; pub mod sql_test_extraction; diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs new file mode 100644 index 0000000000..0d19a3146a --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs @@ -0,0 +1,6 @@ +use crate::compiler::_helpers; + +/// Python's extracted scenario CTEs as JSON, or `None` where Python must extract the scenario. +pub fn extract_scenario_json(sql: &str, file_label: &str) -> Result, String> { + _helpers::sql_tests::scenario_extraction::extract_scenario_json(sql, file_label) +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index b3c3a19419..a057d9d00e 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_sql_scenario_extraction.rs"] +mod sql_scenario_extraction; #[path = "tests/test_sql_test_cte_rename.rs"] mod sql_test_cte_rename; #[path = "tests/test_sql_test_cte_slices.rs"] diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs new file mode 100644 index 0000000000..aceb0557fc --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs @@ -0,0 +1,46 @@ +use crate::compiler::main::sql_scenario_extraction::extract_scenario_json; +use crate::compiler::tests::test_types::ScenarioExtractionTestCase; + +#[test] +fn given_scenarios_when_extracting_natively_then_python_payloads_or_deferrals_are_returned() { + let test_cases = [ + ScenarioExtractionTestCase { + description: "fixtures, helpers, expected results and assertions are classified", + sql: "WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT id FROM helper), \ + __expected__orders AS (SELECT 1 AS id), __assert__positive AS (SELECT 1) SELECT 1", + expected_json: Some( + r#"{"authored":[["helper","SELECT 1 AS id"],["__source__raw_orders","SELECT id FROM helper"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[["__assert__positive","SELECT 1"]],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":["positive"]}"#, + ), + }, + ScenarioExtractionTestCase { + description: "a macro mock defers so Python raises", + sql: "WITH __source__raw AS (SELECT 1), __macro__tidy AS (SELECT 'x'), \ + __expected__orders AS (SELECT 1 AS id) SELECT 1", + expected_json: None, + }, + ScenarioExtractionTestCase { + description: "a quoted CTE name defers to Python's scanner fallback", + sql: "WITH \"__source__raw\" AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)", + expected_json: None, + }, + ScenarioExtractionTestCase { + description: "a keyword Python would match by case mapping defers", + sql: "w\u{131}th __source__raw AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)", + expected_json: None, + }, + ScenarioExtractionTestCase { + description: "a scenario without fixtures defers so Python raises", + sql: "WITH __expected__orders AS (SELECT 1 AS id) SELECT 1", + expected_json: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + extract_scenario_json(test_case.sql, "tests/scenarios/orders.sql"), + Ok(test_case.expected_json.map(str::to_owned)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 39a9489144..d29dd82315 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -131,3 +131,9 @@ pub(crate) struct HelperReferenceTestCase { pub(crate) expected_fragments: &'static [&'static str], pub(crate) expected_absent_fragments: &'static [&'static str], } + +pub(crate) struct ScenarioExtractionTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_json: Option<&'static str>, +} diff --git a/crates/sqlbuild-attachments/Cargo.toml b/crates/sqlbuild-attachments/Cargo.toml index 5cd5e398a9..0aa53b0e26 100644 --- a/crates/sqlbuild-attachments/Cargo.toml +++ b/crates/sqlbuild-attachments/Cargo.toml @@ -12,3 +12,4 @@ workspace = true [dependencies] sqlbuild-sqltext.workspace = true +sqlbuild-core.workspace = true diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/boundaries.rs b/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/boundaries.rs new file mode 100644 index 0000000000..2820bd2271 --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/boundaries.rs @@ -0,0 +1,35 @@ +//! Python's code boundary pattern and `WITH` keyword check. + +/// Python's boundary pattern: parentheses, `/*`, quotes, `$` and the line comment prefixes. +pub(crate) fn next_boundary(bytes: &[u8], start: usize, prefixes: &[String]) -> Option { + (start..bytes.len()).find(|index| is_boundary(&bytes[*index..], prefixes)) +} + +fn is_boundary(rest: &[u8], prefixes: &[String]) -> bool { + if matches!(rest[0], b'(' | b')' | b'\'' | b'"' | b'`' | b'$') || rest.starts_with(b"/*") { + return true; + } + for prefix in prefixes { + if rest.starts_with(prefix.as_bytes()) { + return true; + } + } + false +} + +/// Whether `WITH` (any case) starts at `first` as a whole word; None for non-ASCII text. +pub(crate) fn starts_with_keyword(sql: &str, first: usize) -> Option { + let mut characters = sql[first..].chars(); + let keyword: String = characters.by_ref().take(4).collect(); + if !keyword.is_ascii() { + return None; + } + if !keyword.eq_ignore_ascii_case("WITH") { + return Some(false); + } + match characters.next() { + None => Some(true), + Some(next) if !next.is_ascii() => None, + Some(next) => Some(!(next.is_ascii_alphanumeric() || next == '_')), + } +} diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/mod.rs b/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/mod.rs new file mode 100644 index 0000000000..d679238dff --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod boundaries; +pub(crate) mod segments; diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/segments.rs b/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/segments.rs new file mode 100644 index 0000000000..bbe1e58ffb --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/segments.rs @@ -0,0 +1,51 @@ +//! Python's code segment bounds and statement terminator characters. + +/// Python's `" \t\r\n\f\v;"`. +pub(crate) fn is_terminator(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\r' | b'\n' | 0x0c | 0x0b | b';') +} + +/// `sql` without trailing terminator characters. +pub(crate) fn trimmed_end(sql: &[u8], start: usize, end: usize) -> usize { + let mut index: usize = end; + while index > start && is_terminator(sql[index - 1]) { + index -= 1; + } + index +} + +/// The first and last code bytes so far, extended by the code segment `start..end`. +pub(crate) fn segment_bounds( + sql: &[u8], + start: usize, + end: usize, + bounds: Option<(usize, usize)>, +) -> Option<(usize, usize)> { + let stripped_end: usize = trimmed_end(sql, start, end); + let leading: usize = sql[start..stripped_end] + .iter() + .position(|byte| !is_terminator(*byte))?; + let first: usize = bounds.map_or(start + leading, |(first, _)| first); + Some((first, stripped_end - 1)) +} + +/// Python's `_may_end_with_comment` over the body without trailing terminators. +pub(crate) fn may_end_with_comment(tail: &[u8], prefixes: &[String]) -> bool { + let line_start: usize = tail + .iter() + .rposition(|byte| *byte == b'\n') + .map_or(0, |index| index + 1); + let last_line: &[u8] = &tail[line_start..]; + if tail.ends_with(b"*/") { + return true; + } + for prefix in prefixes { + if last_line + .windows(prefix.len()) + .any(|window| window == prefix.as_bytes()) + { + return true; + } + } + false +} diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/main/mod.rs b/crates/sqlbuild-attachments/src/ceremonial_select/main/mod.rs new file mode 100644 index 0000000000..54c8a51257 --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/main/mod.rs @@ -0,0 +1 @@ +pub mod omitted_select_offset; diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/main/omitted_select_offset.rs b/crates/sqlbuild-attachments/src/ceremonial_select/main/omitted_select_offset.rs new file mode 100644 index 0000000000..f4402ea257 --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/main/omitted_select_offset.rs @@ -0,0 +1,63 @@ +//! Python's `omitted_ceremonial_select_offset` under an adapter's lexical syntax. + +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::ceremonial_select::_helpers::boundaries::{next_boundary, starts_with_keyword}; +use crate::ceremonial_select::_helpers::segments::{ + may_end_with_comment, segment_bounds, trimmed_end, +}; +use crate::ceremonial_select::models::OmittedSelect; + +/// The offset after the body's last top-level `)` when only a terminator follows its CTEs. +#[must_use] +pub fn omitted_select_offset(sql: &str, syntax: &LexicalSyntax) -> OmittedSelect { + let bytes: &[u8] = sql.as_bytes(); + let prefixes: &[String] = &syntax.line_comment_prefixes; + if prefixes + .iter() + .any(|prefix| prefix.is_empty() || !prefix.is_ascii()) + { + return OmittedSelect::Deferred; + } + let tail: &[u8] = &bytes[..trimmed_end(bytes, 0, bytes.len())]; + if !tail.ends_with(b")") && !may_end_with_comment(tail, prefixes) { + return OmittedSelect::Absent; + } + let mut bounds: Option<(usize, usize)> = None; + let mut last_close: Option = None; + let mut depth: i64 = 0; + let mut segment_start: usize = 0; + let mut index: usize = 0; + while let Some(boundary) = next_boundary(bytes, index, prefixes) { + match dialect_non_code_end(bytes, boundary, syntax) { + Err(_) => return OmittedSelect::Absent, + Ok(Some(end)) => { + bounds = segment_bounds(bytes, segment_start, boundary, bounds).or(bounds); + segment_start = end; + index = end; + } + Ok(None) => { + if bytes[boundary] == b'(' { + depth += 1; + } else if bytes[boundary] == b')' { + depth -= 1; + last_close = (depth == 0).then_some(boundary); + } + index = boundary + 1; + } + } + } + bounds = segment_bounds(bytes, segment_start, bytes.len(), bounds).or(bounds); + let Some((first, last)) = bounds else { + return OmittedSelect::Absent; + }; + if Some(last) != last_close { + return OmittedSelect::Absent; + } + match starts_with_keyword(sql, first) { + Some(true) => OmittedSelect::At(sql[..=last].chars().count()), + Some(false) => OmittedSelect::Absent, + None => OmittedSelect::Deferred, + } +} diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/mod.rs b/crates/sqlbuild-attachments/src/ceremonial_select/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/models.rs b/crates/sqlbuild-attachments/src/ceremonial_select/models.rs new file mode 100644 index 0000000000..55d9e386e8 --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/models.rs @@ -0,0 +1,11 @@ +//! Where a SQL test or scenario body omits its trailing ceremonial `SELECT 1`. + +/// Python's answer, or a deferral where only Python's Unicode rules can decide. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OmittedSelect { + /// Insert `SELECT 1` at this code-point offset. + At(usize), + /// The body already ends correctly or is not a bare `WITH` statement. + Absent, + Deferred, +} diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/tests.rs b/crates/sqlbuild-attachments/src/ceremonial_select/tests.rs new file mode 100644 index 0000000000..66ae5f99cf --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_omitted_select_offset.rs"] +mod omitted_select_offset; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_omitted_select_offset.rs b/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_omitted_select_offset.rs new file mode 100644 index 0000000000..e46b61ebc2 --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_omitted_select_offset.rs @@ -0,0 +1,54 @@ +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::ceremonial_select::main::omitted_select_offset::omitted_select_offset; +use crate::ceremonial_select::models::OmittedSelect; +use crate::ceremonial_select::tests::test_types::OmittedSelectTestCase; + +#[test] +fn given_test_bodies_when_locating_omitted_select_then_python_offset_is_returned() { + let test_cases = [ + OmittedSelectTestCase { + description: "a body ending after its last CTE gets the offset after `)`", + sql: "with a AS (SELECT 'é)') ;\n", + expected_offset: OmittedSelect::At(23), + }, + OmittedSelectTestCase { + description: "a trailing comment after the last CTE", + sql: "WITH a AS (SELECT 1) -- done", + expected_offset: OmittedSelect::At(20), + }, + OmittedSelectTestCase { + description: "a body with its ceremonial SELECT is unchanged", + sql: "WITH a AS (SELECT 1) SELECT 1", + expected_offset: OmittedSelect::Absent, + }, + OmittedSelectTestCase { + description: "a body that is not a WITH statement", + sql: "WITHIN (1)", + expected_offset: OmittedSelect::Absent, + }, + OmittedSelectTestCase { + description: "an unclosed quote is not completed", + sql: "WITH a AS (SELECT ')') '", + expected_offset: OmittedSelect::Absent, + }, + OmittedSelectTestCase { + description: "non-ASCII keyword text defers to Python's case mapping", + sql: "w\u{131}th a AS (SELECT 1)", + expected_offset: OmittedSelect::Deferred, + }, + ]; + + for test_case in test_cases { + let syntax: LexicalSyntax = LexicalSyntax { + line_comment_prefixes: vec!["--".to_owned()], + ..LexicalSyntax::default() + }; + assert_eq!( + omitted_select_offset(test_case.sql, &syntax), + test_case.expected_offset, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_types.rs b/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_types.rs new file mode 100644 index 0000000000..2e0dfe7a0b --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_types.rs @@ -0,0 +1,7 @@ +use crate::ceremonial_select::models::OmittedSelect; + +pub(super) struct OmittedSelectTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + pub(super) expected_offset: OmittedSelect, +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs index a925840de3..f38db65c8e 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs @@ -1,31 +1,15 @@ //! Python's cursor intrinsic scan over quotes, comments and identifier boundaries. -use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; -use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; -use sqlbuild_sqltext::sql_scan::models::QuotePolicy; +use crate::sql_lexing::main::python_non_code_end::python_non_code_end; +use crate::sql_lexing::models::NonCode; const INTRINSIC_NAMES: [&[u8]; 2] = [b"__cursor_start", b"__cursor_end"]; -/// Only dollar quotes: Python's own quote scan does not double backticks, so quotes run here. -const DOLLAR_POLICY: QuotePolicy = QuotePolicy { - backtick_identifiers: false, - single_quote_backslash_escapes: false, - double_quote_backslash_escapes: false, - dollar_quotes: true, -}; - /// The next scan offset after `index`, or `Err` where Python finds an intrinsic or raises. pub(crate) fn step(bytes: &[u8], index: usize) -> Result { - if let Some(end) = comment_end(bytes, index).map_err(|_| ())? { - return Ok(end); - } - match bytes[index] { - quote @ (b'\'' | b'"' | b'`') => return quoted_end(bytes, index, quote), - b'$' => { - return Ok(non_code_end(bytes, index, DOLLAR_POLICY) - .map_err(|_| ())? - .unwrap_or(index + 1)); - } - _ => {} + match python_non_code_end(bytes, index) { + NonCode::End(end) => return Ok(end), + NonCode::Raises => return Err(()), + NonCode::Code => {} } for name in INTRINSIC_NAMES { if bytes[index..].starts_with(name) { @@ -40,22 +24,6 @@ pub(crate) fn step(bytes: &[u8], index: usize) -> Result { Ok(index + 1) } -/// Python's quoted text end: `'` and `"` double to escape, backticks do not. -fn quoted_end(bytes: &[u8], start: usize, quote: u8) -> Result { - let mut index: usize = start + 1; - loop { - index += bytes[index..] - .iter() - .position(|byte| *byte == quote) - .ok_or(())?; - if quote != b'`' && bytes.get(index + 1) == Some("e) { - index += 2; - continue; - } - return Ok(index + 1); - } -} - /// Python's `isalnum() or _`; non-ASCII neighbours defer to Python's Unicode rules. fn continues_identifier(byte: Option) -> Result { match byte { diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/mod.rs b/crates/sqlbuild-attachments/src/functions/_helpers/mod.rs new file mode 100644 index 0000000000..ca0050755f --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod namespace; +pub(crate) mod values; diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs b/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs new file mode 100644 index 0000000000..8a302d80e9 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs @@ -0,0 +1,30 @@ +//! One of database or schema of a function namespace. + +pub(crate) type Part = ( + Option, + Option, + Option, + Option, +); + +/// `(physical, logical, fingerprint, fingerprint logical)` for one of database or schema. +pub(crate) fn part( + header: Option<&String>, + default: Option<&String>, + target: Option<&String>, + inherit: bool, +) -> Part { + let logical: Option = match header { + Some(value) => Some(value.clone()), + None if inherit => default.cloned(), + None => None, + }; + let physical: Option = match target { + None => logical.clone(), + Some(value) if header.is_some() || inherit => Some(value.clone()), + Some(_) => None, + }; + let fingerprint_logical: Option = header.or(default).cloned(); + let fingerprint: Option = target.cloned().or_else(|| fingerprint_logical.clone()); + (physical, logical, fingerprint, fingerprint_logical) +} diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/values.rs b/crates/sqlbuild-attachments/src/functions/_helpers/values.rs new file mode 100644 index 0000000000..ba5079c1f3 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/_helpers/values.rs @@ -0,0 +1,69 @@ +//! Python's header value checks: non-empty stripped strings, maps and string lists. + +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::functions::models::{HeaderValue, NamedType}; + +/// Python's `TABLE_FUNCTION_RETURN_KEYS`. +pub(crate) const TABLE_RETURN_KEY: &str = "table"; + +/// `value.strip()` when `value` is a string that is not blank. +pub(crate) fn stripped_text(value: Option<&HeaderValue>) -> Option { + match value? { + HeaderValue::Text(text) if !python_strip(text).is_empty() => { + Some(python_strip(text).to_owned()) + } + _ => None, + } +} + +/// Named types of a map of non-blank strings; None where Python raises. +pub(crate) fn named_types(entries: &[(HeaderValue, HeaderValue)]) -> Option> { + let mut named: Vec = Vec::with_capacity(entries.len()); + for (key, value) in entries { + let HeaderValue::Text(raw_name) = key else { + return None; + }; + named.push(NamedType { + raw_name: raw_name.clone(), + name: stripped_text(Some(key))?, + type_text: stripped_text(Some(value))?, + }); + } + Some(named) +} + +/// Stripped entries of an optional list or tuple of non-blank strings. +pub(crate) fn stripped_list(value: Option<&HeaderValue>) -> Option> { + match value { + None => Some(Vec::new()), + Some(HeaderValue::Sequence(items)) => { + let mut entries: Vec = Vec::with_capacity(items.len()); + for item in items { + entries.push(stripped_text(Some(item))?); + } + Some(entries) + } + Some(_) => None, + } +} + +/// An optional string description, stripped. +pub(crate) fn description(value: Option<&HeaderValue>) -> Option> { + match value { + None => Some(None), + Some(HeaderValue::Text(text)) => Some(Some(python_strip(text).to_owned())), + Some(_) => None, + } +} + +/// The header value under `key`. +pub(crate) fn lookup<'header>( + header: &'header [(String, HeaderValue)], + key: &str, +) -> Option<&'header HeaderValue> { + header + .iter() + .find(|(name, _)| name == key) + .map(|(_, value)| value) +} diff --git a/crates/sqlbuild-attachments/src/functions/main/mod.rs b/crates/sqlbuild-attachments/src/functions/main/mod.rs new file mode 100644 index 0000000000..1d35a99aa7 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/main/mod.rs @@ -0,0 +1,2 @@ +pub mod parse_function_header; +pub mod resolve_function_namespace; diff --git a/crates/sqlbuild-attachments/src/functions/main/parse_function_header.rs b/crates/sqlbuild-attachments/src/functions/main/parse_function_header.rs new file mode 100644 index 0000000000..ce366025c0 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/main/parse_function_header.rs @@ -0,0 +1,61 @@ +//! Python's SQL and Python function header parsing, before template expansion. + +use crate::functions::_helpers::values::{ + TABLE_RETURN_KEY, description, lookup, named_types, stripped_list, stripped_text, +}; +use crate::functions::models::{ + FunctionHeader, FunctionLanguage, FunctionReturns, HeaderValue, NamedType, +}; + +/// The parsed header, or None wherever Python raises for its shape. +#[must_use] +pub fn parse_function_header( + header: &[(String, HeaderValue)], + language: FunctionLanguage, +) -> Option { + let returns: FunctionReturns = match (language, lookup(header, "returns")?) { + (_, HeaderValue::Text(_)) => { + FunctionReturns::Type(stripped_text(lookup(header, "returns"))?) + } + (FunctionLanguage::Sql, HeaderValue::Map(entries)) => table_returns(entries)?, + _ => return None, + }; + let arguments: Vec = match lookup(header, "arguments") { + None => Vec::new(), + Some(HeaderValue::Map(entries)) => named_types(entries)?, + Some(_) => return None, + }; + let python: bool = language == FunctionLanguage::Python; + Some(FunctionHeader { + arguments, + returns, + runtime_version: if python { + Some(stripped_text(lookup(header, "runtime_version"))?) + } else { + None + }, + entry_point: if python { + Some(stripped_text(lookup(header, "entry_point"))?) + } else { + None + }, + packages: if python { + stripped_list(lookup(header, "packages"))? + } else { + Vec::new() + }, + tags: stripped_list(lookup(header, "tags"))?, + description: description(lookup(header, "description"))?, + }) +} + +/// Python's `returns (table (...))`: exactly the `table` key with a non-empty column map. +fn table_returns(entries: &[(HeaderValue, HeaderValue)]) -> Option { + let [(HeaderValue::Text(key), HeaderValue::Map(columns))] = entries else { + return None; + }; + if key != TABLE_RETURN_KEY || columns.is_empty() { + return None; + } + Some(FunctionReturns::Table(named_types(columns)?)) +} diff --git a/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs b/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs new file mode 100644 index 0000000000..ae40e9d40b --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs @@ -0,0 +1,33 @@ +//! Python's SQL and Python function namespace and fingerprint namespace resolution. + +use crate::functions::_helpers::namespace::part; +use crate::functions::models::{FunctionLanguage, FunctionNamespace, NamespaceInputs}; + +/// The physical, logical and fingerprint namespace Python's attachment assigns. +#[must_use] +pub fn resolve_function_namespace(inputs: &NamespaceInputs) -> FunctionNamespace { + let inherit: bool = + inputs.language == FunctionLanguage::Sql || inputs.inherit_default_namespace; + let (database, logical_database, fingerprint_database, fingerprint_logical_database) = part( + inputs.header_database.as_ref(), + inputs.default_database.as_ref(), + inputs.target_database.as_ref(), + inherit, + ); + let (schema, logical_schema, fingerprint_schema, fingerprint_logical_schema) = part( + inputs.header_schema.as_ref(), + inputs.default_schema.as_ref(), + inputs.target_schema.as_ref(), + inherit, + ); + FunctionNamespace { + database, + schema, + logical_database, + logical_schema, + fingerprint_database, + fingerprint_schema, + fingerprint_logical_database, + fingerprint_logical_schema, + } +} diff --git a/crates/sqlbuild-attachments/src/functions/mod.rs b/crates/sqlbuild-attachments/src/functions/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-attachments/src/functions/models.rs b/crates/sqlbuild-attachments/src/functions/models.rs new file mode 100644 index 0000000000..4c9b4a7328 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/models.rs @@ -0,0 +1,74 @@ +//! Function header values as Python holds them and what Python's attachment parses from them. + +/// One authored header value: text, a mapping, a list or tuple, or anything else. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum HeaderValue { + Text(String), + Map(Vec<(HeaderValue, HeaderValue)>), + Sequence(Vec), + Other, +} + +/// Which function file kind's header rules apply. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FunctionLanguage { + Sql, + Python, +} + +/// One argument or return column: the authored name, the stripped name and the stripped type. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NamedType { + /// Python's error messages quote the name as authored. + pub raw_name: String, + pub name: String, + pub type_text: String, +} + +/// A scalar return type or table function columns, with types before template expansion. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FunctionReturns { + Type(String), + Table(Vec), +} + +/// Stripped header values; argument, return and column types are still unexpanded templates. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FunctionHeader { + pub arguments: Vec, + pub returns: FunctionReturns, + pub tags: Vec, + pub description: Option, + /// Python functions only: `runtime_version`, `entry_point` and `packages`. + pub runtime_version: Option, + pub entry_point: Option, + pub packages: Vec, +} + +/// The project and target namespace a function resolves against, already expanded. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NamespaceInputs { + /// The function header's own `database` and `schema`, when they are strings. + pub header_database: Option, + pub header_schema: Option, + pub default_database: Option, + pub default_schema: Option, + pub target_database: Option, + pub target_schema: Option, + /// Python functions without their own namespace inherit the project defaults. + pub language: FunctionLanguage, + pub inherit_default_namespace: bool, +} + +/// Python's resolved physical, logical and fingerprint namespace of one function. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FunctionNamespace { + pub database: Option, + pub schema: Option, + pub logical_database: Option, + pub logical_schema: Option, + pub fingerprint_database: Option, + pub fingerprint_schema: Option, + pub fingerprint_logical_database: Option, + pub fingerprint_logical_schema: Option, +} diff --git a/crates/sqlbuild-attachments/src/functions/tests.rs b/crates/sqlbuild-attachments/src/functions/tests.rs new file mode 100644 index 0000000000..0d422fdc37 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_parse_function_header.rs"] +mod parse_function_header; +#[path = "tests/test_resolve_function_namespace.rs"] +mod resolve_function_namespace; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-attachments/src/functions/tests/helpers.rs b/crates/sqlbuild-attachments/src/functions/tests/helpers.rs new file mode 100644 index 0000000000..fcae3f8eac --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests/helpers.rs @@ -0,0 +1,47 @@ +use crate::functions::models::{FunctionLanguage, HeaderValue, NamedType, NamespaceInputs}; + +pub(super) fn text(value: &str) -> HeaderValue { + HeaderValue::Text(value.to_owned()) +} + +pub(super) fn map(entries: &[(&str, HeaderValue)]) -> HeaderValue { + let mut pairs: Vec<(HeaderValue, HeaderValue)> = Vec::new(); + for (key, value) in entries { + pairs.push((text(key), value.clone())); + } + HeaderValue::Map(pairs) +} + +pub(super) fn header(entries: &[(&str, HeaderValue)]) -> Vec<(String, HeaderValue)> { + let mut pairs: Vec<(String, HeaderValue)> = Vec::new(); + for (key, value) in entries { + pairs.push(((*key).to_owned(), value.clone())); + } + pairs +} + +pub(super) fn owned(value: Option<&str>) -> Option { + value.map(str::to_owned) +} + +/// Header `udfs`, defaults `analytics.shared`, target `prod_db` with no target schema. +pub(super) fn namespace_inputs(language: FunctionLanguage, inherit: bool) -> NamespaceInputs { + NamespaceInputs { + header_database: None, + header_schema: owned(Some("udfs")), + default_database: owned(Some("analytics")), + default_schema: owned(Some("shared")), + target_database: owned(Some("prod_db")), + target_schema: None, + language, + inherit_default_namespace: inherit, + } +} + +pub(super) fn named(raw_name: &str, name: &str, type_text: &str) -> NamedType { + NamedType { + raw_name: raw_name.to_owned(), + name: name.to_owned(), + type_text: type_text.to_owned(), + } +} diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_parse_function_header.rs b/crates/sqlbuild-attachments/src/functions/tests/test_parse_function_header.rs new file mode 100644 index 0000000000..87091ffe2a --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests/test_parse_function_header.rs @@ -0,0 +1,88 @@ +use crate::functions::main::parse_function_header::parse_function_header; +use crate::functions::models::{FunctionHeader, FunctionLanguage, FunctionReturns, HeaderValue}; +use crate::functions::tests::helpers::{header, map, named, text}; +use crate::functions::tests::test_types::FunctionHeaderTestCase; + +#[test] +fn given_function_headers_when_parsing_then_python_values_or_deferrals_are_returned() { + let test_cases = [ + FunctionHeaderTestCase { + description: "a SQL table function with stripped arguments, columns and tags", + header: header(&[ + ("arguments", map(&[(" raw_status ", text(" STRING "))])), + ( + "returns", + map(&[("table", map(&[("id", text("INTEGER"))]))]), + ), + ("tags", HeaderValue::Sequence(vec![text(" orders ")])), + ("description", text(" Order rows. ")), + ]), + language: FunctionLanguage::Sql, + expected_header: Some(FunctionHeader { + arguments: vec![named(" raw_status ", "raw_status", "STRING")], + returns: FunctionReturns::Table(vec![named("id", "id", "INTEGER")]), + tags: vec!["orders".to_owned()], + description: Some("Order rows.".to_owned()), + runtime_version: None, + entry_point: None, + packages: Vec::new(), + }), + }, + FunctionHeaderTestCase { + description: "a Python function with its runtime, entry point and packages", + header: header(&[ + ("returns", text("STRING")), + ("runtime_version", text("3.12")), + ("entry_point", text(" label ")), + ("packages", HeaderValue::Sequence(vec![text("numpy")])), + ]), + language: FunctionLanguage::Python, + expected_header: Some(FunctionHeader { + arguments: Vec::new(), + returns: FunctionReturns::Type("STRING".to_owned()), + tags: Vec::new(), + description: None, + runtime_version: Some("3.12".to_owned()), + entry_point: Some("label".to_owned()), + packages: vec!["numpy".to_owned()], + }), + }, + FunctionHeaderTestCase { + description: "a missing return type defers so Python raises", + header: header(&[("arguments", map(&[]))]), + language: FunctionLanguage::Sql, + expected_header: None, + }, + FunctionHeaderTestCase { + description: "a blank argument type defers so Python raises", + header: header(&[ + ("returns", text("STRING")), + ("arguments", map(&[("value", text(" "))])), + ]), + language: FunctionLanguage::Sql, + expected_header: None, + }, + FunctionHeaderTestCase { + description: "a Python function returning a table defers so Python raises", + header: header(&[ + ( + "returns", + map(&[("table", map(&[("id", text("INTEGER"))]))]), + ), + ("runtime_version", text("3.12")), + ("entry_point", text("label")), + ]), + language: FunctionLanguage::Python, + expected_header: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + parse_function_header(&test_case.header, test_case.language), + test_case.expected_header, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_resolve_function_namespace.rs b/crates/sqlbuild-attachments/src/functions/tests/test_resolve_function_namespace.rs new file mode 100644 index 0000000000..a6f59a1039 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests/test_resolve_function_namespace.rs @@ -0,0 +1,47 @@ +use crate::functions::main::resolve_function_namespace::resolve_function_namespace; +use crate::functions::models::{FunctionLanguage, FunctionNamespace}; +use crate::functions::tests::helpers::{namespace_inputs, owned}; +use crate::functions::tests::test_types::FunctionNamespaceTestCase; + +#[test] +fn given_function_namespaces_when_resolving_then_python_namespaces_are_returned() { + let test_cases = [ + FunctionNamespaceTestCase { + description: "SQL functions inherit the project defaults", + inputs: namespace_inputs(FunctionLanguage::Sql, false), + expected_namespace: FunctionNamespace { + database: owned(Some("prod_db")), + schema: owned(Some("udfs")), + logical_database: owned(Some("analytics")), + logical_schema: owned(Some("udfs")), + fingerprint_database: owned(Some("prod_db")), + fingerprint_schema: owned(Some("udfs")), + fingerprint_logical_database: owned(Some("analytics")), + fingerprint_logical_schema: owned(Some("udfs")), + }, + }, + FunctionNamespaceTestCase { + description: "Python functions that do not inherit keep only their own namespace", + inputs: namespace_inputs(FunctionLanguage::Python, false), + expected_namespace: FunctionNamespace { + database: None, + schema: owned(Some("udfs")), + logical_database: None, + logical_schema: owned(Some("udfs")), + fingerprint_database: owned(Some("prod_db")), + fingerprint_schema: owned(Some("udfs")), + fingerprint_logical_database: owned(Some("analytics")), + fingerprint_logical_schema: owned(Some("udfs")), + }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + resolve_function_namespace(&test_case.inputs), + test_case.expected_namespace, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/functions/tests/test_types.rs b/crates/sqlbuild-attachments/src/functions/tests/test_types.rs new file mode 100644 index 0000000000..d66213f534 --- /dev/null +++ b/crates/sqlbuild-attachments/src/functions/tests/test_types.rs @@ -0,0 +1,16 @@ +use crate::functions::models::{ + FunctionHeader, FunctionLanguage, FunctionNamespace, HeaderValue, NamespaceInputs, +}; + +pub(super) struct FunctionHeaderTestCase { + pub(super) description: &'static str, + pub(super) header: Vec<(String, HeaderValue)>, + pub(super) language: FunctionLanguage, + pub(super) expected_header: Option, +} + +pub(super) struct FunctionNamespaceTestCase { + pub(super) description: &'static str, + pub(super) inputs: NamespaceInputs, + pub(super) expected_namespace: FunctionNamespace, +} diff --git a/crates/sqlbuild-attachments/src/lib.rs b/crates/sqlbuild-attachments/src/lib.rs index 61217c24d9..cd2db15b49 100644 --- a/crates/sqlbuild-attachments/src/lib.rs +++ b/crates/sqlbuild-attachments/src/lib.rs @@ -3,5 +3,10 @@ #![forbid(unsafe_code)] pub mod audits; +pub mod ceremonial_select; pub mod cursor_intrinsics; +pub mod functions; pub mod seeds; +pub mod sql_lexing; +pub mod test_parameters; +pub mod test_targets; diff --git a/crates/sqlbuild-attachments/src/sql_lexing/_helpers/mod.rs b/crates/sqlbuild-attachments/src/sql_lexing/_helpers/mod.rs new file mode 100644 index 0000000000..a91bebfbde --- /dev/null +++ b/crates/sqlbuild-attachments/src/sql_lexing/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod quotes; diff --git a/crates/sqlbuild-attachments/src/sql_lexing/_helpers/quotes.rs b/crates/sqlbuild-attachments/src/sql_lexing/_helpers/quotes.rs new file mode 100644 index 0000000000..9bfb23c727 --- /dev/null +++ b/crates/sqlbuild-attachments/src/sql_lexing/_helpers/quotes.rs @@ -0,0 +1,24 @@ +//! Python's quoted text ends: `'` and `"` double to escape, backticks do not. + +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; + +/// Only dollar quotes: Python's own quote scan does not double backticks, so quotes run here. +pub(crate) const DOLLAR_POLICY: QuotePolicy = QuotePolicy { + backtick_identifiers: false, + single_quote_backslash_escapes: false, + double_quote_backslash_escapes: false, + dollar_quotes: true, +}; + +/// The end of the text quoted at `start`, or None when Python finds it unclosed. +pub(crate) fn quoted_end(bytes: &[u8], start: usize, quote: u8) -> Option { + let mut index: usize = start + 1; + loop { + index += bytes[index..].iter().position(|byte| *byte == quote)?; + if quote != b'`' && bytes.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Some(index + 1); + } +} diff --git a/crates/sqlbuild-attachments/src/sql_lexing/main/mod.rs b/crates/sqlbuild-attachments/src/sql_lexing/main/mod.rs new file mode 100644 index 0000000000..1133013f95 --- /dev/null +++ b/crates/sqlbuild-attachments/src/sql_lexing/main/mod.rs @@ -0,0 +1 @@ +pub mod python_non_code_end; diff --git a/crates/sqlbuild-attachments/src/sql_lexing/main/python_non_code_end.rs b/crates/sqlbuild-attachments/src/sql_lexing/main/python_non_code_end.rs new file mode 100644 index 0000000000..49791fdae4 --- /dev/null +++ b/crates/sqlbuild-attachments/src/sql_lexing/main/python_non_code_end.rs @@ -0,0 +1,28 @@ +//! Python's compiler scan of comments and quoted text outside any adapter dialect. + +use sqlbuild_sqltext::sql_scan::main::comment_end::comment_end; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; + +use crate::sql_lexing::_helpers::quotes::{DOLLAR_POLICY, quoted_end}; +use crate::sql_lexing::models::NonCode; + +/// Python's comment, quote and `$tag$` skipping at `index`; a `$` opening nothing is code. +#[must_use] +pub fn python_non_code_end(bytes: &[u8], index: usize) -> NonCode { + match comment_end(bytes, index) { + Ok(Some(end)) => return NonCode::End(end), + Ok(None) => {} + Err(_) => return NonCode::Raises, + } + match bytes.get(index) { + Some(quote @ (b'\'' | b'"' | b'`')) => { + quoted_end(bytes, index, *quote).map_or(NonCode::Raises, NonCode::End) + } + Some(b'$') => match non_code_end(bytes, index, DOLLAR_POLICY) { + Ok(Some(end)) => NonCode::End(end), + Ok(None) => NonCode::Code, + Err(_) => NonCode::Raises, + }, + _ => NonCode::Code, + } +} diff --git a/crates/sqlbuild-attachments/src/sql_lexing/mod.rs b/crates/sqlbuild-attachments/src/sql_lexing/mod.rs new file mode 100644 index 0000000000..5359000d16 --- /dev/null +++ b/crates/sqlbuild-attachments/src/sql_lexing/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-attachments/src/sql_lexing/models.rs b/crates/sqlbuild-attachments/src/sql_lexing/models.rs new file mode 100644 index 0000000000..0bcf381338 --- /dev/null +++ b/crates/sqlbuild-attachments/src/sql_lexing/models.rs @@ -0,0 +1,9 @@ +//! What Python's compiler scan finds at one offset. + +/// Code, the end of a comment or quoted text, or input that makes Python raise. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NonCode { + Code, + End(usize), + Raises, +} diff --git a/crates/sqlbuild-attachments/src/test_parameters/_helpers/mod.rs b/crates/sqlbuild-attachments/src/test_parameters/_helpers/mod.rs new file mode 100644 index 0000000000..dfe2c26585 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod reference; diff --git a/crates/sqlbuild-attachments/src/test_parameters/_helpers/reference.rs b/crates/sqlbuild-attachments/src/test_parameters/_helpers/reference.rs new file mode 100644 index 0000000000..96348ff2a7 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/_helpers/reference.rs @@ -0,0 +1,54 @@ +//! Python's `@param(?![A-Za-z0-9_])\s*\(\s*"name"\s*\)` reference pattern. + +use sqlbuild_core::text::main::is_python_space::is_python_space; + +const TOKEN: &[u8] = b"@param"; + +/// Python's `_PARAMETER_TOKEN.match` at `index`. +pub(crate) fn parameter_token_at(bytes: &[u8], index: usize) -> bool { + bytes[index..].starts_with(TOKEN) + && !bytes + .get(index + TOKEN.len()) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') +} + +/// The referenced name and the end of the whole reference, or None for a malformed one. +pub(crate) fn reference_at(sql: &str, index: usize) -> Option<(&str, usize)> { + let mut cursor: usize = skip_spaces(sql, index + TOKEN.len()); + cursor = consume(sql, cursor, b'(')?; + cursor = skip_spaces(sql, cursor); + cursor = consume(sql, cursor, b'"')?; + let bytes: &[u8] = sql.as_bytes(); + let name_start: usize = cursor; + if !bytes + .get(cursor) + .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') + { + return None; + } + while bytes + .get(cursor) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + { + cursor += 1; + } + let name_end: usize = cursor; + cursor = consume(sql, cursor, b'"')?; + cursor = skip_spaces(sql, cursor); + cursor = consume(sql, cursor, b')')?; + Some((&sql[name_start..name_end], cursor)) +} + +fn skip_spaces(sql: &str, mut index: usize) -> usize { + while let Some(character) = sql[index..].chars().next() { + if !is_python_space(character) { + break; + } + index += character.len_utf8(); + } + index +} + +fn consume(sql: &str, index: usize, byte: u8) -> Option { + (sql.as_bytes().get(index) == Some(&byte)).then_some(index + 1) +} diff --git a/crates/sqlbuild-attachments/src/test_parameters/main/mod.rs b/crates/sqlbuild-attachments/src/test_parameters/main/mod.rs new file mode 100644 index 0000000000..68de408f64 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/main/mod.rs @@ -0,0 +1 @@ +pub mod parameter_references; diff --git a/crates/sqlbuild-attachments/src/test_parameters/main/parameter_references.rs b/crates/sqlbuild-attachments/src/test_parameters/main/parameter_references.rs new file mode 100644 index 0000000000..250a012692 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/main/parameter_references.rs @@ -0,0 +1,45 @@ +//! Python's `expand_test_parameters` scan, leaving value rendering to the caller. + +use crate::sql_lexing::main::python_non_code_end::python_non_code_end; +use crate::sql_lexing::models::NonCode; +use crate::test_parameters::_helpers::reference::{parameter_token_at, reference_at}; +use crate::test_parameters::models::ParameterReference; + +/// Every active reference in order, or None where Python raises (malformed, undeclared, unclosed). +#[must_use] +pub fn parameter_references(sql: &str, declared: &[String]) -> Option> { + let bytes: &[u8] = sql.as_bytes(); + let mut references: Vec = Vec::new(); + let mut index: usize = 0; + let mut characters: usize = 0; + while index < bytes.len() { + match python_non_code_end(bytes, index) { + NonCode::End(end) => { + characters += sql[index..end].chars().count(); + index = end; + continue; + } + NonCode::Raises => return None, + NonCode::Code => {} + } + if parameter_token_at(bytes, index) { + let (name, end) = reference_at(sql, index)?; + if !declared.iter().any(|candidate| candidate == name) { + return None; + } + let length: usize = sql[index..end].chars().count(); + references.push(ParameterReference { + start: characters, + end: characters + length, + name: name.to_owned(), + }); + characters += length; + index = end; + continue; + } + let width: usize = sql[index..].chars().next().map_or(1, char::len_utf8); + characters += 1; + index += width; + } + Some(references) +} diff --git a/crates/sqlbuild-attachments/src/test_parameters/mod.rs b/crates/sqlbuild-attachments/src/test_parameters/mod.rs new file mode 100644 index 0000000000..e6e5a52eb4 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-attachments/src/test_parameters/models.rs b/crates/sqlbuild-attachments/src/test_parameters/models.rs new file mode 100644 index 0000000000..3cb980c641 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/models.rs @@ -0,0 +1,9 @@ +//! Active `@param("name")` references in one SQL test case. + +/// One reference by Python code-point offsets: `sql[start:end]` is replaced by `name`'s value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParameterReference { + pub start: usize, + pub end: usize, + pub name: String, +} diff --git a/crates/sqlbuild-attachments/src/test_parameters/tests.rs b/crates/sqlbuild-attachments/src/test_parameters/tests.rs new file mode 100644 index 0000000000..35b14f4f17 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_parameter_references.rs"] +mod parameter_references; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-attachments/src/test_parameters/tests/helpers.rs b/crates/sqlbuild-attachments/src/test_parameters/tests/helpers.rs new file mode 100644 index 0000000000..9d72f6fdf3 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/tests/helpers.rs @@ -0,0 +1,12 @@ +use crate::test_parameters::models::ParameterReference; + +/// References as `(start, end, name)` tuples. +pub(super) fn spans( + references: Option>, +) -> Option> { + let mut spans: Vec<(usize, usize, String)> = Vec::new(); + for reference in references? { + spans.push((reference.start, reference.end, reference.name)); + } + Some(spans) +} diff --git a/crates/sqlbuild-attachments/src/test_parameters/tests/test_parameter_references.rs b/crates/sqlbuild-attachments/src/test_parameters/tests/test_parameter_references.rs new file mode 100644 index 0000000000..a084c7d79e --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/tests/test_parameter_references.rs @@ -0,0 +1,49 @@ +use crate::test_parameters::main::parameter_references::parameter_references; +use crate::test_parameters::tests::helpers::spans; +use crate::test_parameters::tests::test_types::ParameterReferenceTestCase; + +#[test] +fn given_test_sql_when_scanning_parameters_then_python_references_are_returned() { + let test_cases = [ + ParameterReferenceTestCase { + description: "references by code point, quotes and comments skipped", + sql: "SELECT 'é@param(\"x\")', @param ( \"region\" ) -- @param(\"x\")\n, @param(\"limit\")", + expected_references: Some(vec![(23, 42, "region"), (60, 75, "limit")]), + }, + ParameterReferenceTestCase { + description: "a longer identifier is not a reference", + sql: "SELECT @params, @param_x", + expected_references: Some(vec![]), + }, + ParameterReferenceTestCase { + description: "a malformed reference defers so Python raises", + sql: "SELECT @param(region)", + expected_references: None, + }, + ParameterReferenceTestCase { + description: "an undeclared parameter defers so Python raises", + sql: "SELECT @param(\"missing\")", + expected_references: None, + }, + ParameterReferenceTestCase { + description: "an unclosed quote defers so Python raises", + sql: "SELECT 'open", + expected_references: None, + }, + ]; + + for test_case in test_cases { + let declared: Vec = vec!["region".to_owned(), "limit".to_owned()]; + assert_eq!( + spans(parameter_references(test_case.sql, &declared)), + test_case.expected_references.map(|references| { + references + .into_iter() + .map(|(start, end, name)| (start, end, name.to_owned())) + .collect() + }), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/test_parameters/tests/test_types.rs b/crates/sqlbuild-attachments/src/test_parameters/tests/test_types.rs new file mode 100644 index 0000000000..c75aa925c5 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_parameters/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(super) struct ParameterReferenceTestCase { + pub(super) description: &'static str, + pub(super) sql: &'static str, + /// `(start, end, name)` by code points, or None where Python raises. + pub(super) expected_references: Option>, +} diff --git a/crates/sqlbuild-attachments/src/test_targets/main/mod.rs b/crates/sqlbuild-attachments/src/test_targets/main/mod.rs new file mode 100644 index 0000000000..298dcb8627 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/main/mod.rs @@ -0,0 +1,2 @@ +pub mod scenario_source_violation; +pub mod unknown_test_target; diff --git a/crates/sqlbuild-attachments/src/test_targets/main/scenario_source_violation.rs b/crates/sqlbuild-attachments/src/test_targets/main/scenario_source_violation.rs new file mode 100644 index 0000000000..ff55774132 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/main/scenario_source_violation.rs @@ -0,0 +1,36 @@ +//! Python's `_validate_scenario_source_references`. + +use std::collections::HashSet; + +use crate::test_targets::models::ScenarioCteSources; + +/// Python's error for the first source a scenario may not read; check CTEs are read first. +#[must_use] +pub fn scenario_source_violation( + file_label: &str, + ctes: &[ScenarioCteSources], + known_sources: &HashSet, +) -> Option { + for cte in ctes.iter().filter(|cte| cte.check) { + if let Some(source) = cte.sources.first() { + return Some(format!( + "SQL scenario file {file_label} CTE '{}' must not reference project source \ + '{source}' with __source(); source-backed scenario data is only allowed \ + in helper and fixture CTEs", + cte.name + )); + } + } + for cte in ctes.iter().filter(|cte| !cte.check) { + if let Some(source) = cte + .sources + .iter() + .find(|source| !known_sources.contains(source.as_str())) + { + return Some(format!( + "SQL scenario file {file_label} references unknown source '{source}'" + )); + } + } + None +} diff --git a/crates/sqlbuild-attachments/src/test_targets/main/unknown_test_target.rs b/crates/sqlbuild-attachments/src/test_targets/main/unknown_test_target.rs new file mode 100644 index 0000000000..fa932b8d9a --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/main/unknown_test_target.rs @@ -0,0 +1,45 @@ +//! Python's `validate_test_ctes` for model SQL tests. + +use std::collections::HashSet; + +use crate::test_targets::models::{TargetCatalog, TestTargets}; + +/// Python's error for the first unknown target in its validation order, or None. +#[must_use] +pub fn unknown_test_target( + file_label: &str, + catalog: &TargetCatalog, + targets: &TestTargets, +) -> Option { + let groups: [(&str, &[String], &HashSet); 7] = [ + ("mocks unknown model", &targets.mock_models, &catalog.models), + ( + "mocks unknown source", + &targets.mock_sources, + &catalog.sources, + ), + ("mocks unknown seed", &targets.mock_seeds, &catalog.seeds), + ( + "mocks unknown table function", + &targets.mock_table_functions, + &catalog.table_functions, + ), + ("mocks unknown macro", &targets.macro_mocks, &catalog.macros), + ( + "expects unknown model", + &targets.expected_models, + &catalog.models, + ), + ( + "assertion references unknown model", + &targets.assertion_targets, + &catalog.models, + ), + ]; + for (phrase, names, known) in groups { + if let Some(name) = names.iter().find(|name| !known.contains(name.as_str())) { + return Some(format!("SQL test file {file_label} {phrase} '{name}'")); + } + } + None +} diff --git a/crates/sqlbuild-attachments/src/test_targets/mod.rs b/crates/sqlbuild-attachments/src/test_targets/mod.rs new file mode 100644 index 0000000000..1a70a48981 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/mod.rs @@ -0,0 +1,5 @@ +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-attachments/src/test_targets/models.rs b/crates/sqlbuild-attachments/src/test_targets/models.rs new file mode 100644 index 0000000000..0e6efa3412 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/models.rs @@ -0,0 +1,34 @@ +//! Targets a SQL test or scenario names, checked against the project's resources. + +use std::collections::HashSet; + +/// The project's model, source, seed, table function and macro names, built once per compile. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TargetCatalog { + pub models: HashSet, + pub sources: HashSet, + pub seeds: HashSet, + pub table_functions: HashSet, + pub macros: HashSet, +} + +/// The names one model SQL test targets, in Python's validation order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TestTargets { + pub mock_models: Vec, + pub mock_sources: Vec, + pub mock_seeds: Vec, + pub mock_table_functions: Vec, + pub macro_mocks: Vec, + pub expected_models: Vec, + pub assertion_targets: Vec, +} + +/// One scenario CTE and the `__source("name")` references in its body. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScenarioCteSources { + pub name: String, + /// Expected-result and assertion CTEs must not read project sources at all. + pub check: bool, + pub sources: Vec, +} diff --git a/crates/sqlbuild-attachments/src/test_targets/tests.rs b/crates/sqlbuild-attachments/src/test_targets/tests.rs new file mode 100644 index 0000000000..efc6bce35d --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_scenario_source_violation.rs"] +mod scenario_source_violation; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_unknown_test_target.rs"] +mod unknown_test_target; diff --git a/crates/sqlbuild-attachments/src/test_targets/tests/helpers.rs b/crates/sqlbuild-attachments/src/test_targets/tests/helpers.rs new file mode 100644 index 0000000000..7e099a9792 --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/tests/helpers.rs @@ -0,0 +1,25 @@ +use std::collections::HashSet; + +use crate::test_targets::models::{ScenarioCteSources, TargetCatalog}; + +pub(super) fn cte(name: &str, check: bool, sources: &[&str]) -> ScenarioCteSources { + ScenarioCteSources { + name: name.to_owned(), + check, + sources: sources.iter().map(|source| (*source).to_owned()).collect(), + } +} + +pub(super) fn names(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// Models `orders` and `customers`, seed `regions` and source `raw`. +pub(super) fn catalog() -> TargetCatalog { + TargetCatalog { + models: names(&["orders", "customers"]).into_iter().collect(), + sources: HashSet::from(["raw".to_owned()]), + seeds: HashSet::from(["regions".to_owned()]), + ..TargetCatalog::default() + } +} diff --git a/crates/sqlbuild-attachments/src/test_targets/tests/test_scenario_source_violation.rs b/crates/sqlbuild-attachments/src/test_targets/tests/test_scenario_source_violation.rs new file mode 100644 index 0000000000..a20986c26c --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/tests/test_scenario_source_violation.rs @@ -0,0 +1,48 @@ +use crate::test_targets::main::scenario_source_violation::scenario_source_violation; +use crate::test_targets::tests::helpers::{catalog, cte}; +use crate::test_targets::tests::test_types::ScenarioSourceTestCase; + +#[test] +fn given_scenario_sources_when_validating_then_python_first_error_is_returned() { + let test_cases = [ + ScenarioSourceTestCase { + description: "known sources in fixtures pass", + ctes: vec![ + cte("__source__raw", false, &["raw"]), + cte("__expected__orders", true, &[]), + ], + expected_error: None, + }, + ScenarioSourceTestCase { + description: "a check CTE reading a source is reported before unknown fixture sources", + ctes: vec![ + cte("helper", false, &["missing"]), + cte("__expected__orders", true, &["raw"]), + ], + expected_error: Some( + "SQL scenario file tests/scenarios/orders.sql CTE '__expected__orders' must not reference project source 'raw' with __source(); source-backed scenario data is only allowed in helper and fixture CTEs", + ), + }, + ScenarioSourceTestCase { + description: "an unknown source in a helper CTE", + ctes: vec![cte("helper", false, &["raw", "missing"])], + expected_error: Some( + "SQL scenario file tests/scenarios/orders.sql references unknown source 'missing'", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + scenario_source_violation( + "tests/scenarios/orders.sql", + &test_case.ctes, + &catalog().sources + ) + .as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-attachments/src/test_targets/tests/test_types.rs b/crates/sqlbuild-attachments/src/test_targets/tests/test_types.rs new file mode 100644 index 0000000000..38a3d15a7e --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/tests/test_types.rs @@ -0,0 +1,13 @@ +use crate::test_targets::models::{ScenarioCteSources, TestTargets}; + +pub(super) struct UnknownTargetTestCase { + pub(super) description: &'static str, + pub(super) targets: TestTargets, + pub(super) expected_error: Option<&'static str>, +} + +pub(super) struct ScenarioSourceTestCase { + pub(super) description: &'static str, + pub(super) ctes: Vec, + pub(super) expected_error: Option<&'static str>, +} diff --git a/crates/sqlbuild-attachments/src/test_targets/tests/test_unknown_test_target.rs b/crates/sqlbuild-attachments/src/test_targets/tests/test_unknown_test_target.rs new file mode 100644 index 0000000000..419719b0ac --- /dev/null +++ b/crates/sqlbuild-attachments/src/test_targets/tests/test_unknown_test_target.rs @@ -0,0 +1,41 @@ +use crate::test_targets::main::unknown_test_target::unknown_test_target; +use crate::test_targets::models::TestTargets; +use crate::test_targets::tests::helpers::{catalog, names}; +use crate::test_targets::tests::test_types::UnknownTargetTestCase; + +#[test] +fn given_test_targets_when_validating_then_python_first_error_is_returned() { + let test_cases = [ + UnknownTargetTestCase { + description: "known targets pass", + targets: TestTargets { + mock_models: names(&["orders"]), + expected_models: names(&["customers"]), + ..TestTargets::default() + }, + expected_error: None, + }, + UnknownTargetTestCase { + description: "the first unknown target of the first failing group is reported", + targets: TestTargets { + mock_models: names(&["orders"]), + mock_seeds: names(&["codes", "regions"]), + expected_models: names(&["missing"]), + ..TestTargets::default() + }, + expected_error: Some( + "SQL test file tests/unit/test_orders.sql mocks unknown seed 'codes'", + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + unknown_test_target("tests/unit/test_orders.sql", &catalog(), &test_case.targets) + .as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs index 5d39624fe3..63fee6eef9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/sql_tests.rs @@ -50,7 +50,23 @@ fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult, + sql: &str, + file_label: &str, +) -> PyResult> { + py.compiler_detach(|| { + sqlbuild_analysis::compiler::main::sql_scenario_extraction::extract_scenario_json( + sql, file_label, + ) + }) + .map_err(value_error) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(extract_sql_scenario_json, module)?)?; module.add_function(wrap_pyfunction!(render_sql_test_comparisons_json, module)?)?; module.add_function(wrap_pyfunction!(plan_and_render_sql_tests_json, module)?)?; module.add_function(wrap_pyfunction!(resolve_sql_test_chains_json, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/ceremonial_select.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/ceremonial_select.rs new file mode 100644 index 0000000000..226c62c4cf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/ceremonial_select.rs @@ -0,0 +1,31 @@ +//! Where a SQL test or scenario body omits its trailing ceremonial `SELECT 1`. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; +use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; + +/// `(True, offset or None)` with Python's answer, or `(False, None)` where Python must decide. +#[pyfunction] +fn omitted_ceremonial_select( + sql: &str, + syntax: LexicalSyntaxInput, +) -> PyResult<(bool, Option)> { + let syntax: LexicalSyntax = syntax.into(); + compiler_guard(|| { + Ok(match omitted_select_offset(sql, &syntax) { + OmittedSelect::At(offset) => (true, Some(offset)), + OmittedSelect::Absent => (true, None), + OmittedSelect::Deferred => (false, None), + }) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(omitted_ceremonial_select, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs new file mode 100644 index 0000000000..faa17d6faf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/function_headers.rs @@ -0,0 +1,171 @@ +//! SQL and Python function header parsing and namespaces for the preview compile attachments. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PyListMethods, PyString, PyTuple, PyTupleMethods, +}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_attachments::functions::main::parse_function_header::parse_function_header; +use sqlbuild_attachments::functions::main::resolve_function_namespace::resolve_function_namespace; +use sqlbuild_attachments::functions::models::{ + FunctionHeader, FunctionLanguage, FunctionNamespace, FunctionReturns, HeaderValue, NamedType, + NamespaceInputs, +}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// `(authored name, name, type)` rows. +type Pairs = Vec<(String, String, String)>; +/// Arguments, scalar type, table columns, tags, description, runtime, entry point, packages. +type HeaderRow = ( + Pairs, + Option, + Option, + Vec, + Option, + Option, + Option, + Vec, +); +type NamespaceRow = [Option; 8]; + +/// The expanded namespace inputs of one function. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct NamespaceInput { + header_database: Option, + header_schema: Option, + default_database: Option, + default_schema: Option, + target_database: Option, + target_schema: Option, + python: bool, + inherit_default_namespace: bool, +} + +/// Parse a function header, or return `None` where Python raises for its shape. +#[pyfunction] +fn parse_function_header_values( + header_values: Bound<'_, PyDict>, + python: bool, +) -> PyResult> { + compiler_guard(|| { + let mut header: Vec<(String, HeaderValue)> = Vec::with_capacity(header_values.len()); + for (key, value) in header_values.iter() { + if value.is_none() { + continue; + } + let Some(name) = text_value(&key) else { + return Ok(None); + }; + header.push((name, header_value(&value)?)); + } + let language: FunctionLanguage = if python { + FunctionLanguage::Python + } else { + FunctionLanguage::Sql + }; + Ok(parse_function_header(&header, language).map(header_row)) + }) +} + +/// Python's physical, logical and fingerprint database and schema of one function. +#[pyfunction] +fn resolve_function_namespace_values(inputs: NamespaceInput) -> PyResult { + compiler_guard(|| { + let namespace: FunctionNamespace = resolve_function_namespace(&NamespaceInputs { + header_database: inputs.header_database, + header_schema: inputs.header_schema, + default_database: inputs.default_database, + default_schema: inputs.default_schema, + target_database: inputs.target_database, + target_schema: inputs.target_schema, + language: if inputs.python { + FunctionLanguage::Python + } else { + FunctionLanguage::Sql + }, + inherit_default_namespace: inputs.inherit_default_namespace, + }); + Ok([ + namespace.database, + namespace.schema, + namespace.logical_database, + namespace.logical_schema, + namespace.fingerprint_database, + namespace.fingerprint_schema, + namespace.fingerprint_logical_database, + namespace.fingerprint_logical_schema, + ]) + }) +} + +fn header_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Ok(text_value(value).map_or(HeaderValue::Other, HeaderValue::Text)); + } + if let Ok(mapping) = value.downcast::() { + let mut entries: Vec<(HeaderValue, HeaderValue)> = Vec::with_capacity(mapping.len()); + for (key, item) in mapping.iter() { + entries.push((header_value(&key)?, header_value(&item)?)); + } + return Ok(HeaderValue::Map(entries)); + } + if let Ok(list) = value.downcast::() { + let mut items: Vec = Vec::with_capacity(list.len()); + for item in list.iter() { + items.push(header_value(&item)?); + } + return Ok(HeaderValue::Sequence(items)); + } + if let Ok(tuple) = value.downcast::() { + let mut items: Vec = Vec::with_capacity(tuple.len()); + for item in tuple.iter() { + items.push(header_value(&item)?); + } + return Ok(HeaderValue::Sequence(items)); + } + Ok(HeaderValue::Other) +} + +/// A `str` (or subclass) as text; text Rust cannot hold, such as lone surrogates, is `None`. +fn text_value(value: &Bound<'_, PyAny>) -> Option { + if !value.is_instance_of::() { + return None; + } + if let Ok(text) = value.extract::() { + return Some(text); + } + None +} + +fn header_row(header: FunctionHeader) -> HeaderRow { + let (scalar, table): (Option, Option) = match header.returns { + FunctionReturns::Type(text) => (Some(text), None), + FunctionReturns::Table(columns) => (None, Some(rows(columns))), + }; + ( + rows(header.arguments), + scalar, + table, + header.tags, + header.description, + header.runtime_version, + header.entry_point, + header.packages, + ) +} + +fn rows(named: Vec) -> Pairs { + let mut rows: Pairs = Vec::with_capacity(named.len()); + for item in named { + rows.push((item.raw_name, item.name, item.type_text)); + } + rows +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_function_header_values, module)?)?; + module.add_function(wrap_pyfunction!(resolve_function_namespace_values, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs index 94501a40c5..8b62154217 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/mod.rs @@ -1,5 +1,9 @@ pub(crate) mod argument_values; pub(crate) mod audit_rendering; +pub(crate) mod ceremonial_select; pub(crate) mod cursor_intrinsics; +pub(crate) mod function_headers; pub(crate) mod registration; pub(crate) mod seed_pairing; +pub(crate) mod test_parameters; +pub(crate) mod test_targets; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs index 840225b1d2..78cc73e983 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/registration.rs @@ -2,11 +2,18 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::attachments::{audit_rendering, cursor_intrinsics, seed_pairing}; +use crate::bindings::_helpers::attachments::{ + audit_rendering, ceremonial_select, cursor_intrinsics, function_headers, seed_pairing, + test_parameters, test_targets, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { audit_rendering::register(module)?; seed_pairing::register(module)?; cursor_intrinsics::register(module)?; + test_parameters::register(module)?; + ceremonial_select::register(module)?; + test_targets::register(module)?; + function_headers::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_parameters.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_parameters.rs new file mode 100644 index 0000000000..6ca3028e98 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_parameters.rs @@ -0,0 +1,30 @@ +//! SQL test `@param("name")` reference scanning for the preview compile attachments. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_attachments::test_parameters::main::parameter_references::parameter_references; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// `(start, end, name)` code-point spans of every reference, or `None` where Python must scan. +#[pyfunction] +fn scan_test_parameter_references( + sql: &str, + declared: Vec, +) -> PyResult>> { + compiler_guard(|| { + let Some(references) = parameter_references(sql, &declared) else { + return Ok(None); + }; + let mut spans: Vec<(usize, usize, String)> = Vec::with_capacity(references.len()); + for reference in references { + spans.push((reference.start, reference.end, reference.name)); + } + Ok(Some(spans)) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(scan_test_parameter_references, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_targets.rs b/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_targets.rs new file mode 100644 index 0000000000..822a394702 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/attachments/test_targets.rs @@ -0,0 +1,110 @@ +//! SQL test target and scenario source validation for the preview compile attachments. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_attachments::test_targets::main::scenario_source_violation::scenario_source_violation; +use sqlbuild_attachments::test_targets::main::unknown_test_target::unknown_test_target; +use sqlbuild_attachments::test_targets::models::{ScenarioCteSources, TargetCatalog, TestTargets}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// A model test's target name groups, in `TestTargets` field order. +type TargetRow = ( + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, + Vec, +); + +/// The project's resource names, read once per compile and shared by every test and scenario. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct SqlTestTargetCatalog { + catalog: TargetCatalog, +} + +#[pymethods] +impl SqlTestTargetCatalog { + #[new] + fn new( + models: HashSet, + sources: HashSet, + seeds: HashSet, + resources: (HashSet, HashSet), + ) -> Self { + let (table_functions, macros) = resources; + Self { + catalog: TargetCatalog { + models, + sources, + seeds, + table_functions, + macros, + }, + } + } + + /// Python's error for a model test's first unknown target, or `None`. + fn unknown_test_target( + &self, + file_label: &str, + targets: TargetRow, + ) -> PyResult> { + compiler_guard(|| { + let ( + mock_models, + mock_sources, + mock_seeds, + mock_table_functions, + macro_mocks, + expected_models, + assertion_targets, + ) = targets; + Ok(unknown_test_target( + file_label, + &self.catalog, + &TestTargets { + mock_models, + mock_sources, + mock_seeds, + mock_table_functions, + macro_mocks, + expected_models, + assertion_targets, + }, + )) + }) + } + + /// Python's error for a scenario's first disallowed `(cte, is check, sources)` read, or `None`. + fn scenario_source_error( + &self, + file_label: &str, + ctes: Vec<(String, bool, Vec)>, + ) -> PyResult> { + compiler_guard(|| { + let mut cte_sources: Vec = Vec::with_capacity(ctes.len()); + for (name, check, sources) in ctes { + cte_sources.push(ScenarioCteSources { + name, + check, + sources, + }); + } + Ok(scenario_source_violation( + file_label, + &cte_sources, + &self.catalog.sources, + )) + }) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 47e6dab755..854d26027b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -175,7 +175,46 @@ class SqlReferenceScanner: def scope_expected_model_names( sqls: list[str], syntax: dict[str, object] ) -> list[list[str] | None]: ... +def extract_sql_scenario_json(sql: str, file_label: str) -> str | None: ... + +class SqlTestTargetCatalog: + def __init__( + self, + models: set[str], + sources: set[str], + seeds: set[str], + resources: tuple[set[str], set[str]], + ) -> None: ... + def unknown_test_target( + self, + file_label: str, + targets: tuple[list[str], list[str], list[str], list[str], list[str], list[str], list[str]], + ) -> str | None: ... + def scenario_source_error( + self, file_label: str, ctes: list[tuple[str, bool, list[str]]] + ) -> str | None: ... + +def omitted_ceremonial_select(sql: str, syntax: dict[str, object]) -> tuple[bool, int | None]: ... +def scan_test_parameter_references( + sql: str, declared: list[str] +) -> list[tuple[int, int, str]] | None: ... def sql_free_of_cursor_intrinsics(sql: str, reserved_markers: list[str]) -> bool: ... +def parse_function_header_values( + header_values: dict[str, object], python: bool +) -> ( + tuple[ + list[tuple[str, str, str]], + str | None, + list[tuple[str, str, str]] | None, + list[str], + str | None, + str | None, + str | None, + list[str], + ] + | None +): ... +def resolve_function_namespace_values(inputs: dict[str, object]) -> list[str | None]: ... def pair_seed_files( declarations: list[str], stems: list[str] ) -> tuple[list[int], None] | tuple[None, int]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/observe.py b/src/sqlbuild/cli/commands/_helpers/freshness/observe.py index 03281a4c2b..24daf99bae 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/observe.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/observe.py @@ -87,7 +87,6 @@ def _observe_source_freshness_for_command( previous_records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] | None, previous_records_by_source_name: dict[str, SourceFreshnessRecord] | None, ) -> FreshnessCommandResult: - selected_sources: tuple[SourceEntry, ...] = _selected_sources( sources=sources, select=select, diff --git a/src/sqlbuild/compiler/attachments/main/_native_test_target_catalog.py b/src/sqlbuild/compiler/attachments/main/_native_test_target_catalog.py new file mode 100644 index 0000000000..ce59be2df8 --- /dev/null +++ b/src/sqlbuild/compiler/attachments/main/_native_test_target_catalog.py @@ -0,0 +1,22 @@ +"""Build the native SQL test target catalog once per compile for the preview compiler engine.""" + +from __future__ import annotations + +from collections.abc import Collection + +import sqlbuild._native as _native + + +def native_test_target_catalog( + *, + models: Collection[str] = (), + sources: Collection[str] = (), + seeds: Collection[str] = (), + table_functions: Collection[str] = (), + macros: Collection[str] = (), +) -> _native.SqlTestTargetCatalog: + """Return the project's resource names as one native catalog shared by every test.""" + + return _native.SqlTestTargetCatalog( + set(models), set(sources), set(seeds), (set(table_functions), set(macros)) + ) diff --git a/src/sqlbuild/compiler/attachments/main/_parse_native_function_header.py b/src/sqlbuild/compiler/attachments/main/_parse_native_function_header.py new file mode 100644 index 0000000000..d3f4b2403f --- /dev/null +++ b/src/sqlbuild/compiler/attachments/main/_parse_native_function_header.py @@ -0,0 +1,45 @@ +"""Parse a SQL or Python function header natively for the preview compiler engine.""" + +from __future__ import annotations + +import sqlbuild._native as _native +from sqlbuild.compiler.attachments.models import NativeFunctionHeader, NativeNamedType + + +def parse_native_function_header( + *, header_values: dict[str, object], python: bool +) -> NativeFunctionHeader | None: + """Return the parsed header, or None where Python must parse it and raise its errors.""" + + if not isinstance(header_values, dict): + return None + row: ( + tuple[ + list[tuple[str, str, str]], + str | None, + list[tuple[str, str, str]] | None, + list[str], + str | None, + str | None, + str | None, + list[str], + ] + | None + ) = _native.parse_function_header_values(header_values, python) + if row is None: + return None + arguments, returns, columns, tags, description, runtime_version, entry_point, packages = row + return NativeFunctionHeader( + arguments=_named_types(arguments), + returns=returns, + return_columns=None if columns is None else _named_types(columns), + tags=tuple(tags), + description=description, + runtime_version=runtime_version, + entry_point=entry_point, + packages=tuple(packages), + ) + + +def _named_types(rows: list[tuple[str, str, str]]) -> tuple[NativeNamedType, ...]: + return tuple(NativeNamedType(*row) for row in rows) diff --git a/src/sqlbuild/compiler/attachments/main/_resolve_native_function_namespace.py b/src/sqlbuild/compiler/attachments/main/_resolve_native_function_namespace.py new file mode 100644 index 0000000000..a30bfbcac1 --- /dev/null +++ b/src/sqlbuild/compiler/attachments/main/_resolve_native_function_namespace.py @@ -0,0 +1,19 @@ +"""Resolve a function's namespace natively for the preview compiler engine.""" + +from __future__ import annotations + +from dataclasses import asdict + +import sqlbuild._native as _native +from sqlbuild.compiler.attachments.models import ( + NativeFunctionNamespace, + NativeFunctionNamespaceInputs, +) + + +def resolve_native_function_namespace( + inputs: NativeFunctionNamespaceInputs, +) -> NativeFunctionNamespace: + """Return the physical, logical and fingerprint namespace Python's attachment assigns.""" + + return NativeFunctionNamespace(*_native.resolve_function_namespace_values(asdict(inputs))) diff --git a/src/sqlbuild/compiler/attachments/models.py b/src/sqlbuild/compiler/attachments/models.py index 5df6644dca..3711e17c4f 100644 --- a/src/sqlbuild/compiler/attachments/models.py +++ b/src/sqlbuild/compiler/attachments/models.py @@ -27,3 +27,54 @@ class NativeRenderedAudit: evidence_sql: str | None severity: str run_scope_source: str + + +@dataclass(frozen=True, slots=True) +class NativeNamedType: + """One function argument or return column: authored name, stripped name, stripped type.""" + + raw_name: str + name: str + type_text: str + + +@dataclass(frozen=True, slots=True) +class NativeFunctionHeader: + """A function header Python accepts, with argument and return types still unexpanded.""" + + arguments: tuple[NativeNamedType, ...] + returns: str | None + return_columns: tuple[NativeNamedType, ...] | None + tags: tuple[str, ...] + description: str | None + runtime_version: str | None + entry_point: str | None + packages: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class NativeFunctionNamespaceInputs: + """Expanded header, project default and target namespaces of one function.""" + + header_database: str | None + header_schema: str | None + default_database: str | None + default_schema: str | None + target_database: str | None + target_schema: str | None + python: bool + inherit_default_namespace: bool + + +@dataclass(frozen=True, slots=True) +class NativeFunctionNamespace: + """Physical, logical and fingerprint database and schema of one function.""" + + database: str | None + schema: str | None + logical_database: str | None + logical_schema: str | None + fingerprint_database: str | None + fingerprint_schema: str | None + fingerprint_logical_database: str | None + fingerprint_logical_schema: str | None diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index ada62426b1..08929955a1 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -480,7 +480,6 @@ def _build_model_inputs( legacy_schema_files: tuple[DiscoveredSchemaFile, ...], render_reuse: CompileRenderReuseSession | None, ) -> tuple[CompileModelInput, ...]: - effective_vars: dict[str, object] = context.effective_vars effective_settings: SettingsConfig = context.effective_settings known_model_names: set[str] = build_known_ref_names(discovered_inputs) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index 331e1e78bf..53a07fa639 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -6,6 +6,17 @@ from pathlib import Path from typing import Any, cast +from sqlbuild.compiler.attachments.main._parse_native_function_header import ( + parse_native_function_header, +) +from sqlbuild.compiler.attachments.main._resolve_native_function_namespace import ( + resolve_native_function_namespace, +) +from sqlbuild.compiler.attachments.models import ( + NativeFunctionHeader, + NativeFunctionNamespace, + NativeFunctionNamespaceInputs, +) from sqlbuild.compiler.compile._helpers.analysis.compact import infer_columns_with_sql_analysis from sqlbuild.compiler.compile._helpers.analysis.validation import ( validate_function_sql_syntax, @@ -99,6 +110,7 @@ def build_sql_function_inputs( """Attach and validate SQL function metadata.""" adapter_name: str = macro_context.adapter_name + native: bool = native_stage_enabled(NativeStage.ATTACHMENTS) known_model_names: set[str] = build_known_ref_names(discovered_inputs) known_seed_names: set[str] = build_known_seed_names(discovered_inputs) known_source_names: set[str] = build_known_source_names(discovered_inputs) @@ -121,21 +133,12 @@ def build_sql_function_inputs( raise CompileInputError(f"Duplicate SQL function name '{function_name}'") known_names.add(function_name) header_values: dict[str, object] = function_file.header_values - raw_returns: object | None = header_values.get("returns") - if raw_returns is None: - raise CompileInputError( - f"SQL function file {function_file.relative_path} must declare returns" - ) - arguments: tuple[FunctionArgument, ...] = _parse_function_arguments( - function_file=function_file, - effective_vars=effective_vars, - ) + native_header: NativeFunctionHeader | None + arguments: tuple[FunctionArgument, ...] returns: str return_columns: tuple[FunctionReturnColumn, ...] - returns, return_columns = _parse_sql_function_returns( - raw_returns=raw_returns, - function_file=function_file, - effective_vars=effective_vars, + native_header, arguments, returns, return_columns = _sql_function_header( + function_file=function_file, effective_vars=effective_vars, native=native ) raw_database: object | None = header_values.get("database") raw_schema: object | None = header_values.get("schema") @@ -163,11 +166,13 @@ def build_sql_function_inputs( logical_schema=function_logical_schema, target_config=target_config, ) - function_database: str | None = ( - function_logical_database if target_database is None else target_database - ) - function_schema: str | None = ( - function_logical_schema if target_schema is None else target_schema + function_database, function_schema = _sql_function_namespace( + header_database=function_logical_database if isinstance(raw_database, str) else None, + header_schema=function_logical_schema if isinstance(raw_schema, str) else None, + logical_namespace=(function_logical_database, function_logical_schema), + defaults=(logical_database, logical_schema), + targets=(target_database, target_schema), + native=native, ) scoped_declarations: DeclarationExpansionContext = resolve_declaration_expansion( context=declaration_expansion, @@ -266,12 +271,16 @@ def build_sql_function_inputs( fingerprint_schema=function_schema, fingerprint_logical_database=function_logical_database, fingerprint_logical_schema=function_logical_schema, - tags=_parse_function_tags( + tags=native_header.tags + if native_header is not None + else _parse_function_tags( header_values=header_values, relative_path=function_file.relative_path, language="SQL", ), - description=_parse_function_description( + description=native_header.description + if native_header is not None + else _parse_function_description( header_values=header_values, relative_path=function_file.relative_path, language="SQL", @@ -306,6 +315,119 @@ def build_sql_function_inputs( return tuple(function_inputs) +def _sql_function_header( + *, function_file: DiscoveredSqlFunctionFile, effective_vars: dict[str, object], native: bool +) -> tuple[ + NativeFunctionHeader | None, tuple[FunctionArgument, ...], str, tuple[FunctionReturnColumn, ...] +]: + header_values: dict[str, object] = function_file.header_values + native_header: NativeFunctionHeader | None = ( + parse_native_function_header(header_values=header_values, python=False) if native else None + ) + if native_header is not None: + arguments: tuple[FunctionArgument, ...] = _expanded_native_arguments( + header=native_header, + label=f"SQL function {function_file.relative_path}", + effective_vars=effective_vars, + ) + return ( + native_header, + arguments, + *_expanded_native_sql_returns( + header=native_header, + relative_path=function_file.relative_path, + effective_vars=effective_vars, + ), + ) + raw_returns: object | None = header_values.get("returns") + if raw_returns is None: + raise CompileInputError( + f"SQL function file {function_file.relative_path} must declare returns" + ) + arguments = _parse_function_arguments( + function_file=function_file, + effective_vars=effective_vars, + ) + return ( + None, + arguments, + *_parse_sql_function_returns( + raw_returns=raw_returns, + function_file=function_file, + effective_vars=effective_vars, + ), + ) + + +def _sql_function_namespace( + *, + header_database: str | None, + header_schema: str | None, + logical_namespace: tuple[str | None, str | None], + defaults: tuple[str | None, str | None], + targets: tuple[str | None, str | None], + native: bool, +) -> tuple[str | None, str | None]: + if not native: + return ( + logical_namespace[0] if targets[0] is None else targets[0], + logical_namespace[1] if targets[1] is None else targets[1], + ) + namespace: NativeFunctionNamespace = resolve_native_function_namespace( + NativeFunctionNamespaceInputs( + header_database=header_database, + header_schema=header_schema, + default_database=defaults[0], + default_schema=defaults[1], + target_database=targets[0], + target_schema=targets[1], + python=False, + inherit_default_namespace=True, + ) + ) + return namespace.database, namespace.schema + + +def _expanded_native_arguments( + *, header: NativeFunctionHeader, label: str, effective_vars: dict[str, object] +) -> tuple[FunctionArgument, ...]: + return tuple( + FunctionArgument( + name=argument.name, + type=_expand_function_header_value( + raw_value=argument.type_text, + effective_vars=effective_vars, + context_label=f"{label} argument '{argument.raw_name}' type", + ), + ) + for argument in header.arguments + ) + + +def _expanded_native_sql_returns( + *, header: NativeFunctionHeader, relative_path: Path, effective_vars: dict[str, object] +) -> tuple[str, tuple[FunctionReturnColumn, ...]]: + if header.return_columns is None: + return _expand_function_header_value( + raw_value=cast(str, header.returns), + effective_vars=effective_vars, + context_label=f"SQL function {relative_path} returns", + ), () + return "TABLE", tuple( + FunctionReturnColumn( + name=column.name, + type=_expand_function_header_value( + raw_value=column.type_text, + effective_vars=effective_vars, + context_label=( + f"SQL function {relative_path} return column '{column.raw_name}' type" + ), + ), + ) + for column in header.return_columns + ) + + def _validate_table_function_output_contract( *, body_sql: str, @@ -335,35 +457,33 @@ def _build_python_function_input( effective_vars: dict[str, object] = context.effective_vars function_name: str = python_function_file.file_path.stem header_values: dict[str, object] = python_function_file.header_values - raw_returns: object | None = header_values.get("returns") - if not isinstance(raw_returns, str) or not raw_returns.strip(): - raise CompileInputError( - f"Python function file {python_function_file.relative_path} must declare returns" - ) - arguments: tuple[FunctionArgument, ...] = _parse_python_function_arguments( - function_file=python_function_file, effective_vars=effective_vars - ) - returns: str = _expand_function_header_value( - raw_value=raw_returns.strip(), - effective_vars=effective_vars, - context_label=f"Python function {python_function_file.relative_path} returns", - ) - runtime_version: str = _parse_required_string_header( - header_values=header_values, - key="runtime_version", - relative_path=python_function_file.relative_path, - language="Python", - ) - entry_point: str = _parse_required_string_header( - header_values=header_values, - key="entry_point", - relative_path=python_function_file.relative_path, - language="Python", - ) - packages: tuple[str, ...] = _parse_python_packages( - raw_packages=header_values.get("packages"), - relative_path=python_function_file.relative_path, + native: bool = native_stage_enabled(NativeStage.ATTACHMENTS) + native_header: NativeFunctionHeader | None = ( + parse_native_function_header(header_values=header_values, python=True) if native else None ) + arguments: tuple[FunctionArgument, ...] + returns: str + runtime_version: str + entry_point: str + packages: tuple[str, ...] + if native_header is not None: + arguments = _expanded_native_arguments( + header=native_header, + label=f"Python function {python_function_file.relative_path}", + effective_vars=effective_vars, + ) + returns = _expand_function_header_value( + raw_value=cast(str, native_header.returns), + effective_vars=effective_vars, + context_label=f"Python function {python_function_file.relative_path} returns", + ) + runtime_version = cast(str, native_header.runtime_version) + entry_point = cast(str, native_header.entry_point) + packages = native_header.packages + else: + arguments, returns, runtime_version, entry_point, packages = _python_function_header( + python_function_file=python_function_file, effective_vars=effective_vars + ) raw_database: object | None = header_values.get("database") raw_schema: object | None = header_values.get("schema") inherit: bool = context.python_functions_inherit_default_namespace @@ -385,38 +505,36 @@ def _build_python_function_input( ) else: function_logical_schema = context.logical_schema if inherit else None - function_database: str | None = ( - function_logical_database - if context.target_database is None - else context.target_database - if isinstance(raw_database, str) or inherit - else None - ) - function_schema: str | None = ( - function_logical_schema - if context.target_schema is None - else context.target_schema - if isinstance(raw_schema, str) or inherit - else None - ) - fingerprint_logical_database: str | None = ( - function_logical_database if isinstance(raw_database, str) else context.logical_database - ) - fingerprint_logical_schema: str | None = ( - function_logical_schema if isinstance(raw_schema, str) else context.logical_schema + namespace: NativeFunctionNamespace = ( + resolve_native_function_namespace( + NativeFunctionNamespaceInputs( + header_database=function_logical_database + if isinstance(raw_database, str) + else None, + header_schema=function_logical_schema if isinstance(raw_schema, str) else None, + default_database=context.logical_database, + default_schema=context.logical_schema, + target_database=context.target_database, + target_schema=context.target_schema, + python=True, + inherit_default_namespace=inherit, + ) + ) + if native + else _python_function_namespace( + raw_database=raw_database, + raw_schema=raw_schema, + logical_database=function_logical_database, + logical_schema=function_logical_schema, + context=context, + ) ) validate_preserved_logical_namespace( resource_label=f"Python function '{function_name}'", - logical_database=fingerprint_logical_database, - logical_schema=fingerprint_logical_schema, + logical_database=namespace.fingerprint_logical_database, + logical_schema=namespace.fingerprint_logical_schema, target_config=context.target_config, ) - fingerprint_database: str | None = ( - fingerprint_logical_database if context.target_database is None else context.target_database - ) - fingerprint_schema: str | None = ( - fingerprint_logical_schema if context.target_schema is None else context.target_schema - ) if context.effective_settings.sql_analysis and not context.no_sql_validation: argument: FunctionArgument for argument in arguments: @@ -439,24 +557,28 @@ def _build_python_function_input( arguments=arguments, returns=returns, body_sql=python_function_file.body_python, - database=function_database, - schema=function_schema, + database=namespace.database, + schema=namespace.schema, logical_database=function_logical_database, logical_schema=function_logical_schema, - fingerprint_database=fingerprint_database, - fingerprint_schema=fingerprint_schema, - fingerprint_logical_database=fingerprint_logical_database, - fingerprint_logical_schema=fingerprint_logical_schema, + fingerprint_database=namespace.fingerprint_database, + fingerprint_schema=namespace.fingerprint_schema, + fingerprint_logical_database=namespace.fingerprint_logical_database, + fingerprint_logical_schema=namespace.fingerprint_logical_schema, language=FunctionLanguage.PYTHON, runtime_version=runtime_version, entry_point=entry_point, packages=packages, - tags=_parse_function_tags( + tags=native_header.tags + if native_header is not None + else _parse_function_tags( header_values=header_values, relative_path=python_function_file.relative_path, language="Python", ), - description=_parse_function_description( + description=native_header.description + if native_header is not None + else _parse_function_description( header_values=header_values, relative_path=python_function_file.relative_path, language="Python", @@ -464,6 +586,81 @@ def _build_python_function_input( ) +def _python_function_header( + *, python_function_file: DiscoveredPythonFunctionFile, effective_vars: dict[str, object] +) -> tuple[tuple[FunctionArgument, ...], str, str, str, tuple[str, ...]]: + header_values: dict[str, object] = python_function_file.header_values + raw_returns: object | None = header_values.get("returns") + if not isinstance(raw_returns, str) or not raw_returns.strip(): + raise CompileInputError( + f"Python function file {python_function_file.relative_path} must declare returns" + ) + arguments: tuple[FunctionArgument, ...] = _parse_python_function_arguments( + function_file=python_function_file, effective_vars=effective_vars + ) + returns: str = _expand_function_header_value( + raw_value=raw_returns.strip(), + effective_vars=effective_vars, + context_label=f"Python function {python_function_file.relative_path} returns", + ) + runtime_version: str = _parse_required_string_header( + header_values=header_values, + key="runtime_version", + relative_path=python_function_file.relative_path, + language="Python", + ) + entry_point: str = _parse_required_string_header( + header_values=header_values, + key="entry_point", + relative_path=python_function_file.relative_path, + language="Python", + ) + packages: tuple[str, ...] = _parse_python_packages( + raw_packages=header_values.get("packages"), + relative_path=python_function_file.relative_path, + ) + return arguments, returns, runtime_version, entry_point, packages + + +def _python_function_namespace( + *, + raw_database: object | None, + raw_schema: object | None, + logical_database: str | None, + logical_schema: str | None, + context: _PythonFunctionBuildContext, +) -> NativeFunctionNamespace: + inherit: bool = context.python_functions_inherit_default_namespace + fingerprint_logical_database: str | None = ( + logical_database if isinstance(raw_database, str) else context.logical_database + ) + fingerprint_logical_schema: str | None = ( + logical_schema if isinstance(raw_schema, str) else context.logical_schema + ) + return NativeFunctionNamespace( + database=logical_database + if context.target_database is None + else context.target_database + if isinstance(raw_database, str) or inherit + else None, + schema=logical_schema + if context.target_schema is None + else context.target_schema + if isinstance(raw_schema, str) or inherit + else None, + logical_database=logical_database, + logical_schema=logical_schema, + fingerprint_database=fingerprint_logical_database + if context.target_database is None + else context.target_database, + fingerprint_schema=fingerprint_logical_schema + if context.target_schema is None + else context.target_schema, + fingerprint_logical_database=fingerprint_logical_database, + fingerprint_logical_schema=fingerprint_logical_schema, + ) + + def _parse_function_arguments( *, function_file: DiscoveredSqlFunctionFile, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index e471b8d787..a2ad27f925 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -5,6 +5,10 @@ from dataclasses import dataclass, replace from pathlib import Path +import sqlbuild._native as _native +from sqlbuild.compiler.attachments.main._native_test_target_catalog import ( + native_test_target_catalog, +) from sqlbuild.compiler.compile._helpers.attachment.references import ( build_known_function_names, build_known_ref_names, @@ -74,6 +78,8 @@ DiscoveredSqlTestFile, EnumDeclaration, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.profiling.main.record import record_compile_timing from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind from sqlbuild.compiler.scopes.models import ( @@ -302,6 +308,17 @@ def build_test_inputs( cache_root=compile_cache_dir, syntax=sql_lexical_syntax, ) + target_catalog: _native.SqlTestTargetCatalog | None = ( + native_test_target_catalog( + models=known_model_names, + sources=known_source_names, + seeds=known_seed_names, + table_functions=known_table_function_names, + macros=loaded_macros, + ) + if native_stage_enabled(NativeStage.ATTACHMENTS) + else None + ) for test, test_ctes in zip(expanded_tests, test_ctes_batch, strict=True): assertion_target_model_names: tuple[str, ...] = ( extract_assertion_target_model_names( @@ -321,6 +338,7 @@ def build_test_inputs( known_table_function_names=known_table_function_names, loaded_macros=loaded_macros, assertion_target_model_names=assertion_target_model_names, + target_catalog=target_catalog, ) helper_targets: tuple[str, ...] = () if isinstance(test_ctes.payload, CompileModelSqlTestCtes): @@ -632,6 +650,11 @@ def build_scenario_inputs( ) ) scenario_inputs: list[CompileSqlScenarioInput] = [] + source_catalog: _native.SqlTestTargetCatalog | None = ( + native_test_target_catalog(sources=known_source_names) + if native_stage_enabled(NativeStage.ATTACHMENTS) + else None + ) scenario_file: DiscoveredSqlScenarioFile for scenario_file in discovered_inputs.scenario_files: resource: ResourceIdentity = ResourceIdentity(ResourceKind.SCENARIO, scenario_file.name) @@ -669,6 +692,7 @@ def build_scenario_inputs( scenario_file=scenario_file, known_source_names=known_source_names, syntax=sql_lexical_syntax, + target_catalog=source_catalog, ) assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( assertion_sql=tuple(cte.sql_body for cte in scenario_ctes.assertion_ctes), @@ -716,7 +740,22 @@ def _validate_scenario_source_references( scenario_file: DiscoveredSqlScenarioFile, known_source_names: set[str], syntax: SqlLexicalSyntax, + target_catalog: _native.SqlTestTargetCatalog | None = None, ) -> None: + native_sources: list[tuple[str, bool, list[str]]] | None = ( + _native_scenario_cte_sources( + scenario_ctes=scenario_ctes, scenario_file=scenario_file, syntax=syntax + ) + if native_stage_enabled(NativeStage.ATTACHMENTS) + else None + ) + if native_sources is not None: + error: str | None = ( + target_catalog or native_test_target_catalog(sources=known_source_names) + ).scenario_source_error(str(scenario_file.relative_path), native_sources) + if error is not None: + raise CompileInputError(error) + return cte: CompileSqlScenarioCte for cte in (*scenario_ctes.expected_ctes, *scenario_ctes.assertion_ctes): references: tuple[CompileSqlReference, ...] = extract_sql_references( @@ -748,6 +787,40 @@ def _validate_scenario_source_references( ) +def _native_scenario_cte_sources( + *, + scenario_ctes: CompileSqlScenarioCtes, + scenario_file: DiscoveredSqlScenarioFile, + syntax: SqlLexicalSyntax, +) -> list[tuple[str, bool, list[str]]] | None: + """Every CTE's source references, or None where extraction raises so Python orders errors.""" + + checks: tuple[CompileSqlScenarioCte, ...] = ( + *scenario_ctes.expected_ctes, + *scenario_ctes.assertion_ctes, + ) + cte_sources: list[tuple[str, bool, list[str]]] = [] + try: + for position, cte in enumerate((*checks, *scenario_ctes.authored_ctes)): + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=cte.sql_body, syntax=syntax, origin=_sql_file_reference_origin(scenario_file) + ) + cte_sources.append( + ( + cte.name, + position < len(checks), + [ + reference.ref_name + for reference in references + if reference.ref_kind == SqlReferenceKind.SOURCE + ], + ) + ) + except CompileInputError: + return None + return cte_sources + + def validate_test_ctes( *, test_ctes: CompileSqlTestCtes, @@ -758,13 +831,37 @@ def validate_test_ctes( known_table_function_names: set[str], loaded_macros: dict[str, LoadedMacro], assertion_target_model_names: tuple[str, ...], + target_catalog: _native.SqlTestTargetCatalog | None = None, ) -> None: - """Validate SQL-native test CTE targets against discovered inputs.""" + """Validate SQL-native test CTE targets; preview reuses one native catalog per compile.""" if isinstance(test_ctes.payload, CompileDirectLogicSqlTestCtes): return model_payload: CompileModelSqlTestCtes = test_ctes.payload + if native_stage_enabled(NativeStage.ATTACHMENTS): + catalog: _native.SqlTestTargetCatalog = target_catalog or native_test_target_catalog( + models=known_model_names, + sources=known_source_names, + seeds=known_seed_names, + table_functions=known_table_function_names, + macros=loaded_macros, + ) + error: str | None = catalog.unknown_test_target( + str(test_file.relative_path), + ( + [*model_payload.mock_model_names], + [*model_payload.mock_source_names], + [*model_payload.mock_seed_names], + [*model_payload.mock_table_function_names], + [*model_payload.macro_mocks], + [*model_payload.expected_model_names], + [*assertion_target_model_names], + ), + ) + if error is not None: + raise CompileInputError(error) + return mock_model_name: str for mock_model_name in model_payload.mock_model_names: diff --git a/src/sqlbuild/compiler/compile/_helpers/render/parameters.py b/src/sqlbuild/compiler/compile/_helpers/render/parameters.py index c35e14782f..0c339f505c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/parameters.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/parameters.py @@ -5,9 +5,12 @@ import re from pathlib import Path +import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import SQL_QUOTE_TOKENS from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import TypedSqlValueRenderer +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import skip_quoted_text @@ -33,6 +36,20 @@ def expand_test_parameters( """Render active parameter references while leaving comments and quoted text unchanged.""" value_lookup: dict[str, SqlValue] = dict(values) + references: list[tuple[int, int, str]] | None = ( + _native.scan_test_parameter_references(sql, list(value_lookup)) + if native_stage_enabled(NativeStage.ATTACHMENTS) + else None + ) + if references is not None: + return _splice_parameter_references( + sql=sql, + references=references, + value_lookup=value_lookup, + value_renderer=value_renderer, + test_name=test_name, + case_name=case_name, + ) used_names: set[str] = set() parts: list[str] = [] cursor: int = 0 @@ -68,21 +85,67 @@ def expand_test_parameters( f"SQL test '{test_name}' case '{case_name}' in '{file_path}' references " f"undeclared parameter '{name}'" ) - try: - rendered: str = value_renderer.render_typed_scalar(value=value) - validate_rendered_sql_value_size( - rendered_sql=rendered, - context=(f"SQL test '{test_name}' case '{case_name}' parameter '{name}'"), + parts.append( + _render_parameter( + name=name, + value=value, + value_renderer=value_renderer, + test_name=test_name, + case_name=case_name, ) - except (SqlValueRenderingError, SqlValueValidationError) as error: - raise CompileInputError( - f"SQL test '{test_name}' case '{case_name}' parameter '{name}' could not " - f"be rendered by adapter '{value_renderer.adapter_name}': {error}" - ) from error - parts.append(rendered) + ) used_names.add(name) cursor = match.end() continue parts.append(character) cursor += 1 return "".join(parts), frozenset(used_names) + + +def _splice_parameter_references( + *, + sql: str, + references: list[tuple[int, int, str]], + value_lookup: dict[str, SqlValue], + value_renderer: TypedSqlValueRenderer, + test_name: str, + case_name: str, +) -> tuple[str, frozenset[str]]: + parts: list[str] = [] + cursor: int = 0 + for start, end, name in references: + parts.append(sql[cursor:start]) + parts.append( + _render_parameter( + name=name, + value=value_lookup[name], + value_renderer=value_renderer, + test_name=test_name, + case_name=case_name, + ) + ) + cursor = end + parts.append(sql[cursor:]) + return "".join(parts), frozenset(name for _start, _end, name in references) + + +def _render_parameter( + *, + name: str, + value: SqlValue, + value_renderer: TypedSqlValueRenderer, + test_name: str, + case_name: str, +) -> str: + try: + rendered: str = value_renderer.render_typed_scalar(value=value) + validate_rendered_sql_value_size( + rendered_sql=rendered, + context=(f"SQL test '{test_name}' case '{case_name}' parameter '{name}'"), + ) + except (SqlValueRenderingError, SqlValueValidationError) as error: + raise CompileInputError( + f"SQL test '{test_name}' case '{case_name}' parameter '{name}' could not " + f"be rendered by adapter '{value_renderer.adapter_name}': {error}" + ) from error + return rendered diff --git a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py index 144cb43faf..0f0d3388bb 100644 --- a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py @@ -2,6 +2,11 @@ from __future__ import annotations +from typing import Any + +import orjson + +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.analysis.ctes import ( extract_top_level_ctes_with_sql_analysis, ) @@ -27,6 +32,8 @@ CompileSqlScenarioCte, CompileSqlScenarioCtes, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _CONTEXT: str = "SQL scenario" @@ -40,6 +47,14 @@ def extract_sql_scenario_ctes( ) -> CompileSqlScenarioCtes: """Extract top-level SQL-native scenario fixture, expected, and assertion CTEs.""" + native_ctes: CompileSqlScenarioCtes | None = ( + _native_scenario_ctes(sql=sql, file_label=file_label) + if native_stage_enabled(NativeStage.ATTACHMENTS) + and not syntax.reads_differently_from_generic(sql) + else None + ) + if native_ctes is not None: + return native_ctes try: ctes: tuple[CompileSqlScenarioCte, ...] = extract_top_level_ctes_with_scanner( sql=sql, @@ -61,6 +76,28 @@ def extract_sql_scenario_ctes( return _classify_sql_scenario_ctes(ctes=ctes, file_label=file_label, syntax=syntax) +def _native_scenario_ctes(*, sql: str, file_label: str) -> CompileSqlScenarioCtes | None: + response: str | None = _native.extract_sql_scenario_json(sql, file_label) + if response is None: + return None + payload: dict[str, list[Any]] = orjson.loads(response) + return CompileSqlScenarioCtes( + authored_ctes=_scenario_ctes(payload["authored"]), + expected_ctes=_scenario_ctes(payload["expected"]), + assertion_ctes=_scenario_ctes(payload["assertions"]), + source_fixture_names=tuple(payload["sourceFixtures"]), + ref_fixture_names=tuple(payload["refFixtures"]), + seed_fixture_names=tuple(payload["seedFixtures"]), + dbt_ref_fixture_names=tuple(payload["dbtRefFixtures"]), + expected_model_names=tuple(payload["expectedModels"]), + assertion_names=tuple(payload["assertionNames"]), + ) + + +def _scenario_ctes(values: list[list[str]]) -> tuple[CompileSqlScenarioCte, ...]: + return tuple(CompileSqlScenarioCte(name=name, sql_body=body) for name, body in values) + + def extract_sql_scenario_expected_model_names( *, sql: str, file_label: str, syntax: SqlLexicalSyntax ) -> tuple[str, ...]: diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index 3673fdaace..4ec560a073 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Any +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.analysis.ctes import ( extract_top_level_ctes_with_sql_analysis, ) @@ -58,6 +59,8 @@ from sqlbuild.compiler.discovery.main.omitted_ceremonial_select import ( omitted_ceremonial_select_offset, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main._find_matching_paren import find_matching_paren from sqlbuild.compiler.sql_analysis.main._is_identifier_character import ( @@ -262,7 +265,15 @@ def extract_top_level_ctes_with_scanner[CteT]( def complete_omitted_ceremonial_select(*, sql: str, syntax: SqlLexicalSyntax) -> str: """Return test or scenario SQL as one statement, adding an omitted trailing `SELECT 1`.""" - offset: int | None = omitted_ceremonial_select_offset(sql=sql, syntax=syntax) + answered: bool + offset: int | None + answered, offset = ( + _native.omitted_ceremonial_select(sql, syntax.native_mapping) + if native_stage_enabled(NativeStage.ATTACHMENTS) + else (False, None) + ) + if not answered: + offset = omitted_ceremonial_select_offset(sql=sql, syntax=syntax) if offset is None: return sql return f"{sql[:offset]}{OMITTED_CEREMONIAL_SELECT_SQL}{sql[offset:]}" diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/observation.py b/src/sqlbuild/compiler/source_freshness/_helpers/observation.py index a1dba0b765..67863bbd85 100644 --- a/src/sqlbuild/compiler/source_freshness/_helpers/observation.py +++ b/src/sqlbuild/compiler/source_freshness/_helpers/observation.py @@ -63,7 +63,6 @@ def _observe_configured_source_freshness( source: SourceEntry, observed_at: datetime, ) -> SourceFreshnessObservation: - config: SourceFreshnessConfig | None = source.freshness if config is None: raise SourceFreshnessObservationError( diff --git a/src/sqlbuild/executor/janitor/main/execute.py b/src/sqlbuild/executor/janitor/main/execute.py index e94b82ee62..bcda614982 100644 --- a/src/sqlbuild/executor/janitor/main/execute.py +++ b/src/sqlbuild/executor/janitor/main/execute.py @@ -51,7 +51,6 @@ def _execute_janitor_plan( adapter: BaseAdapter, connection: Any, ) -> JanitorExecutionResult: - recorder: StatementRecorder = StatementRecorder() candidate: JanitorDeleteCandidate for candidate in () if plan.direct_mode else plan.candidates: diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index 06804ab1db..ea10e1add6 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -105,7 +105,6 @@ def _execute_source_load( ref_bindings: LoaderRefBindings, on_progress: Callable[[str], None] | None = None, ) -> LoadExecutionResult: - destination_name: str = ( source_entry.table if source_entry.table is not None else source_entry.name ) diff --git a/src/sqlbuild/executor/scenario/_helpers/capture/core.py b/src/sqlbuild/executor/scenario/_helpers/capture/core.py index b210b1bd9c..7cd3e0e670 100644 --- a/src/sqlbuild/executor/scenario/_helpers/capture/core.py +++ b/src/sqlbuild/executor/scenario/_helpers/capture/core.py @@ -77,7 +77,6 @@ def _execute_scenario_snapshot_capture_steps( settings: ScenarioCaptureSettings, local_type_overrides: dict[str, str] | None, ) -> ScenarioSnapshotCaptureRunResult: - prepare_result: ScenarioCleanupExecutionResult = execute_scenario_cleanup( scenario_plan=scenario_plan, adapter=adapter, diff --git a/src/sqlbuild/executor/scenario/_helpers/execution/run.py b/src/sqlbuild/executor/scenario/_helpers/execution/run.py index f72438428e..46012a9668 100644 --- a/src/sqlbuild/executor/scenario/_helpers/execution/run.py +++ b/src/sqlbuild/executor/scenario/_helpers/execution/run.py @@ -84,7 +84,6 @@ def _execute_scenario_run_steps( retain: bool, options: ScenarioRunOptions, ) -> ScenarioRunResult: - _raise_if_stop_requested(options) prepare_result: ScenarioCleanupExecutionResult = execute_scenario_cleanup( scenario_plan=scenario_plan, diff --git a/src/sqlbuild/executor/scenario/main/_cleanup.py b/src/sqlbuild/executor/scenario/main/_cleanup.py index 831fc6c66f..78e1b5f68d 100644 --- a/src/sqlbuild/executor/scenario/main/_cleanup.py +++ b/src/sqlbuild/executor/scenario/main/_cleanup.py @@ -47,7 +47,6 @@ def _execute_scenario_cleanup( adapter: BaseAdapter, connection: Any, ) -> ScenarioCleanupExecutionResult: - statement_recorder: StatementRecorder = StatementRecorder() cleanup_targets: tuple[ScenarioCleanupTarget, ...] = collect_scenario_cleanup_targets( scenario_plan=scenario_plan, diff --git a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py index 746dcf7d9c..cca7097b9f 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py @@ -4,6 +4,8 @@ from collections.abc import Callable from dataclasses import dataclass +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + @dataclass(frozen=True) class AuthoredSqlParityTestCase: @@ -34,7 +36,9 @@ class AttachmentProjectTestCase: """One project variant compiled to compile inputs by each engine.""" description: str + overrides: dict[str, str] expected_preview_entries: frozenset[str] + expected_outcome_fragment: str @dataclass(frozen=True) @@ -46,3 +50,73 @@ class CursorIntrinsicParityTestCase: count: int expected_minimum_free: int expected_minimum_python_errors: int + + +@dataclass(frozen=True) +class ParameterParityTestCase: + """Seeded SQL test bodies whose `@param` references each engine expands.""" + + description: str + seed: int + count: int + expected_minimum_expanded: int + expected_minimum_python_errors: int + + +@dataclass(frozen=True) +class OmittedSelectParityTestCase: + """Seeded test bodies completed with an omitted `SELECT 1` under one lexical syntax.""" + + description: str + seed: int + count: int + syntax: SqlLexicalSyntax + expected_minimum_completed: int + + +@dataclass(frozen=True) +class RawDirectLogicParityTestCase: + """Seeded unexpanded direct-logic test bodies extracted natively and by Python.""" + + description: str + seed: int + count: int + expected_minimum_extracted: int + expected_minimum_python_errors: int + + +@dataclass(frozen=True) +class ScenarioParityTestCase: + """Seeded scenario bodies extracted by each engine under one lexical syntax.""" + + description: str + seed: int + count: int + syntax: SqlLexicalSyntax + expected_minimum_extracted: int + expected_minimum_native: int + expected_minimum_python_errors: int + + +@dataclass(frozen=True) +class TargetParityTestCase: + """Seeded model test targets validated by each engine against known resources.""" + + description: str + seed: int + count: int + expected_minimum_valid: int + expected_minimum_python_errors: int + + +@dataclass(frozen=True) +class FunctionHeaderParityTestCase: + """Seeded SQL and Python function headers attached by each engine.""" + + description: str + seed: int + count: int + target_schema: str | None + inherit_default_namespace: bool + expected_minimum_attached: int + expected_minimum_python_errors: int diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index 7433042c98..37d89ce9db 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -24,19 +24,42 @@ resolve_audit_run_scope, resolve_audit_severity, ) +from sqlbuild.compiler.compile._helpers.attachment.functions import build_sql_function_inputs +from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( + build_test_inputs, + validate_test_ctes, +) from sqlbuild.compiler.compile._helpers.render import macros from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics +from sqlbuild.compiler.compile._helpers.render.parameters import expand_test_parameters from sqlbuild.compiler.compile._helpers.render.sql_vars import expand_authored_sql_result +from sqlbuild.compiler.compile._helpers.scenarios.core import extract_sql_scenario_ctes +from sqlbuild.compiler.compile._helpers.sql_tests.core import ( + complete_omitted_ceremonial_select, + extract_sql_test_ctes, +) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, CompileAdapterContext, + CompileModelSqlTestCtes, CompileProjectInputs, + CompileSqlTestCtes, + DeclarationExpansionContext, + DeclarationResolutionContext, + LoadedMacro, MacroContext, ) +from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.discovery.models import ( + DiscoveredProjectInputs, + DiscoveredPythonFunctionFile, + DiscoveredSqlFunctionFile, + DiscoveredSqlTestBlock, + DiscoveredSqlTestFile, +) from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage @@ -44,10 +67,23 @@ MICROBATCH_END_SENTINEL, MICROBATCH_START_SENTINEL, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.main.resolve_effective_collection_rendering import ( resolve_effective_collection_rendering, ) +from sqlbuild.spec.contracts.models import ( + DefaultsConfig, + LocalConfig, + ProjectConfig, + SettingsConfig, + TargetConfig, +) +from sqlbuild.sql_values.main.normalize import normalize_sql_value +from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import CollectionRendering +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + FunctionHeaderParityTestCase, +) from tests.integration.src.sqlbuild.compiler.model_loop.helpers import ( DECLARATIONS, generated_reference_sql, @@ -312,6 +348,10 @@ def _python_rendering(audit: GeneratedAudit) -> tuple[object, ...] | str: NATIVE_ATTACHMENT_ENTRIES: tuple[str, ...] = ( + "scan_test_parameter_references", + "omitted_ceremonial_select", + "extract_sql_scenario_json", + "SqlTestTargetCatalog", "pair_seed_files", "render_attached_generic_audit", "expand_config_templates", @@ -374,6 +414,20 @@ def _python_rendering(audit: GeneratedAudit) -> tuple[object, ...] | str: "__expected__orders AS (\n SELECT 1 AS id, 1.5 AS amount, 'web' AS label\n)\n" "SELECT 1\n" ), + "tests/unit/test_orders_cases.sql": ( + 'TEST (\n name "orders_keep_region",\n parameters (\n region_value string,\n ),\n' + ' cases (\n north (region_value "north"),\n' + ' quoted (region_value "it\'s"),\n ),\n);\n\n' + "WITH\n__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label),\n" + '__source__order_events AS (SELECT 1 AS id, @param("region_value") AS region),\n' + "__expected__orders AS (SELECT 1 AS id, 1.5 AS amount, 'web' AS label);\n" + ), + "tests/unit/test_tidy_label.sql": ( + 'TEST (mode macro, name "tidies_labels");\n\n' + "WITH\ninput_values AS (SELECT ' Web ' AS label),\n" + '__macro_actual__ AS (SELECT @tidy_label("label") AS label FROM input_values),\n' + "__macro_expected__ AS (SELECT 'web' AS label)\nSELECT 1\n" + ), "tests/scenarios/orders_scenario.sql": ( 'SCENARIO (\n description "Orders join their channel"\n);\n\n' "WITH\n__seed__channel_codes AS (\n SELECT 1 AS id, @tidy_label(\"'Web'\") AS label\n),\n" @@ -388,11 +442,12 @@ def attachment_engine_outcome( *, project_dir: Path, engine: CompilerEngine, + overrides: dict[str, str], monkeypatch: pytest.MonkeyPatch, ) -> tuple[frozenset[str], str]: """Build compile inputs under `engine`; return the native entries called and the outcome.""" - for relative_path, contents in ATTACHMENT_PROJECT.items(): + for relative_path, contents in {**ATTACHMENT_PROJECT, **overrides}.items(): path: Path = project_dir / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(contents, encoding="utf-8") @@ -402,10 +457,13 @@ def attachment_engine_outcome( bridged: list[Path] = [] monkeypatch.setattr(macros, "_bridged_sql_macros", _recorded_bridge(bridged=bridged)) monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) - inputs: CompileProjectInputs = compile_frontier( - until=CompilerStage.COMPILE_PROJECT_INPUTS, - python_stage=partial(_compile_inputs, project_dir=project_dir), - ) + try: + inputs: CompileProjectInputs = compile_frontier( + until=CompilerStage.COMPILE_PROJECT_INPUTS, + python_stage=partial(_compile_inputs, project_dir=project_dir), + ) + except CompileInputError as error: + return frozenset(called), f"error: {error}".replace(str(project_dir), "") called.update(f"bridged:{str(path).replace(str(project_dir), '')}" for path in bridged) return frozenset(called), repr( ( @@ -415,6 +473,7 @@ def attachment_engine_outcome( inputs.audit_inputs, inputs.test_inputs, inputs.scenario_inputs, + inputs.diagnostics, ) ).replace(str(project_dir), "") @@ -511,3 +570,461 @@ def native_intrinsic_free(sql: str) -> bool: return _native.sql_free_of_cursor_intrinsics( sql, [MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL] ) + + +_PARAMETER_PIECES: tuple[str, ...] = ( + "SELECT ", + '@param("region")', + '@param ( "limit_rows" )', + '@param(\n"region"\n)', + '@param(\u00a0"region")', + '@param("missing")', + "@param(region)", + "@params", + "@param_x", + "'@param(\"region\")'", + '"quoted"', + "`tick`", + '$$ @param("region") $$', + "$1", + '-- @param("region")\n', + '/* @param("region") */', + "'", + "/*", + "\u00e9", + ", ", +) +PARAMETER_VALUES: tuple[tuple[str, SqlValue], ...] = ( + ("region", normalize_sql_value(raw_value="north", context="region")), + ("limit_rows", normalize_sql_value(raw_value=10, context="limit_rows")), +) + + +def generated_parameter_sql(*, rng: random.Random) -> str: + """Return a test body mixing parameter references with quotes and comments.""" + + return "".join(rng.choices(_PARAMETER_PIECES, k=rng.randint(1, 8))) + + +def parameter_outcome( + *, sql: str, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch +) -> tuple[str, frozenset[str]] | str: + """Expand one body's parameters under `engine`, returning Python's error text on failure.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + try: + return expand_test_parameters( + sql=sql, + file_path=Path("tests/unit/test_orders.sql"), + values=PARAMETER_VALUES, + value_renderer=DuckDbAdapter(), + test_name="orders", + case_name="north", + ) + except CompileInputError as error: + return str(error) + + +_BODY_PIECES: tuple[str, ...] = ( + "WITH ", + "with ", + "WITHIN ", + "w\u0131th ", + "__seed__orders AS (SELECT 1)", + "a AS (SELECT ')' AS x)", + "b AS (SELECT (1))", + ", ", + ")", + "(", + " SELECT 1", + ";", + "\n", + " -- done", + " # done", + " /* done */", + "/* /* nested */ */", + "'", + "'it\\'s'", + "'''tri)ple'''", + "$$)$$", + "$1", + "\u00e9", + "\v", +) + + +def generated_test_body(*, rng: random.Random) -> str: + """Return a test or scenario body around CTEs, terminators, comments and quotes.""" + + keyword: str = rng.choice(_BODY_PIECES[:4]) + ctes: str = ", ".join(rng.choices(_BODY_PIECES[4:7], k=rng.randint(1, 3))) + return keyword + ctes + "".join(rng.choices(_BODY_PIECES, k=rng.randint(0, 3))) + + +def completed_body( + *, + sql: str, + syntax: SqlLexicalSyntax, + engine: CompilerEngine, + monkeypatch: pytest.MonkeyPatch, +) -> str: + """Complete one body's omitted `SELECT 1` under `engine`.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + return complete_omitted_ceremonial_select(sql=sql, syntax=syntax) + + +_RAW_MODE_CTES: dict[SqlTestMode, tuple[str, str]] = { + SqlTestMode.MACRO: ("__macro_actual__", "__macro_expected__"), + SqlTestMode.UDF: ("__udf_actual__", "__udf_expected__"), + SqlTestMode.TABLE_FN: ("__table_fn_actual__", "__table_fn_expected__"), +} +_RAW_CALLS: tuple[str, ...] = ( + "", + "@tidy_label(1)", + "@MY_MACRO(1)", + '__udf("order_label")', + '__udf ("order_label")', + "__udf('order_label')", + '__udf("order_label", 1)', + '__table_fn("order_rows")', + "'__udf(\"order_label\")'", + '-- __udf("order_label")\n', + "/* @tidy_label(1) */", + '$$__udf("order_label")$$', + "@m()", + "@@region", + '__ref("orders")', + '__source("events", "orders")', + '__udf("\u00e9")', + '`__udf("order_label")`', + '__udf\u00a0("order_label")', + '\u00e9__udf("order_label")', +) +_RAW_CTE_NAMES: tuple[str, ...] = ( + "helper", + "a\u00e9", + '"quoted"', + "`ticked`", + "a$b", + "__ref__orders", + "__expected__orders", + "__macro__tidy", + "__udf_actual__", + "__udf_expected__", + "__macro_expected__", + "__table_fn_expected__", + "HELPER", + "\u0131", + "\u017f", +) +_RAW_TAILS: tuple[str, ...] = ("", " FROM helper", " -- c", " UNION ALL SELECT 2", ", 'x' AS b") + + +def _raw_body(rng: random.Random) -> str: + call: str = rng.choice(("1", "1", "1", "1", *_RAW_CALLS)) + projection: str = rng.choice((f"SELECT {call} AS a", f"SELECT {call} AS a", f"SELECT {call}")) + return projection + rng.choice(("", "", "", *_RAW_TAILS)) + + +def generated_raw_direct_logic_test(*, rng: random.Random) -> tuple[str, SqlTestMode]: + """Return one unexpanded direct-logic test with the shapes Python and native read apart.""" + + mode: SqlTestMode = rng.choice(tuple(_RAW_MODE_CTES)) + actual, expected = _RAW_MODE_CTES[mode] + ctes: list[tuple[str, str]] = [ + (actual, _raw_body(rng)), + (expected, _raw_body(rng)), + *((rng.choice(_RAW_CTE_NAMES), _raw_body(rng)) for _ in range(rng.choice((0, 0, 1, 2)))), + ] + rng.shuffle(ctes) + body: str = ", ".join(f"{name} AS ({sql})" for name, sql in ctes) + return f"WITH {body}" + rng.choice(("", "", " SELECT 1", ";")), mode + + +def python_raw_extraction_accepts(*, sql: str, mode: SqlTestMode) -> bool: + """Whether Python's raw direct-logic pass accepts `sql` before expansion.""" + + try: + extract_sql_test_ctes( + sql=sql, + file_label="tests/unit/test_logic.sql", + syntax=DuckDbAdapter().sql_lexical_syntax, + mode=mode, + ) + except CompileInputError: + return False + return True + + +def raw_test_compile_outcome( + *, sql: str, mode: SqlTestMode, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch +) -> str: + """Build one unexpanded direct-logic test's input under `engine`, or Python's error text.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + adapter: DuckDbAdapter = DuckDbAdapter() + try: + return repr( + build_test_inputs( + discovered_inputs=DiscoveredProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(), + test_files=( + DiscoveredSqlTestFile( + file_path=Path("/project/tests/unit/test_logic.sql"), + relative_path=Path("tests/unit/test_logic.sql"), + contents=sql, + blocks=( + DiscoveredSqlTestBlock( + test_index=0, header_values={}, sql_body=sql, mode=mode + ), + ), + ), + ), + ), + effective_vars={"region": "north"}, + macro_context=_MACRO_CONTEXT, + loaded_macros={}, + declaration_expansion=DeclarationExpansionContext( + declarations=DeclarationResolutionContext(), + value_renderer=adapter, + collection_rendering=CollectionRendering.VALUE_LIST, + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ) + ) + except CompileInputError as error: + return f"error: {error}" + + +_SCENARIO_CTES: tuple[str, ...] = ( + "__source__raw_orders AS (SELECT 1 AS id, 'placed' AS status)", + "__ref__customers AS (SELECT 1 AS id)", + "__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label)", + "__dbt_ref__legacy__orders AS (SELECT 1 AS id)", + "__table_fn__order_rows AS (SELECT 1 AS id)", + "helper AS (SELECT id FROM __source__raw_orders)", + "__expected__orders AS (SELECT 1 AS id)", + "__expected__orders_view AS (SELECT id FROM helper)", + "__assert__positive AS (SELECT id FROM helper WHERE id < 0)", + "__assert__matches AS (SELECT * FROM __expected__orders)", + "__expected__nested AS (WITH __assert__inner AS (SELECT 1) SELECT 1 AS id)", + "__macro__tidy AS (SELECT 'x')", + '"__source__quoted" AS (SELECT 1)', + "__expected__ AS (SELECT 1)", + "__source__raw_orders AS (SELECT 2 AS id)", + "notes AS (SELECT 'caf\u00e9 -- )' AS note /* ) */)", + "dollar AS (SELECT $$ ) $$ AS note)", + "hashed AS (SELECT 1 # )\n)", +) +_SCENARIO_TAILS: tuple[str, ...] = ("\nSELECT 1\n", "", ";", "\nSELECT 2", " -- end", ";\n") +_SCENARIO_KEYWORDS: tuple[str, ...] = ( + "WITH\n", + "with ", + "WITH RECURSIVE ", + "w\u0131th ", + "SELECT ", +) + + +def generated_scenario(*, rng: random.Random) -> str: + """Return one scenario body mixing fixtures, checks, helpers and invalid CTEs.""" + + ctes: list[str] = rng.sample(_SCENARIO_CTES, k=rng.randint(1, 5)) + return rng.choice(_SCENARIO_KEYWORDS) + ",\n".join(ctes) + rng.choice(_SCENARIO_TAILS) + + +def scenario_outcome( + *, + sql: str, + syntax: SqlLexicalSyntax, + engine: CompilerEngine, + monkeypatch: pytest.MonkeyPatch, +) -> object: + """Extract one scenario under `engine`, or return Python's error text.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + try: + return extract_sql_scenario_ctes( + sql=sql, file_label="tests/scenarios/orders.sql", syntax=syntax + ) + except CompileInputError as error: + return str(error) + + +def native_scenario_answered(sql: str) -> bool: + """Whether the native extraction answers instead of deferring to Python.""" + + return _native.extract_sql_scenario_json(sql, "tests/scenarios/orders.sql") is not None + + +_TARGET_NAMES: tuple[str, ...] = ("orders", "customers", "channel_codes", "returns", "order_rows") +_KNOWN_NAMES: frozenset[str] = frozenset({"orders", "customers", "channel_codes", "order_rows"}) + + +def generated_test_targets( + *, rng: random.Random +) -> tuple[CompileModelSqlTestCtes, tuple[str, ...]]: + """Return one model test payload and assertion targets mixing known and unknown names.""" + + def names() -> tuple[str, ...]: + return tuple(rng.sample(_TARGET_NAMES, k=rng.choice((0, 0, 1, 1, 2)))) + + return ( + CompileModelSqlTestCtes( + macro_mocks=dict.fromkeys(names(), "'x'"), + mock_model_names=names(), + mock_source_names=names(), + mock_seed_names=names(), + mock_table_function_names=names(), + expected_model_names=names(), + ), + names(), + ) + + +def target_validation_outcome( + *, + payload: CompileModelSqlTestCtes, + assertion_targets: tuple[str, ...], + engine: CompilerEngine, + monkeypatch: pytest.MonkeyPatch, +) -> str | None: + """Validate one payload's targets under `engine`, returning Python's error text.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + known: set[str] = set(_KNOWN_NAMES) + try: + validate_test_ctes( + test_ctes=CompileSqlTestCtes(mode=SqlTestMode.MODEL, payload=payload), + test_file=DiscoveredSqlTestFile( + file_path=Path("/project/tests/unit/test_orders.sql"), + relative_path=Path("tests/unit/test_orders.sql"), + contents="", + blocks=(), + ), + known_model_names=known, + known_seed_names=known, + known_source_names=known, + known_table_function_names=known, + loaded_macros=dict.fromkeys(known, cast(LoadedMacro, None)), + assertion_target_model_names=assertion_targets, + ) + except CompileInputError as error: + return str(error) + return None + + +_TYPES: tuple[object, ...] = ( + "STRING", + " INTEGER ", + " ", + "${coalesce(ENV:SQB_ATTACHMENTS_UNSET, 'DOUBLE')}", + "${missing}", + 2, +) +_NAMES: tuple[object, ...] = ("raw_status", " amount ", "", " ", 1) +_TEXTS: tuple[object, ...] = ("orders", " Orders. ", "", " ", 3, None) +_SCHEMAS: tuple[object, ...] = ( + "udfs", + "${coalesce(ENV:SQB_ATTACHMENTS_UNSET, 'udfs')}", + 1, + None, +) + + +def _random_map(rng: random.Random, keys: tuple[object, ...]) -> dict[object, object]: + return {rng.choice(keys): rng.choice(_TYPES) for _ in range(rng.randint(0, 3))} + + +def generated_function_header(*, rng: random.Random, python: bool) -> dict[str, object]: + """Return SQL or Python function header values Python may accept or reject.""" + + returns: tuple[object, ...] = ( + "STRING", + " STRING ", + "", + {"table": _random_map(rng, _NAMES)}, + {"table": {"id": "INTEGER"}, "extra": 1}, + ["STRING"], + ) + candidates: dict[str, object] = { + "arguments": rng.choice((_random_map(rng, _NAMES), ["x"], None)), + "returns": rng.choice((*returns, *returns[:2] * 3)), + "tags": rng.choice((["orders", " sales "], ("orders",), [""], "orders", [1], None)), + "description": rng.choice(_TEXTS), + "database": rng.choice(_SCHEMAS), + "schema": rng.choice(_SCHEMAS), + "runtime_version": rng.choice(("3.12", " 3.12 ", "", None, 3)), + "entry_point": rng.choice(("label", "", None)), + "packages": rng.choice((["numpy"], ("numpy",), [""], "numpy", None)), + } + present: list[str] = rng.sample(sorted(candidates), k=rng.randint(4, len(candidates))) + header: dict[str, object] = {key: candidates[key] for key in present} + required: dict[bool, dict[str, object]] = { + False: {"returns": "STRING"}, + True: {"returns": "STRING", "runtime_version": "3.12", "entry_point": "label"}, + } + return {**required[python], **header} + + +def function_outcome( + *, + header_values: dict[str, object], + python: bool, + test_case: FunctionHeaderParityTestCase, + engine: CompilerEngine, + monkeypatch: pytest.MonkeyPatch, +) -> str: + """Attach one function under `engine`, returning its input or Python's error text.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, engine.value) + sql_file: DiscoveredSqlFunctionFile = DiscoveredSqlFunctionFile( + file_path=Path("/project/functions/sql/order_label.sql"), + relative_path=Path("functions/sql/order_label.sql"), + contents="", + header_values=header_values, + body_sql="UPPER(raw_status)", + ) + python_file: DiscoveredPythonFunctionFile = DiscoveredPythonFunctionFile( + file_path=Path("/project/functions/python/order_label.py"), + relative_path=Path("functions/python/order_label.py"), + contents="", + header_values=header_values, + entry_point="label", + body_python="def label(value):\n return value\n", + ) + sql_files: tuple[DiscoveredSqlFunctionFile, ...] = (sql_file,)[python:] + python_files: tuple[DiscoveredPythonFunctionFile, ...] = (python_file,)[not python :] + adapter: DuckDbAdapter = DuckDbAdapter() + try: + return repr( + build_sql_function_inputs( + discovered_inputs=DiscoveredProjectInputs( + project_config=ProjectConfig( + name="orders", + adapter="duckdb", + defaults=DefaultsConfig(database="analytics", schema="shared"), + ), + local_config=LocalConfig(), + sql_function_files=sql_files, + python_function_files=python_files, + ), + effective_vars={}, + effective_settings=SettingsConfig(), + target_config=TargetConfig(database="warehouse", schema=test_case.target_schema), + macro_context=_MACRO_CONTEXT, + loaded_macros={}, + declaration_expansion=DeclarationExpansionContext( + declarations=DeclarationResolutionContext(), + value_renderer=adapter, + collection_rendering=CollectionRendering.VALUE_LIST, + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + no_sql_validation=True, + python_functions_inherit_default_namespace=test_case.inherit_default_namespace, + ) + ) + except CompileInputError as error: + return f"error: {error}" diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py index e05a158982..5e7d8ca6d5 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_attachment_project.py @@ -27,7 +27,17 @@ } ) _PREVIEW_ONLY_ENTRIES: frozenset[str] = ( - frozenset({"pair_seed_files", "render_attached_generic_audit", "expand_config_templates"}) + frozenset( + { + "pair_seed_files", + "render_attached_generic_audit", + "expand_config_templates", + "scan_test_parameter_references", + "omitted_ceremonial_select", + "extract_sql_scenario_json", + "SqlTestTargetCatalog", + } + ) | _BRIDGED_CONSUMERS ) _ATTACHMENT_ENTRIES: frozenset[str] = frozenset( @@ -37,8 +47,20 @@ "expand_config_templates", "substitute_static_project_vars", "scan_sql_declaration_references", + "scan_test_parameter_references", + "omitted_ceremonial_select", + "extract_sql_scenario_json", + "SqlTestTargetCatalog", } ) +_SCENARIO: str = "tests/scenarios/orders_scenario.sql" +_SCENARIO_HEADER: str = 'SCENARIO (\n description "Orders join their channel"\n);\n\n' +_UNIT_TEST: str = "tests/unit/test_orders.sql" +_HELPER_TEST_MOCKS: str = ( + "TEST();\n\nWITH\n__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label),\n" + "__source__order_events AS (SELECT 1 AS id, 'north' AS region),\n" +) +_HELPER_TEST_EXPECTED: str = "__expected__orders AS (SELECT 1 AS id, 1.5 AS amount, 'web' AS label)" @pytest.mark.parametrize( @@ -46,7 +68,86 @@ [ AttachmentProjectTestCase( description="macros in every attachment kind, seeds, templates and generic audits", + overrides={}, expected_preview_entries=_ATTACHMENT_ENTRIES | _BRIDGED_CONSUMERS, + expected_outcome_fragment="((CompileSeedInput(", + ), + AttachmentProjectTestCase( + description="a scenario check CTE reading a project source", + overrides={ + _SCENARIO: _SCENARIO_HEADER + + "WITH\n__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label),\n" + + '__expected__orders AS (SELECT id, 1.5 AS amount, label FROM __source("order_events"))\n' + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="error: SQL scenario file tests/scenarios/orders_scenario.sql CTE", + ), + AttachmentProjectTestCase( + description="a scenario helper reading an unknown source", + overrides={ + _SCENARIO: _SCENARIO_HEADER + + 'WITH\nhelper AS (SELECT * FROM __source("missing_events")),\n' + + "__seed__channel_codes AS (SELECT 1 AS id, 'web' AS label),\n" + + "__expected__orders AS (SELECT 1 AS id, 1.5 AS amount, 'web' AS label)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="error: SQL scenario file tests/scenarios/orders_scenario.sql " + "references unknown source 'missing_events'", + ), + AttachmentProjectTestCase( + description="a helper CTE reading a model through __ref and read by an assertion", + overrides={ + _UNIT_TEST: _HELPER_TEST_MOCKS + + 'built_orders AS (SELECT id FROM __ref("orders")),\n' + + _HELPER_TEST_EXPECTED + + ",\n__assert__has_rows AS (SELECT 1 FROM built_orders WHERE id IS NULL)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="helper_target_model_names=('orders',)", + ), + AttachmentProjectTestCase( + description="a helper chain where one helper reads another", + overrides={ + _UNIT_TEST: _HELPER_TEST_MOCKS + + 'built_orders AS (SELECT id FROM __ref("orders")),\n' + + "built_ids AS (SELECT id FROM built_orders),\n" + + _HELPER_TEST_EXPECTED + + ",\n__assert__has_ids AS (SELECT 1 FROM built_ids WHERE id IS NULL)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="helper_target_model_names=('orders',)", + ), + AttachmentProjectTestCase( + description="an unread helper referencing an unknown model", + overrides={ + _UNIT_TEST: _HELPER_TEST_MOCKS + + 'unused_returns AS (SELECT id FROM __ref("returns")),\n' + + _HELPER_TEST_EXPECTED + + "\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="helper_target_model_names=()", + ), + AttachmentProjectTestCase( + description="a read helper referencing an unknown model reports P013", + overrides={ + _UNIT_TEST: _HELPER_TEST_MOCKS + + 'built_returns AS (SELECT id FROM __ref("returns")),\n' + + _HELPER_TEST_EXPECTED + + ",\n__assert__no_returns AS (SELECT 1 FROM built_returns)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="P013", + ), + AttachmentProjectTestCase( + description="a model test expecting an unknown model", + overrides={ + _UNIT_TEST: "TEST();\n\nWITH\n__seed__channel_codes AS (SELECT 1 AS id),\n" + "__expected__returns AS (SELECT 1 AS id)\n" + }, + expected_preview_entries=frozenset({"SqlTestTargetCatalog"}), + expected_outcome_fragment="error: SQL test file tests/unit/test_orders.sql expects " + "unknown model 'returns'", ), ], ids=lambda case: case.description, @@ -58,6 +159,7 @@ def test_given_attachment_project_when_building_inputs_then_preview_matches_pyth engine: attachment_engine_outcome( project_dir=tmp_path / engine.value, engine=engine, + overrides=test_case.overrides, monkeypatch=monkeypatch, ) for engine in CompilerEngine @@ -69,9 +171,10 @@ def test_given_attachment_project_when_building_inputs_then_preview_matches_pyth assert ( preview_outcome.replace(CompilerEngine.NATIVE_PREVIEW.value, "python"), native_outcome.replace(CompilerEngine.NATIVE.value, "python"), + test_case.expected_outcome_fragment in python_outcome, preview_called >= test_case.expected_preview_entries, (python_called | native_called) & _PREVIEW_ONLY_ENTRIES, - ) == (python_outcome, python_outcome, True, frozenset()), test_case.description + ) == (python_outcome, python_outcome, True, True, frozenset()), test_case.description if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_function_headers.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_function_headers.py new file mode 100644 index 0000000000..917182f63c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_function_headers.py @@ -0,0 +1,84 @@ +"""SQL and Python function headers attach identically under the preview engine.""" + +from __future__ import annotations + +import random + +import pytest + +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + FunctionHeaderParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + function_outcome, + generated_function_header, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + FunctionHeaderParityTestCase( + description="target schema, Python functions inherit the default namespace", + seed=20261008, + count=1500, + target_schema="prod", + inherit_default_namespace=True, + expected_minimum_attached=150, + expected_minimum_python_errors=800, + ), + FunctionHeaderParityTestCase( + description="no target schema, Python functions keep their own namespace", + seed=20261009, + count=1500, + target_schema=None, + inherit_default_namespace=False, + expected_minimum_attached=150, + expected_minimum_python_errors=800, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_function_headers_when_attaching_with_preview_then_python_matches( + test_case: FunctionHeaderParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + headers: list[tuple[dict[str, object], bool]] = [ + (generated_function_header(rng=rng, python=python), python) + for python in rng.choices((False, True), k=test_case.count) + ] + + python: list[str] = [ + function_outcome( + header_values=header, + python=python_function, + test_case=test_case, + engine=CompilerEngine.PYTHON, + monkeypatch=monkeypatch, + ) + for header, python_function in headers + ] + preview: list[str] = [ + function_outcome( + header_values=header, + python=python_function, + test_case=test_case, + engine=CompilerEngine.NATIVE_PREVIEW, + monkeypatch=monkeypatch, + ) + for header, python_function in headers + ] + + assert ( + mismatches(inputs=[*headers], expected=[*python], actual=[*preview]), + sum(not item.startswith("error: ") for item in python) + >= test_case.expected_minimum_attached, + sum(item.startswith("error: ") for item in python) + >= test_case.expected_minimum_python_errors, + ) == ([], True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_omitted_select.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_omitted_select.py new file mode 100644 index 0000000000..7fa88f1c09 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_omitted_select.py @@ -0,0 +1,82 @@ +"""Test and scenario bodies gain an omitted `SELECT 1` identically under the preview engine.""" + +from __future__ import annotations + +import random + +import pytest + +from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + OmittedSelectParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + completed_body, + generated_test_body, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + OmittedSelectParityTestCase( + description="generic syntax", + seed=20261008, + count=4000, + syntax=SqlLexicalSyntax(), + expected_minimum_completed=1000, + ), + OmittedSelectParityTestCase( + description="backslash escapes, raw and triple-quoted strings, hash comments", + seed=20261009, + count=4000, + syntax=SqlLexicalSyntax( + backslash_escape_quotes=frozenset({"'", '"'}), + raw_string_prefix=True, + triple_quoted_strings=True, + line_comment_prefixes=frozenset({"--", "#"}), + ), + expected_minimum_completed=1000, + ), + OmittedSelectParityTestCase( + description="escape strings and nested block comments", + seed=20261010, + count=4000, + syntax=SqlLexicalSyntax(escape_string_prefix=True, nested_block_comments=True), + expected_minimum_completed=1000, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_bodies_when_completing_select_then_python_output_matches( + test_case: OmittedSelectParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + sqls: list[str] = [generated_test_body(rng=rng) for _ in range(test_case.count)] + + python: list[str] = [ + completed_body( + sql=sql, syntax=test_case.syntax, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch + ) + for sql in sqls + ] + preview: list[str] = [ + completed_body( + sql=sql, + syntax=test_case.syntax, + engine=CompilerEngine.NATIVE_PREVIEW, + monkeypatch=monkeypatch, + ) + for sql in sqls + ] + + assert ( + mismatches(inputs=[*sqls], expected=[*python], actual=[*preview]), + sum(map(str.__ne__, python, sqls)) >= test_case.expected_minimum_completed, + ) == ([], True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_raw_direct_logic_tests.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_raw_direct_logic_tests.py new file mode 100644 index 0000000000..1190e376e1 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_raw_direct_logic_tests.py @@ -0,0 +1,70 @@ +"""Unexpanded direct-logic SQL tests keep Python's extraction under the preview engine. + +The native extractor reads expanded tests and accepts shapes Python rejects before expansion, +such as quoted CTE names or calls in `__macro_expected__`; the raw pass must stay Python's. +""" + +from __future__ import annotations + +import random + +import pytest + +from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + RawDirectLogicParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + generated_raw_direct_logic_test, + python_raw_extraction_accepts, + raw_test_compile_outcome, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + RawDirectLogicParityTestCase( + description="quoted, non-ASCII and $ CTE names, calls in expected CTEs, bad reference calls", + seed=20261008, + count=3000, + expected_minimum_extracted=400, + expected_minimum_python_errors=600, + ), + ], + ids=lambda case: case.description, +) +def test_given_unexpanded_direct_logic_tests_when_compiling_in_preview_then_python_matches( + test_case: RawDirectLogicParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + tests: list[tuple[str, SqlTestMode]] = [ + generated_raw_direct_logic_test(rng=rng) for _ in range(test_case.count) + ] + + python: list[str] = [ + raw_test_compile_outcome( + sql=sql, mode=mode, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch + ) + for sql, mode in tests + ] + native: list[str] = [ + raw_test_compile_outcome( + sql=sql, mode=mode, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch + ) + for sql, mode in tests + ] + + assert ( + mismatches(inputs=[*tests], expected=[*python], actual=[*native]), + sum(python_raw_extraction_accepts(sql=sql, mode=mode) for sql, mode in tests) + >= test_case.expected_minimum_extracted, + sum(item.startswith("error: ") for item in python) + >= test_case.expected_minimum_python_errors, + ) == ([], True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_scenario_extraction.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_scenario_extraction.py new file mode 100644 index 0000000000..823ba4607c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_scenario_extraction.py @@ -0,0 +1,79 @@ +"""Scenario CTEs extract and classify identically under the preview engine.""" + +from __future__ import annotations + +import random + +import pytest + +from sqlbuild.compiler.compile.models import CompileSqlScenarioCtes +from sqlbuild.compiler.frontier.types import CompilerEngine +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + ScenarioParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + generated_scenario, + native_scenario_answered, + scenario_outcome, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + ScenarioParityTestCase( + description="generic syntax", + seed=20261008, + count=3000, + syntax=SqlLexicalSyntax(), + expected_minimum_extracted=200, + expected_minimum_native=140, + expected_minimum_python_errors=2000, + ), + ScenarioParityTestCase( + description="hash comments, which keep the Python scanner", + seed=20261009, + count=1000, + syntax=SqlLexicalSyntax(line_comment_prefixes=frozenset({"--", "#"})), + expected_minimum_extracted=60, + expected_minimum_native=30, + expected_minimum_python_errors=600, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_scenarios_when_extracting_with_preview_then_python_output_matches( + test_case: ScenarioParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + sqls: list[str] = [generated_scenario(rng=rng) for _ in range(test_case.count)] + + python: list[object] = [ + scenario_outcome( + sql=sql, syntax=test_case.syntax, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch + ) + for sql in sqls + ] + preview: list[object] = [ + scenario_outcome( + sql=sql, + syntax=test_case.syntax, + engine=CompilerEngine.NATIVE_PREVIEW, + monkeypatch=monkeypatch, + ) + for sql in sqls + ] + + assert ( + mismatches(inputs=[*sqls], expected=python, actual=preview), + sum(isinstance(item, CompileSqlScenarioCtes) for item in python) + >= test_case.expected_minimum_extracted, + sum(map(native_scenario_answered, sqls)) >= test_case.expected_minimum_native, + sum(isinstance(item, str) for item in python) >= test_case.expected_minimum_python_errors, + ) == ([], True, True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_test_parameters.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_test_parameters.py new file mode 100644 index 0000000000..c28bf3f6c5 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_test_parameters.py @@ -0,0 +1,56 @@ +"""SQL test `@param` references expand identically under the preview engine.""" + +from __future__ import annotations + +import random + +import pytest + +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + ParameterParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + generated_parameter_sql, + parameter_outcome, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + ParameterParityTestCase( + description="seeded references, malformed and undeclared names, quotes, comments", + seed=20261008, + count=3000, + expected_minimum_expanded=800, + expected_minimum_python_errors=800, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_test_bodies_when_expanding_parameters_then_python_output_matches( + test_case: ParameterParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + sqls: list[str] = [generated_parameter_sql(rng=rng) for _ in range(test_case.count)] + + python: list[object] = [ + parameter_outcome(sql=sql, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch) + for sql in sqls + ] + preview: list[object] = [ + parameter_outcome(sql=sql, engine=CompilerEngine.NATIVE_PREVIEW, monkeypatch=monkeypatch) + for sql in sqls + ] + + assert ( + mismatches(inputs=[*sqls], expected=python, actual=preview), + sum(isinstance(item, tuple) for item in python) >= test_case.expected_minimum_expanded, + sum(isinstance(item, str) for item in python) >= test_case.expected_minimum_python_errors, + ) == ([], True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_test_targets.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_test_targets.py new file mode 100644 index 0000000000..a0840a9ac6 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_test_targets.py @@ -0,0 +1,69 @@ +"""Model SQL test targets validate identically under the preview engine.""" + +from __future__ import annotations + +import random + +import pytest + +from sqlbuild.compiler.compile.models import CompileModelSqlTestCtes +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.integration.src.sqlbuild.compiler.attachments._test_types import ( + TargetParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( + generated_test_targets, + target_validation_outcome, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + + +@pytest.mark.parametrize( + "test_case", + [ + TargetParityTestCase( + description="seeded mocks, macro mocks, expected models and assertion targets", + seed=20261008, + count=2000, + expected_minimum_valid=150, + expected_minimum_python_errors=1000, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_test_targets_when_validating_with_preview_then_python_errors_match( + test_case: TargetParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + targets: list[tuple[CompileModelSqlTestCtes, tuple[str, ...]]] = [ + generated_test_targets(rng=rng) for _ in range(test_case.count) + ] + + python: list[str | None] = [ + target_validation_outcome( + payload=payload, + assertion_targets=assertion_targets, + engine=CompilerEngine.PYTHON, + monkeypatch=monkeypatch, + ) + for payload, assertion_targets in targets + ] + preview: list[str | None] = [ + target_validation_outcome( + payload=payload, + assertion_targets=assertion_targets, + engine=CompilerEngine.NATIVE_PREVIEW, + monkeypatch=monkeypatch, + ) + for payload, assertion_targets in targets + ] + + assert ( + mismatches(inputs=[*targets], expected=[*python], actual=[*preview]), + python.count(None) >= test_case.expected_minimum_valid, + len(python) - python.count(None) >= test_case.expected_minimum_python_errors, + ) == ([], True, True), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])