diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/glue.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/glue.rs new file mode 100644 index 0000000000..d776c9896b --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/glue.rs @@ -0,0 +1,31 @@ +//! Projections of planned SQL tests into the facts the compiler reports. + +use std::collections::HashSet; + +use crate::compiler::models::SqlTestPlan; + +const ERROR_SEVERITY: &str = "error"; +const DEFAULT_WORKERS: usize = 4; + +/// Each distinct error-severity warning message of one planned test, in warning order. +pub(crate) fn error_messages(plan: &SqlTestPlan) -> Vec { + let mut seen: HashSet<&str> = HashSet::new(); + plan.warnings + .iter() + .filter(|warning| warning.severity == ERROR_SEVERITY) + .filter(|warning| seen.insert(warning.message.as_str())) + .map(|warning| warning.message.clone()) + .collect() +} + +pub(crate) fn default_workers() -> usize { + DEFAULT_WORKERS +} + +pub(crate) fn default_true() -> bool { + true +} + +pub(crate) fn default_set_difference() -> String { + "EXCEPT".to_string() +} diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs index de20c4f5ee..364bfe13c4 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs @@ -4,6 +4,7 @@ pub mod cte_slices; pub mod cte_sql; pub mod expected_columns; pub mod extraction; +pub mod glue; pub mod helper_names; pub mod helper_scope; pub mod markers; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index 3bf4a939ff..e879c9da90 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -7,7 +7,7 @@ use std::time::Instant; use polyglot_sql::{Dialect, Expression}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use regex::Regex; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::{SliceDialect, strip_statement_terminators}; @@ -29,8 +29,12 @@ use crate::compiler::_helpers::sql_tests::reader_references::{ }; use crate::compiler::_helpers::sql_tests::relation_markers::relation_marker_calls; use crate::compiler::_helpers::sql_tests::rendering::{ - AssertionStep, ChainStep, RenderRequest, render_comparison_sql, render_dialect, - rendered_chain_steps, + RenderRequest, render_comparison_sql, render_dialect, rendered_chain_steps, +}; +use crate::compiler::models::{ + SqlTestAssertionStep, SqlTestChainBatch, SqlTestChainStep, SqlTestCte, SqlTestPlan, + SqlTestPlanBatch, SqlTestPlanBatchOutcome, SqlTestPlanFunction, SqlTestPlanPayload, + SqlTestPlanTest, SqlTestPlanWarning, }; use crate::constants::{ SQL_TEST_ACTUAL_CTE, SQL_TEST_ACTUAL_CTE_PREFIX, SQL_TEST_EXPECTED_CTE, @@ -38,7 +42,6 @@ use crate::constants::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -const DEFAULT_WORKERS: usize = 4; const MAX_WORKERS: usize = 4; const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; pub(crate) const REF_PREFIX: &str = "__ref__"; @@ -62,160 +65,22 @@ pub(crate) const DBT_REF_FUNCTION: &str = "__dbt_ref"; const UDF_FUNCTION: &str = "__udf"; const TABLE_FUNCTION: &str = "__table_fn"; -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PlanBatchRequest { - models: Vec, - #[serde(default)] - functions: Vec, - tests: Vec, - #[serde(default = "default_true")] - sql_analysis_enabled: bool, - #[serde(default)] - sql_analysis_dialect: Option, - #[serde(default = "default_set_difference")] - set_difference_operator: String, - #[serde(default)] - requires_derived_table_aliases: bool, - #[serde(default = "default_workers")] - workers: usize, - #[serde(default = "default_true")] - render_sql: bool, - #[serde(default = "default_true")] - include_plan: bool, - lexical_syntax: LexicalSyntax, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ChainBatchRequest { - models: Vec, - tests: Vec, - lexical_syntax: LexicalSyntax, -} - -/// Chain ordering reads only declared dependencies, so chain requests omit model SQL. -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ChainModelInput { - name: String, - #[serde(default)] - model_dependencies: Vec, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct ChainBatchResponse { chains: Vec>, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ModelInput { - name: String, - query_sql: String, - #[serde(default)] - model_dependencies: Vec, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct FunctionInput { - name: String, - #[serde(default)] - udf_prefix: Option, - #[serde(default)] - udf_suffix: Option, - #[serde(default)] - table_function_prefix: Option, - #[serde(default)] - table_function_suffix: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct TestInput { - name: String, - file_label: String, - payload: TestPayload, -} - -#[derive(Debug, Deserialize)] -#[serde( - tag = "kind", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -enum TestPayload { - Model { - #[serde(default)] - authored_ctes: Vec, - #[serde(default)] - model_query_overrides: BTreeMap, - #[serde(default)] - expected_ctes: Vec, - #[serde(default)] - expected_model_names: Vec, - #[serde(default)] - assertion_ctes: Vec, - #[serde(default)] - read_helper_names: Option>, - #[serde(default)] - reference_target_model_names: Option>, - }, - Direct { - mode: String, - actual_cte: CteInput, - expected_cte: CteInput, - #[serde(default)] - helper_ctes: Vec, - }, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct CteInput { - pub(crate) name: String, - pub(crate) sql_body: String, -} - -/// One planned SQL test: the executable chain and assertion steps plus optional rendered SQL. -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct PlanResponse { - sql: Option, - chain: Vec, - assertions: Vec, - model_names: Vec, - warnings: Vec, -} - struct PlannedResponse { request: RenderRequest, model_names: Vec, - warnings: Vec, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct PlanBatchResponse { - artifacts: Vec, - planning_ns: u64, - rendering_ns: u64, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct PlanWarning { - model_name: Option, - severity: &'static str, - message: String, + warnings: Vec, } #[derive(Clone)] struct ProjectContext { models: HashMap, - functions: HashMap, + functions: HashMap, sql_analysis_enabled: bool, dialect: String, set_difference_operator: String, @@ -286,7 +151,7 @@ pub(crate) struct TestFixtures { pub(crate) mock_seeds: BTreeMap, pub(crate) mock_dbt_refs: BTreeMap, pub(crate) mock_table_functions: BTreeMap, - pub(crate) helpers: Vec, + pub(crate) helpers: Vec, pub(crate) scope: ScopeGraph, pub(crate) expected: BTreeMap, pub(crate) assertions: Vec<(String, String)>, @@ -295,19 +160,19 @@ pub(crate) struct TestFixtures { struct DirectTestPlan { name: String, mode: String, - actual_cte: CteInput, - expected_cte: CteInput, - helpers: Vec, + actual_cte: SqlTestCte, + expected_cte: SqlTestCte, + helpers: Vec, } struct ModelTestPlan { test_name: String, file_label: String, - authored_ctes: Vec, + authored_ctes: Vec, model_query_overrides: BTreeMap, - expected_ctes: Vec, + expected_ctes: Vec, expected_model_names: Vec, - assertion_ctes: Vec, + assertion_ctes: Vec, reads: CompilerReads, } @@ -441,7 +306,7 @@ struct AssertionResolutionRequest<'a> { assertion_sql: &'a str, fixtures: &'a TestFixtures, chain: &'a TextualChain, - functions: &'a HashMap, + functions: &'a HashMap, requires_flat_ctes: bool, patterns: &'a SqlTestPatterns, } @@ -451,7 +316,7 @@ struct AnalysisResolutionRequest<'a> { fixture_ctes: &'a [(String, String)], fixtures: &'a TestFixtures, resolved_chain: &'a HashMap, - functions: &'a HashMap, + functions: &'a HashMap, file_label: &'a str, dialect_name: &'a str, templates: &'a AnalysisTemplateCache, @@ -463,7 +328,7 @@ struct TextualResolutionRequest<'a> { fixtures: &'a TestFixtures, resolved_chain: &'a HashMap, chain_references: Option<&'a mut Vec>, - functions: &'a HashMap, + functions: &'a HashMap, patterns: &'a SqlTestPatterns, } @@ -562,8 +427,14 @@ impl GeneratedCteState { /// Return each test's topologically ordered unmocked model chain without planning SQL. pub(crate) fn resolve_chains_json(request_json: &str) -> Result { - let request: ChainBatchRequest = + let request: SqlTestChainBatch = serde_json::from_str(request_json).map_err(|error| error.to_string())?; + let chains: Vec> = resolve_chains(request)?; + serde_json::to_string(&ChainBatchResponse { chains }).map_err(|error| error.to_string()) +} + +/// Each test's topologically ordered unmocked model chain, in request order. +pub(crate) fn resolve_chains(request: SqlTestChainBatch) -> Result>, String> { let patterns = SqlTestPatterns::new(request.lexical_syntax)?; let models: HashMap = request .models @@ -582,8 +453,8 @@ pub(crate) fn resolve_chains_json(request_json: &str) -> Result .tests .into_iter() .map(|test| match test.payload { - TestPayload::Direct { .. } => Ok(Vec::new()), - TestPayload::Model { + SqlTestPlanPayload::Direct { .. } => Ok(Vec::new()), + SqlTestPlanPayload::Model { authored_ctes, model_query_overrides, expected_ctes, @@ -610,7 +481,7 @@ pub(crate) fn resolve_chains_json(request_json: &str) -> Result } }) .collect::>()?; - serde_json::to_string(&ChainBatchResponse { chains }).map_err(|error| error.to_string()) + Ok(chains) } /// Models the test runs: expected models, assertion targets and the compiler's reference targets. @@ -626,8 +497,13 @@ fn chain_root_names( } pub(crate) fn plan_and_render_json(request_json: &str) -> Result { - let request: PlanBatchRequest = + let request: SqlTestPlanBatch = serde_json::from_str(request_json).map_err(|error| error.to_string())?; + serde_json::to_string(&plan_batch(request)?).map_err(|error| error.to_string()) +} + +/// Plan every test of `request` on a bounded pool, keeping request order. +pub(crate) fn plan_batch(request: SqlTestPlanBatch) -> Result { let render_sql = request.render_sql; let include_plan = request.include_plan; let render_dialect = Arc::new(render_dialect(request.sql_analysis_dialect.as_deref())); @@ -670,7 +546,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result .build() .map_err(|error| error.to_string())?; let batch_start = Instant::now(); - let responses: Vec> = pool.install(|| { + let responses: Vec> = pool.install(|| { request .tests .into_par_iter() @@ -685,7 +561,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result let sql = match rendered { Some(Ok(sql)) => render_sql.then_some(sql), Some(Err(message)) => { - planned.warnings.push(PlanWarning { + planned.warnings.push(SqlTestPlanWarning { model_name: None, severity: "error", message: format!("test '{test_name}' cannot be rendered: {message}"), @@ -698,7 +574,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result planned.request.chain.clear(); planned.request.assertions.clear(); } - let response = PlanResponse { + let response = SqlTestPlan { sql, chain: planned.request.chain, assertions: planned.request.assertions, @@ -709,7 +585,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result }) .collect() }); - let responses: Vec<(PlanResponse, u128, u128)> = + let responses: Vec<(SqlTestPlan, u128, u128)> = responses.into_iter().collect::>()?; let batch_ns = batch_start.elapsed().as_nanos(); let planning_cpu_ns: u128 = responses.iter().map(|(_, value, _)| *value).sum(); @@ -720,21 +596,20 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result .checked_div(measured_cpu_ns) .unwrap_or(batch_ns); let rendering_ns = batch_ns.saturating_sub(planning_ns); - let artifacts: Vec = responses + let artifacts: Vec = responses .into_iter() .map(|(response, _, _)| response) .collect(); - serde_json::to_string(&PlanBatchResponse { - artifacts, + Ok(SqlTestPlanBatchOutcome { + plans: artifacts, planning_ns: planning_ns.min(u128::from(u64::MAX)) as u64, rendering_ns: rendering_ns.min(u128::from(u64::MAX)) as u64, }) - .map_err(|error| error.to_string()) } -fn plan_test(test: TestInput, context: &ProjectContext) -> Result { +fn plan_test(test: SqlTestPlanTest, context: &ProjectContext) -> Result { match test.payload { - TestPayload::Direct { + SqlTestPlanPayload::Direct { mode, actual_cte, expected_cte, @@ -749,7 +624,7 @@ fn plan_test(test: TestInput, context: &ProjectContext) -> Result = Vec::new(); + let mut warnings: Vec = Vec::new(); let mut reported_missing_mocks: HashSet<(&'static str, String)> = HashSet::new(); let mut reachable_mocks: HashSet = HashSet::new(); let mut analysis_resolved: HashMap = HashMap::new(); let mut textual_chain: TextualChain = TextualChain::default(); - let mut chain: Vec = Vec::new(); + let mut chain: Vec = Vec::new(); for (model_index, model_name) in ordered_names.iter().enumerate() { let Some(model) = context.models.get(model_name) else { - warnings.push(PlanWarning { + warnings.push(SqlTestPlanWarning { model_name: None, severity: "error", message: format!( @@ -953,7 +828,7 @@ fn plan_model_test( } None => Vec::new(), }; - chain.push(ChainStep { + chain.push(SqlTestChainStep { model_name: model_name.clone(), resolved_sql, expected_columns: authored_expected @@ -983,7 +858,7 @@ fn plan_model_test( reachable_mocks: &mut reachable_mocks, })?; - let mut assertions: Vec = Vec::new(); + let mut assertions: Vec = Vec::new(); for (assertion_name, assertion_sql) in &fixtures.assertions { let placed_sql = fixtures.scope.reader_sql(assertion_sql, &context.patterns); let resolved = readers.resolve( @@ -1013,7 +888,7 @@ fn plan_model_test( comparison_body_sql.as_deref().unwrap_or(&resolved_sql), ); } - assertions.push(AssertionStep { + assertions.push(SqlTestAssertionStep { name: assertion_name.clone(), resolved_sql, lifted_ctes, @@ -1047,7 +922,7 @@ struct HelperResolution<'r, 'a> { readers: &'r mut ReaderContext<'a>, fixtures: &'r mut TestFixtures, read_helper_names: &'r [String], - chain: &'r mut [ChainStep], + chain: &'r mut [SqlTestChainStep], reachable_mocks: &'r mut HashSet, } @@ -1237,7 +1112,10 @@ impl ReaderContext<'_> { } /// Drop SQL from chain steps the renderer never emits, keeping plan output linear in chain length. -fn omit_unrendered_step_sql(chain: Vec, assertions: &[AssertionStep]) -> Vec { +fn omit_unrendered_step_sql( + chain: Vec, + assertions: &[SqlTestAssertionStep], +) -> Vec { let rendered_steps = rendered_chain_steps(&chain, assertions); chain .into_iter() @@ -1246,7 +1124,7 @@ fn omit_unrendered_step_sql(chain: Vec, assertions: &[AssertionStep]) if rendered { step } else { - ChainStep { + SqlTestChainStep { model_name: step.model_name, resolved_sql: String::new(), expected_cte_sql: step.expected_cte_sql, @@ -1350,9 +1228,9 @@ fn resolve_assertion_textual_sql( } fn classify_fixtures( - authored: Vec, - expected: Vec, - assertions: Vec, + authored: Vec, + expected: Vec, + assertions: Vec, ) -> TestFixtures { let mut fixtures = TestFixtures { mock_refs: BTreeMap::new(), @@ -1760,7 +1638,7 @@ fn resolve_table_function_fixtures( fn resolve_function_calls( sql: &str, - functions: &HashMap, + functions: &HashMap, table_function: bool, patterns: &SqlTestPatterns, ) -> Result { @@ -1836,7 +1714,9 @@ struct UnresolvedReferenceRequest<'a> { } /// Report each unmocked reference once per test, attributed to the first step reaching it. -fn unresolved_reference_warnings(request: UnresolvedReferenceRequest<'_>) -> Vec { +fn unresolved_reference_warnings( + request: UnresolvedReferenceRequest<'_>, +) -> Vec { let UnresolvedReferenceRequest { sql, test_name, @@ -1847,10 +1727,10 @@ fn unresolved_reference_warnings(request: UnresolvedReferenceRequest<'_>) -> Vec if !patterns.test_reference.is_match(sql) { return Vec::new(); } - let mut warnings: Vec = Vec::new(); + let mut warnings: Vec = Vec::new(); let mut warn = |kind: &'static str, name: String, message: String| { if reported.insert((kind, name)) { - warnings.push(PlanWarning { + warnings.push(SqlTestPlanWarning { model_name: Some(model_name.to_string()), severity: "error", message, @@ -1901,7 +1781,7 @@ fn unreachable_mock_warnings( test_name: &str, reachable: &HashSet, fixtures: &TestFixtures, -) -> Vec { +) -> Vec { let groups = [ ( &fixtures.mock_refs, @@ -1917,10 +1797,10 @@ fn unreachable_mock_warnings( " is unreachable", ), ]; - let mut warnings: Vec = Vec::new(); + let mut warnings: Vec = Vec::new(); for (mocks, prefix, suffix) in groups { for name in mocks.keys().filter(|name| !reachable.contains(*name)) { - warnings.push(PlanWarning { + warnings.push(SqlTestPlanWarning { model_name: None, severity: "warning", message: format!("test '{test_name}' mock {prefix}{name}{suffix}"), @@ -1941,7 +1821,7 @@ fn has_unresolved_test_reference(sql: &str, patterns: &SqlTestPatterns) -> bool || patterns.table_function.is_match(sql) } -fn helper_with_clause(helpers: &[CteInput]) -> String { +fn helper_with_clause(helpers: &[SqlTestCte]) -> String { if helpers.is_empty() { return String::new(); } @@ -1956,7 +1836,7 @@ fn helper_with_clause(helpers: &[CteInput]) -> String { } /// Prefix every helper CTE to a direct test's actual and expected queries. -fn with_helper_ctes(sql: &str, helpers: &[CteInput]) -> String { +fn with_helper_ctes(sql: &str, helpers: &[SqlTestCte]) -> String { if helpers.is_empty() { sql.to_string() } else { @@ -1975,18 +1855,6 @@ fn compile_pattern(pattern: &str) -> Result { .map_err(|error| planner_error(&format!("invalid SQL-test pattern: {error}"))) } -fn default_workers() -> usize { - DEFAULT_WORKERS -} - -fn default_true() -> bool { - true -} - -fn default_set_difference() -> String { - "EXCEPT".to_string() -} - pub(crate) fn compile_error(message: &str) -> String { format!("compile_input:{message}") } diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs index 72a24956ab..efc1b7d5f2 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs @@ -13,6 +13,7 @@ use crate::compiler::_helpers::sql_tests::cte_slices::{ use crate::compiler::_helpers::sql_tests::cte_sql::{cte_definition_sql, leading_with_prefix_end}; use crate::compiler::_helpers::sql_tests::helper_names::HELPER_PREFIX; use crate::compiler::_helpers::sql_tests::planning::REF_PREFIX; +use crate::compiler::models::{SqlTestAssertionStep, SqlTestChainStep}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; @@ -32,7 +33,7 @@ struct RenderBatchRequest { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct DifferenceSampleRequest { - step: ChainStep, + step: SqlTestChainStep, #[serde(default = "default_true")] sql_analysis_enabled: bool, #[serde(default = "default_set_difference")] @@ -56,9 +57,9 @@ enum DifferenceDirection { #[serde(rename_all = "camelCase")] pub(crate) struct RenderRequest { #[serde(default)] - pub(crate) chain: Vec, + pub(crate) chain: Vec, #[serde(default)] - pub(crate) assertions: Vec, + pub(crate) assertions: Vec, #[serde(default = "default_true")] pub(crate) sql_analysis_enabled: bool, #[serde(default = "default_set_difference")] @@ -69,34 +70,6 @@ pub(crate) struct RenderRequest { pub(crate) probe_step_index: Option, } -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ChainStep { - pub(crate) model_name: String, - pub(crate) resolved_sql: String, - #[serde(default)] - pub(crate) expected_cte_sql: Option, - #[serde(default)] - pub(crate) expected_lifted_ctes: Vec<(String, String)>, - #[serde(default)] - pub(crate) lifted_ctes: Vec<(String, String)>, - #[serde(default)] - pub(crate) comparison_body_sql: Option, - #[serde(default)] - pub(crate) expected_columns: Option>, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AssertionStep { - pub(crate) name: String, - pub(crate) resolved_sql: String, - #[serde(default)] - pub(crate) lifted_ctes: Vec<(String, String)>, - #[serde(default)] - pub(crate) comparison_body_sql: Option, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct RenderResponse { @@ -202,7 +175,7 @@ impl RenderCteState { /// Define a step's inputs and its model once at top level, the model as a nested `__ref__` CTE. fn actual_step_sql( &mut self, - step: &ChainStep, + step: &SqlTestChainStep, suffix: &str, enabled: bool, ) -> Result { @@ -225,7 +198,7 @@ impl RenderCteState { /// Place an expected step's helper CTEs at top level and return its comparison body. fn expected_step_sql( &mut self, - step: &ChainStep, + step: &SqlTestChainStep, expected_sql: &str, enabled: bool, ) -> Result { @@ -237,7 +210,7 @@ impl RenderCteState { /// Place an assertion's helper CTEs at top level and return its comparison body. fn assertion_sql( &mut self, - assertion: &AssertionStep, + assertion: &SqlTestAssertionStep, enabled: bool, ) -> Result { let origin = format!("assertion '{}'", assertion.name); @@ -575,7 +548,10 @@ impl CteSuffixCounter { } /// Mark the chain steps whose actual SQL the comparison renderer emits. -pub(crate) fn rendered_chain_steps(chain: &[ChainStep], assertions: &[AssertionStep]) -> Vec { +pub(crate) fn rendered_chain_steps( + chain: &[SqlTestChainStep], + assertions: &[SqlTestAssertionStep], +) -> Vec { let mut suffixes = CteSuffixCounter::default(); chain .iter() @@ -666,7 +642,7 @@ fn render_difference_sample_sql(request: &DifferenceSampleRequest) -> Result String { +fn compared_projection(step: &SqlTestChainStep) -> String { step.expected_columns .as_ref() .map_or_else(|| "*".to_string(), |columns| columns.join(", ")) @@ -789,7 +765,7 @@ fn sanitize_cte_suffix(model_name: &str) -> String { } } -fn assertions_use_actual(assertions: &[AssertionStep], actual_cte: &str) -> bool { +fn assertions_use_actual(assertions: &[SqlTestAssertionStep], actual_cte: &str) -> bool { let needle = actual_cte.to_ascii_lowercase(); assertions.iter().any(|assertion| { assertion diff --git a/crates/sqlbuild-analysis/src/compiler/main/mod.rs b/crates/sqlbuild-analysis/src/compiler/main/mod.rs index 0267d8f76c..357fadc9d0 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/mod.rs @@ -1,7 +1,10 @@ pub mod sql_scenario_extraction; pub mod sql_test_chain_resolution; +pub mod sql_test_chains; pub mod sql_test_difference_sampling; pub mod sql_test_extraction; pub mod sql_test_fixture_facts; +pub mod sql_test_glue; +pub mod sql_test_plan_errors; pub mod sql_test_planning; pub mod sql_test_rendering; diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_test_chains.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_chains.rs new file mode 100644 index 0000000000..c54b8b3e32 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_chains.rs @@ -0,0 +1,9 @@ +//! SQL-test model chains for compiled projects, from a typed request. + +use crate::compiler::_helpers::sql_tests::planning; +use crate::compiler::models::SqlTestChainBatch; + +/// Each test's ordered unmocked model chain; direct-logic tests have none. +pub fn resolve_sql_test_chains(request: SqlTestChainBatch) -> Result>, String> { + planning::resolve_chains(request) +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_test_glue.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_glue.rs new file mode 100644 index 0000000000..972ab2a837 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_glue.rs @@ -0,0 +1,9 @@ +//! SQL-test planning for compiled projects, from a typed request. + +use crate::compiler::_helpers::sql_tests::planning; +use crate::compiler::models::{SqlTestPlanBatch, SqlTestPlanBatchOutcome}; + +/// Plan every test of `request` in request order; errors carry the planner's message prefixes. +pub fn plan_sql_test_batch(request: SqlTestPlanBatch) -> Result { + planning::plan_batch(request) +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_test_plan_errors.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_plan_errors.rs new file mode 100644 index 0000000000..115be96298 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_plan_errors.rs @@ -0,0 +1,10 @@ +//! The errors a planned SQL test reports. + +use crate::compiler::_helpers::sql_tests::glue; +use crate::compiler::models::SqlTestPlan; + +/// Each distinct error message of one planned test, in the order its warnings report them. +#[must_use] +pub fn sql_test_plan_error_messages(plan: &SqlTestPlan) -> Vec { + glue::error_messages(plan) +} diff --git a/crates/sqlbuild-analysis/src/compiler/models.rs b/crates/sqlbuild-analysis/src/compiler/models.rs index bef142348d..03e40d082d 100644 --- a/crates/sqlbuild-analysis/src/compiler/models.rs +++ b/crates/sqlbuild-analysis/src/compiler/models.rs @@ -1,5 +1,184 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SqlTestFixtureFacts { pub mock: bool, pub empty_fixture_marker: bool, } + +/// One batch of SQL tests to plan against the project's models and functions. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanBatch { + pub models: Vec, + #[serde(default)] + pub functions: Vec, + pub tests: Vec, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_true")] + pub sql_analysis_enabled: bool, + #[serde(default)] + pub sql_analysis_dialect: Option, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_set_difference")] + pub set_difference_operator: String, + #[serde(default)] + pub requires_derived_table_aliases: bool, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_workers")] + pub workers: usize, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_true")] + pub render_sql: bool, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_true")] + pub include_plan: bool, + pub lexical_syntax: LexicalSyntax, +} + +/// The tests whose model chains to resolve; chain ordering reads only declared dependencies. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestChainBatch { + pub models: Vec, + pub tests: Vec, + pub lexical_syntax: LexicalSyntax, +} + +/// One project model's name and the models it depends on. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestChainModel { + pub name: String, + #[serde(default)] + pub model_dependencies: Vec, +} + +/// One project model as SQL tests read it: its SQL and the models it depends on. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanModel { + pub name: String, + pub query_sql: String, + #[serde(default)] + pub model_dependencies: Vec, +} + +/// One project function and the adapter's call templates around its arguments. +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanFunction { + pub name: String, + #[serde(default)] + pub udf_prefix: Option, + #[serde(default)] + pub udf_suffix: Option, + #[serde(default)] + pub table_function_prefix: Option, + #[serde(default)] + pub table_function_suffix: Option, +} + +/// One compiled SQL test to plan. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanTest { + pub name: String, + pub file_label: String, + pub payload: SqlTestPlanPayload, +} + +/// A model-mode test's CTEs and overrides, or a direct-logic test's actual and expected CTEs. +#[derive(Debug, Deserialize)] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum SqlTestPlanPayload { + Model { + #[serde(default)] + authored_ctes: Vec, + #[serde(default)] + model_query_overrides: BTreeMap, + #[serde(default)] + expected_ctes: Vec, + #[serde(default)] + expected_model_names: Vec, + #[serde(default)] + assertion_ctes: Vec, + #[serde(default)] + read_helper_names: Option>, + #[serde(default)] + reference_target_model_names: Option>, + }, + Direct { + mode: String, + actual_cte: SqlTestCte, + expected_cte: SqlTestCte, + #[serde(default)] + helper_ctes: Vec, + }, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestCte { + pub name: String, + pub sql_body: String, +} + +/// One planned SQL test: the executable chain and assertion steps plus optional rendered SQL. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlan { + pub sql: Option, + pub chain: Vec, + pub assertions: Vec, + pub model_names: Vec, + pub warnings: Vec, +} + +/// Every planned test in request order, with the batch's wall time split by phase. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanBatchOutcome { + #[serde(rename = "artifacts")] + pub plans: Vec, + pub planning_ns: u64, + pub rendering_ns: u64, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanWarning { + pub model_name: Option, + pub severity: &'static str, + pub message: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestChainStep { + pub model_name: String, + pub resolved_sql: String, + #[serde(default)] + pub expected_cte_sql: Option, + #[serde(default)] + pub expected_lifted_ctes: Vec<(String, String)>, + #[serde(default)] + pub lifted_ctes: Vec<(String, String)>, + #[serde(default)] + pub comparison_body_sql: Option, + #[serde(default)] + pub expected_columns: Option>, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestAssertionStep { + pub name: String, + pub resolved_sql: String, + #[serde(default)] + pub lifted_ctes: Vec<(String, String)>, + #[serde(default)] + pub comparison_body_sql: Option, +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs index eb6f3a9359..a54b22b9de 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs @@ -1,3 +1,11 @@ +use serde_json::{Value, json}; + +use crate::compiler::main::sql_test_chain_resolution::resolve_chains_json; +use crate::compiler::main::sql_test_chains::resolve_sql_test_chains; +use crate::compiler::main::sql_test_glue::plan_sql_test_batch; +use crate::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; +use crate::compiler::main::sql_test_planning::plan_and_render_json; +use crate::compiler::models::{SqlTestPlan, SqlTestPlanWarning}; use crate::compiler::tests::helpers::{ chain_resolution_orders_unmocked_models, concurrent_requests_initialize_shared_template_once, deep_shared_graph_reports_missing_mock_once, @@ -9,7 +17,69 @@ use crate::compiler::tests::helpers::{ unicode_cte_after_leading_with_preserves_identifier, unresolved_reference_fast_rejection_preserves_warning, upstream_fallback_resolves, }; -use crate::compiler::tests::test_types::SqlTestPlanningTestCase; +use crate::compiler::tests::test_types::{ + GlueErrorProjectionTestCase, GluePlanParityTestCase, GluePlannerErrorTestCase, + SqlTestPlanningTestCase, +}; + +/// A mocked model test, a test missing its source mock and a UDF test over two models. +const GLUE_REQUEST: &str = r#"{ + "lexicalSyntax": { + "backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, + "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"] + }, + "models": [ + {"name": "stg_orders", "querySql": "SELECT order_id, amount FROM __source(\"raw_orders\")"}, + { + "name": "orders", + "querySql": "WITH big AS (SELECT * FROM __ref(\"stg_orders\") WHERE __udf(\"udf__is_big\")(amount)) SELECT order_id FROM big", + "modelDependencies": ["stg_orders"] + } + ], + "functions": [{ + "name": "udf__is_big", "udfPrefix": "main.udf__is_big(", "udfSuffix": ")", + "tableFunctionPrefix": "main.udf__is_big(", "tableFunctionSuffix": ")" + }], + "tests": [ + { + "name": "mocked", "fileLabel": "tests/mocked.sql", + "payload": { + "kind": "model", + "authoredCtes": [ + {"name": "__source__raw_orders", "sqlBody": "SELECT 1 AS order_id, 5 AS amount"}, + {"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"} + ], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], + "expectedModelNames": ["orders"], + "assertionCtes": [{ + "name": "__assert__orders_exist", + "sqlBody": "SELECT * FROM __ref(\"orders\") WHERE 1 = 0" + }], + "readHelperNames": [], "referenceTargetModelNames": [] + } + }, + { + "name": "missing_mock", "fileLabel": "tests/missing_mock.sql", + "payload": { + "kind": "model", + "authoredCtes": [{"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"}], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], + "expectedModelNames": ["orders"], + "readHelperNames": [], "referenceTargetModelNames": [] + } + }, + { + "name": "udf_case", "fileLabel": "tests/udf_case.sql", + "payload": { + "kind": "direct", "mode": "udf", + "actualCte": { + "name": "__udf_actual__", "sqlBody": "SELECT __udf(\"udf__is_big\")(150) AS is_big" + }, + "expectedCte": {"name": "__udf_expected__", "sqlBody": "SELECT TRUE AS is_big"} + } + } + ] +}"#; #[test] fn given_sql_test_upstream_fallback_when_planning_then_descendants_and_assertions_resolve() { @@ -107,3 +177,150 @@ fn given_sql_test_planning_cases_when_exercising_native_planner_then_expected_be ); } } + +#[test] +fn given_typed_requests_when_planning_then_plans_and_chains_equal_the_json_entry_points() { + let test_cases = [ + GluePlanParityTestCase { + description: "generic SQL, planned without rendering", + dialect: None, + render_sql: false, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "DuckDB, rendered", + dialect: Some("duckdb"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "Snowflake, rendered", + dialect: Some("snowflake"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "SQL Server rejects nested WITH and renders anyway", + dialect: Some("tsql"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + ]; + let without_timings = |mut value: Value| { + for key in ["planningNs", "renderingNs"] { + let _ = value.as_object_mut().map(|object| object.remove(key)); + } + value + }; + for test_case in test_cases { + let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); + request["sqlAnalysisDialect"] = json!(test_case.dialect); + request["renderSql"] = json!(test_case.render_sql); + request["includePlan"] = json!(!test_case.render_sql); + let typed = plan_sql_test_batch(serde_json::from_value(request.clone()).expect("typed")) + .expect("typed plans"); + let typed: Value = without_timings(serde_json::to_value(&typed).expect("serializable")); + let json: Value = without_timings( + serde_json::from_str(&plan_and_render_json(&request.to_string()).expect("plans")) + .expect("JSON plans"), + ); + let typed_chains = resolve_sql_test_chains( + serde_json::from_value(request.clone()).expect("typed chains request"), + ) + .expect("typed chains"); + let json_chains: Value = + serde_json::from_str(&resolve_chains_json(&request.to_string()).expect("chains")) + .expect("JSON chains"); + assert_eq!( + ( + typed["artifacts"].as_array().map(Vec::len), + typed["artifacts"][1]["warnings"][0]["severity"].clone(), + typed == json, + json!({"chains": typed_chains}) == json_chains, + ), + ( + Some(test_case.expected_plan_count), + json!(test_case.expected_missing_mock_severity), + true, + true, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_test_without_compiler_reads_when_planning_then_the_typed_batch_returns_its_error() { + let test_cases = [ + GluePlannerErrorTestCase { + description: "helper reads are required for model tests", + payload_field: "readHelperNames", + expected_error_prefix: "planner_input:", + }, + GluePlannerErrorTestCase { + description: "reference targets are required for model tests", + payload_field: "referenceTargetModelNames", + expected_error_prefix: "planner_input:", + }, + ]; + for test_case in test_cases { + let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); + request["tests"][0]["payload"][test_case.payload_field] = json!(null); + let error = plan_sql_test_batch(serde_json::from_value(request).expect("typed")).err(); + assert_eq!( + error.map(|message| message.starts_with(test_case.expected_error_prefix)), + Some(true), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_plan_warnings_when_projecting_errors_then_each_distinct_error_message_is_kept_in_order() { + let test_cases = [ + GlueErrorProjectionTestCase { + description: "warnings are not errors", + warnings: &[("warning", "mock is unreachable"), ("info", "note")], + expected_messages: &[], + }, + GlueErrorProjectionTestCase { + description: "repeated errors are reported once, in first-seen order", + warnings: &[ + ("error", "b has no mock"), + ("warning", "a is unreachable"), + ("error", "a has no mock"), + ("error", "b has no mock"), + ], + expected_messages: &["b has no mock", "a has no mock"], + }, + ]; + for test_case in test_cases { + let plan = SqlTestPlan { + sql: None, + chain: Vec::new(), + assertions: Vec::new(), + model_names: Vec::new(), + warnings: test_case + .warnings + .iter() + .map(|(severity, message)| SqlTestPlanWarning { + model_name: None, + severity, + message: (*message).to_owned(), + }) + .collect(), + }; + assert_eq!( + sql_test_plan_error_messages(&plan), + test_case.expected_messages.to_vec(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 2f53e15443..c109a747fb 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -166,3 +166,23 @@ pub(crate) struct AuthoredCtesTestCase { pub(crate) sql: &'static str, pub(crate) expected_response: serde_json::Value, } + +pub(crate) struct GluePlanParityTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: Option<&'static str>, + pub(crate) render_sql: bool, + pub(crate) expected_plan_count: usize, + pub(crate) expected_missing_mock_severity: &'static str, +} + +pub(crate) struct GluePlannerErrorTestCase { + pub(crate) description: &'static str, + pub(crate) payload_field: &'static str, + pub(crate) expected_error_prefix: &'static str, +} + +pub(crate) struct GlueErrorProjectionTestCase { + pub(crate) description: &'static str, + pub(crate) warnings: &'static [(&'static str, &'static str)], + pub(crate) expected_messages: &'static [&'static str], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 7444bd0374..8be67c2871 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1 +1,2 @@ +pub(crate) mod planning; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs new file mode 100644 index 0000000000..43b10157f0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -0,0 +1,304 @@ +//! Plan compiled SQL tests from the compiled project's own objects, without a JSON request. + +use std::collections::{BTreeMap, HashMap}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiler::main::sql_test_chains::resolve_sql_test_chains; +use sqlbuild_analysis::compiler::main::sql_test_glue::plan_sql_test_batch; +use sqlbuild_analysis::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; +use sqlbuild_analysis::compiler::models::{ + SqlTestAssertionStep, SqlTestChainBatch, SqlTestChainModel, SqlTestChainStep, SqlTestCte, + SqlTestPlan, SqlTestPlanBatch, SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, + SqlTestPlanTest, +}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::bindings::types::CompilerDetach; + +const MODEL_RESOURCE_TYPE: &str = "model"; +const PLANNING_WORKERS: usize = 4; + +type CtePair = (String, String); +/// `(model, SQL, expected SQL, lifted CTEs, body, expected columns, expected lifted CTEs)`. +type ChainRow = ( + String, + String, + Option, + Vec, + Option, + Option>, + Vec, +); +/// `(name, resolved SQL, lifted CTEs, comparison body)`. +type AssertionRow = (String, String, Vec, Option); +/// `(model, severity, message)`. +type WarningRow = (Option, &'static str, String); +/// `(SQL, chain, assertions, model names, warnings, distinct error messages)`. +type PlanRow = ( + Option, + Vec, + Vec, + Vec, + Vec, + Vec, +); + +/// The Python planning request: compiled models and tests, with the adapter's facts. +#[derive(FromPyObject)] +struct PlanningRequest<'py> { + #[pyo3(attribute)] + models: Vec>, + /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. + #[pyo3(attribute)] + rendered_model_sql: HashMap, + #[pyo3(attribute)] + functions: Vec<(String, String, String, String, String)>, + #[pyo3(attribute)] + tests: Vec>, + /// Cursor-rendered model query overrides by test index, replacing the test's own. + #[pyo3(attribute)] + rendered_test_overrides: HashMap>, + #[pyo3(attribute)] + sql_analysis_enabled: bool, + #[pyo3(attribute)] + sql_analysis_dialect: Option, + #[pyo3(attribute)] + set_difference_operator: String, + #[pyo3(attribute)] + requires_derived_table_aliases: bool, + #[pyo3(attribute)] + lexical_syntax: LexicalSyntaxInput, + #[pyo3(attribute)] + render_sql: bool, + #[pyo3(attribute)] + include_plan: bool, +} + +/// The Python chain request: compiled models and the tests whose chains to resolve. +#[derive(FromPyObject)] +struct ChainRequest<'py> { + #[pyo3(attribute)] + models: Vec>, + #[pyo3(attribute)] + tests: Vec>, + #[pyo3(attribute)] + lexical_syntax: LexicalSyntaxInput, +} + +/// Each test's ordered unmocked model chain; direct-logic tests have none. +#[pyfunction] +fn resolve_compiled_sql_test_chains( + py: Python<'_>, + request: ChainRequest<'_>, +) -> PyResult>> { + let batch = SqlTestChainBatch { + models: request + .models + .iter() + .map(|model| { + Ok(SqlTestChainModel { + name: model.getattr("name")?.extract()?, + model_dependencies: model_dependencies(model)?, + }) + }) + .collect::>()?, + tests: request + .tests + .iter() + .map(|test| plan_test(test, None)) + .collect::>()?, + lexical_syntax: request.lexical_syntax.into(), + }; + py.compiler_detach(|| resolve_sql_test_chains(batch)) + .map_err(value_error) +} + +/// Plan SQL tests in one batch and return each plan with its distinct error messages. +#[pyfunction] +fn plan_compiled_sql_tests( + py: Python<'_>, + request: PlanningRequest<'_>, +) -> PyResult<(Vec, u64, u64)> { + let batch: SqlTestPlanBatch = planning_batch(request)?; + let outcome = py + .compiler_detach(|| plan_sql_test_batch(batch)) + .map_err(value_error)?; + Ok(( + outcome.plans.into_iter().map(plan_row).collect(), + outcome.planning_ns, + outcome.rendering_ns, + )) +} + +fn planning_batch(mut request: PlanningRequest<'_>) -> PyResult { + let models: Vec = request + .models + .iter() + .enumerate() + .map(|(index, model)| plan_model(model, request.rendered_model_sql.remove(&index))) + .collect::>()?; + let tests: Vec = request + .tests + .iter() + .enumerate() + .map(|(index, test)| plan_test(test, request.rendered_test_overrides.remove(&index))) + .collect::>()?; + Ok(SqlTestPlanBatch { + models, + functions: request.functions.into_iter().map(plan_function).collect(), + tests, + sql_analysis_enabled: request.sql_analysis_enabled, + sql_analysis_dialect: request.sql_analysis_dialect, + set_difference_operator: request.set_difference_operator, + requires_derived_table_aliases: request.requires_derived_table_aliases, + workers: PLANNING_WORKERS, + render_sql: request.render_sql, + include_plan: request.include_plan, + lexical_syntax: request.lexical_syntax.into(), + }) +} + +fn plan_model( + model: &Bound<'_, PyAny>, + rendered_sql: Option, +) -> PyResult { + Ok(SqlTestPlanModel { + name: model.getattr("name")?.extract()?, + query_sql: match rendered_sql { + Some(sql) => sql, + None => model.getattr("query_sql")?.extract()?, + }, + model_dependencies: model_dependencies(model)?, + }) +} + +/// The names of the models among a compiled model's dependencies, in dependency order. +fn model_dependencies(model: &Bound<'_, PyAny>) -> PyResult> { + let mut names: Vec = Vec::new(); + for dependency in model.getattr("deps")?.try_iter()? { + let dependency = dependency?; + if dependency + .getattr("resource_type")? + .eq(MODEL_RESOURCE_TYPE)? + { + names.push(dependency.getattr("name")?.extract()?); + } + } + Ok(names) +} + +fn plan_function( + (name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix): ( + String, + String, + String, + String, + String, + ), +) -> SqlTestPlanFunction { + SqlTestPlanFunction { + name, + udf_prefix: Some(udf_prefix), + udf_suffix: Some(udf_suffix), + table_function_prefix: Some(table_function_prefix), + table_function_suffix: Some(table_function_suffix), + } +} + +fn plan_test( + test: &Bound<'_, PyAny>, + rendered_overrides: Option>, +) -> PyResult { + let payload = test.getattr("payload")?; + let payload: SqlTestPlanPayload = if payload.hasattr("actual_cte")? { + SqlTestPlanPayload::Direct { + mode: payload.getattr("mode")?.extract()?, + actual_cte: cte(&payload.getattr("actual_cte")?)?, + expected_cte: cte(&payload.getattr("expected_cte")?)?, + helper_ctes: ctes(&payload, "helper_ctes")?, + } + } else { + SqlTestPlanPayload::Model { + authored_ctes: ctes(&payload, "authored_ctes")?, + model_query_overrides: match rendered_overrides { + Some(overrides) => overrides, + None => payload.getattr("model_query_overrides")?.extract()?, + }, + expected_ctes: ctes(&payload, "expected_ctes")?, + expected_model_names: payload.getattr("expected_model_names")?.extract()?, + assertion_ctes: ctes(&payload, "assertion_ctes")?, + read_helper_names: Some(test.getattr("read_helper_names")?.extract()?), + reference_target_model_names: Some( + test.getattr("reference_target_model_names")?.extract()?, + ), + } + }; + Ok(SqlTestPlanTest { + name: test.getattr("name")?.extract()?, + file_label: test + .getattr("test_file")? + .getattr("relative_path")? + .str()? + .extract()?, + payload, + }) +} + +fn ctes(owner: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { + owner + .getattr(attribute)? + .try_iter()? + .map(|value| cte(&value?)) + .collect() +} + +fn cte(value: &Bound<'_, PyAny>) -> PyResult { + Ok(SqlTestCte { + name: value.getattr("name")?.extract()?, + sql_body: value.getattr("sql_body")?.extract()?, + }) +} + +fn plan_row(plan: SqlTestPlan) -> PlanRow { + let error_messages: Vec = sql_test_plan_error_messages(&plan); + ( + plan.sql, + plan.chain.into_iter().map(chain_row).collect(), + plan.assertions.into_iter().map(assertion_row).collect(), + plan.model_names, + plan.warnings + .into_iter() + .map(|warning| (warning.model_name, warning.severity, warning.message)) + .collect(), + error_messages, + ) +} + +fn chain_row(step: SqlTestChainStep) -> ChainRow { + ( + step.model_name, + step.resolved_sql, + step.expected_cte_sql, + step.lifted_ctes, + step.comparison_body_sql, + step.expected_columns, + step.expected_lifted_ctes, + ) +} + +fn assertion_row(step: SqlTestAssertionStep) -> AssertionRow { + ( + step.name, + step.resolved_sql, + step.lifted_ctes, + step.comparison_body_sql, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_compiled_sql_tests, module)?)?; + module.add_function(wrap_pyfunction!(resolve_compiled_sql_test_chains, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 641f84817b..d7cf7c6c23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -2,6 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { - Ok(()) +use crate::bindings::_helpers::sql_test_glue::planning; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + planning::register(module) } diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py b/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py index 456d975490..04d7b4b944 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py @@ -1,9 +1,78 @@ """Failure cases owned by the native SQL-test planning glue lane.""" +from scripts.compiler_differential._helpers.corpus.case_builder import failure_case from scripts.compiler_differential.models import FailureCase +_TEST_PATH: str = "tests/unit/test_customer_totals.sql" +_WINDOWED_PATH: str = "models/marts/windowed_orders.sql" +_WINDOWED_MODEL: str = ( + "MODEL (\n description 'Orders in the cursor window',\n materialized incremental,\n" + " incremental_strategy delete_insert,\n cursor order_id,\n cursor_type integer,\n" + " cursor_inputs (raw_orders order_id,),\n);\n\n" + 'SELECT order_id, amount\nFROM __source("raw_orders")\n' + "WHERE order_id >= __cursor_start() AND order_id < __cursor_end()\n" +) +_ORDERS_FIXTURE: str = "SELECT 1 AS order_id, 10 AS customer_id, 5.0 AS amount, 'placed' AS status" + + +def _test(*, header: str, ctes: str) -> dict[str, str]: + return {_TEST_PATH: f"TEST ({header});\n\nWITH\n{ctes}\nSELECT 1\n"} + def sql_test_glue_failure_cases() -> tuple[FailureCase, ...]: """Return this lane's failure cases; the lane appends here without editing shared lists.""" - return () + return ( + failure_case( + name="sql-test-glue-missing-mock", + expected_code="S000", + expected_message=( + "test 'totals_without_source': model 'stg_orders' references " + '__source("raw_orders") which has no mock' + ), + files=_test( + header='name "totals_without_source"', + ctes=( + "__ref__stg_orders AS (SELECT 10 AS customer_id, 5.0 AS amount),\n" + "__expected__stg_orders AS (SELECT 10 AS customer_id, 5.0 AS amount),\n" + "__expected__customer_totals AS (\n" + " SELECT 10 AS customer_id, 5.0 AS total_amount\n)" + ), + ), + ), + failure_case( + name="sql-test-glue-inverted-window", + expected_code="P001", + expected_message=( + "SQL test 'inverted_window' in 'tests/unit/test_customer_totals.sql': " + "cursor_start '20' must be before the exclusive cursor_end '10' for model " + "'windowed_orders'" + ), + files={ + _WINDOWED_PATH: _WINDOWED_MODEL, + **_test( + header='name "inverted_window", cursor_start 20, cursor_end 10', + ctes=( + f"__source__raw_orders AS ({_ORDERS_FIXTURE}),\n" + "__expected__windowed_orders AS (SELECT 1 AS order_id, 5.0 AS amount)" + ), + ), + }, + ), + failure_case( + name="sql-test-glue-window-without-cursor-model", + expected_code="P001", + expected_message=( + "SQL test 'plain_window' in 'tests/unit/test_customer_totals.sql' declares " + "cursor_start or cursor_end, but no model it evaluates uses __cursor_start() or " + "__cursor_end()" + ), + files=_test( + header='name "plain_window", cursor_start 1', + ctes=( + f"__source__raw_orders AS ({_ORDERS_FIXTURE}),\n" + f"__expected__stg_orders AS ({_ORDERS_FIXTURE})" + ), + ), + ), + ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d9199a38ca..b98b504587 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3,6 +3,12 @@ from collections.abc import Sequence from typing import Any, TypedDict +from sqlbuild.compiler.sql_test_glue.models import ( + NativeSqlTestChainRequest, + NativeSqlTestPlanningRequest, +) +from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow + BUILD_IDENTITY: str class LintPreparationRequest(TypedDict): @@ -610,6 +616,10 @@ def build_fast_column_lineage( ) -> list[tuple[str, list[tuple[str, str, str, list[tuple[str, str, str]]]], bool, str | None]]: ... # Native analysis: SQL test planning glue. +def plan_compiled_sql_tests( + request: NativeSqlTestPlanningRequest, / +) -> tuple[list[NativeSqlTestPlanRow], int, int]: ... +def resolve_compiled_sql_test_chains(request: NativeSqlTestChainRequest, /) -> list[list[str]]: ... # Native analysis: compiled project assembly. diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index 8fda15011b..6f04c17d01 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -574,23 +574,17 @@ def _plan_static_test_artifacts( with OperationLifecycle( operation_kind="project", operation_name="sql_test_planning" ) as lifecycle: - native_artifacts: tuple[NativeSqlTestArtifact, ...] | None = ( - plan_native_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - ) + plan_artifacts: Callable[..., tuple[NativeSqlTestArtifact, ...]] = ( + plan_native_sql_test_artifacts if native_stage_enabled(NativeStage.SQL_TEST_GLUE) - else None + else plan_and_render_sql_test_artifacts + ) + native_artifacts: tuple[NativeSqlTestArtifact, ...] = plan_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, ) - if native_artifacts is None: - native_artifacts = plan_and_render_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - ) lifecycle.completed(metadata={"item_count": len(native_artifacts)}) return native_artifacts diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py index 4f01af717f..92b6bae348 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py @@ -31,7 +31,6 @@ from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( plan_sql_tests_natively, resolve_sql_test_model_chains, - sql_test_plan_error_messages, ) from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError from sqlbuild.compiler.planner.models import ( @@ -94,7 +93,7 @@ def plan_sql_tests( ) continue plan_diagnostics: tuple[str, ...] = ( - *sql_test_plan_error_messages(warnings=plan.warnings), + *plan.error_messages, *_expected_column_diagnostics( test=test, plan=plan, fixture_planning_context=validation_context ), diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py index 89a301f85a..54c533f253 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py @@ -21,6 +21,8 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( declared_window_cursor_models, declares_cursor_window, @@ -38,6 +40,16 @@ from sqlbuild.compiler.profiling.classes.context import CompileTimingContext from sqlbuild.compiler.profiling.models import CompileTimingCollector from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.compiler.sql_test_glue.constants import CURSOR_INTRINSIC_NAMES +from sqlbuild.compiler.sql_test_glue.models import ( + NativeSqlTestChainRequest, + NativeSqlTestPlanningRequest, +) +from sqlbuild.compiler.sql_test_glue.types import ( + NativeAssertionStepRow, + NativeChainStepRow, + NativeSqlTestPlanRow, +) from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule _UNRESOLVED_REFERENCE_PREFIX: str = "sql_test_reference:" @@ -56,7 +68,7 @@ def plan_and_render_sql_test_artifacts( """Plan and render SQL tests in one deterministic native project call.""" artifacts: list[NativeSqlTestArtifact] = [] - for plan in plan_sql_tests_natively( + for plan in _plan_sql_tests_from_json( project=project, tests=tests, adapter=adapter, @@ -70,7 +82,35 @@ def plan_and_render_sql_test_artifacts( NativeSqlTestArtifact( sql=plan.sql, model_names=plan.model_names, - error_messages=sql_test_plan_error_messages(warnings=plan.warnings), + error_messages=plan.error_messages, + ) + ) + return tuple(artifacts) + + +def plan_compiled_sql_test_artifacts( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, +) -> tuple[NativeSqlTestArtifact, ...]: + """Plan and render SQL tests from the compiled objects, with natively projected errors.""" + + artifacts: list[NativeSqlTestArtifact] = [] + for plan in _plan_compiled_sql_tests( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + render_sql=True, + include_plan=False, + ): + if plan.sql is None: + raise NativeSqlTestPlanningError("native SQL-test planning omitted rendered SQL") + artifacts.append( + NativeSqlTestArtifact( + sql=plan.sql, model_names=plan.model_names, error_messages=plan.error_messages ) ) return tuple(artifacts) @@ -97,6 +137,185 @@ def plan_sql_tests_natively( ) -> tuple[NativeSqlTestPlan, ...]: """Plan SQL-test chains and assertions in one native batch, optionally rendering SQL.""" + if native_stage_enabled(NativeStage.SQL_TEST_GLUE): + return _plan_compiled_sql_tests( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + render_sql=render_sql, + include_plan=include_plan, + ) + return _plan_sql_tests_from_json( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + render_sql=render_sql, + include_plan=include_plan, + ) + + +def _plan_compiled_sql_tests( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, + render_sql: bool, + include_plan: bool, +) -> tuple[NativeSqlTestPlan, ...]: + """Plan from the compiled objects natively; Python supplies only the adapter's renderings.""" + + if not tests: + return () + start_ns: int = time.perf_counter_ns() + rendered_model_sql: dict[int, str] = { + index: render_test_cursor_intrinsics( + sql=model.query_sql, model=model, adapter=adapter, test=None + ) + for index, model in enumerate(project.models) + if _mentions_cursor_intrinsic(model.query_sql) + } + functions: tuple[tuple[str, str, str, str, str], ...] = _function_templates( + project=project, adapter=adapter + ) + rendered_test_overrides: dict[int, dict[str, str]] = _rendered_test_overrides( + project=project, tests=tests, adapter=adapter + ) + request: NativeSqlTestPlanningRequest = NativeSqlTestPlanningRequest( + models=project.models, + rendered_model_sql=rendered_model_sql, + functions=functions, + tests=tests, + rendered_test_overrides=rendered_test_overrides, + sql_analysis_enabled=sql_analysis_enabled, + sql_analysis_dialect=adapter.sql_analysis_dialect(), + set_difference_operator=adapter.render_set_difference_operator(), + requires_derived_table_aliases=adapter.requires_derived_table_aliases(), + lexical_syntax=project.sql_lexical_syntax.native_mapping, + render_sql=render_sql, + include_plan=include_plan, + ) + try: + rows, planning_ns, rendering_ns = _native.plan_compiled_sql_tests(request) + except ValueError as error: + raise _planning_error(error=error, tests=tests) from None + _record_planning_timing(start_ns=start_ns, planning_ns=planning_ns, rendering_ns=rendering_ns) + return tuple(_plan_from_row(row=row) for row in rows) + + +def _plan_from_row(*, row: NativeSqlTestPlanRow) -> NativeSqlTestPlan: + sql, chain, assertions, model_names, warnings, error_messages = row + return NativeSqlTestPlan( + chain=tuple(_chain_step_from_row(row=step) for step in chain), + assertions=tuple(_assertion_step_from_row(row=step) for step in assertions), + model_names=tuple(model_names), + warnings=tuple( + PlanWarning(model_name=model_name, severity=WarningSeverity(severity), message=message) + for model_name, severity, message in warnings + ), + error_messages=tuple(error_messages), + sql=sql, + ) + + +def _assertion_step_from_row(*, row: NativeAssertionStepRow) -> SqlTestAssertionStep: + name, resolved_sql, lifted_ctes, comparison_body_sql = row + return SqlTestAssertionStep( + name=name, + resolved_sql=resolved_sql, + lifted_ctes=tuple(lifted_ctes), + comparison_body_sql=comparison_body_sql, + ) + + +def _chain_step_from_row(*, row: NativeChainStepRow) -> ChainStep: + ( + model_name, + resolved_sql, + expected_cte_sql, + lifted_ctes, + comparison_body_sql, + expected_columns, + expected_lifted_ctes, + ) = row + return ChainStep( + model_name=model_name, + resolved_sql=resolved_sql, + expected_cte_sql=expected_cte_sql, + lifted_ctes=tuple(lifted_ctes), + comparison_body_sql=comparison_body_sql, + expected_columns=None if expected_columns is None else tuple(expected_columns), + expected_lifted_ctes=tuple(expected_lifted_ctes), + ) + + +def _mentions_cursor_intrinsic(sql: str) -> bool: + """Whether SQL names a cursor intrinsic; SQL that does not is never rendered differently.""" + + return any(name in sql for name in CURSOR_INTRINSIC_NAMES) + + +def _rendered_test_overrides( + *, project: CompiledProject, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter +) -> dict[int, dict[str, str]]: + """Cursor-rendered model overrides for the tests whose overrides cursor windows change.""" + + models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + windowed_indexes: tuple[int, ...] = tuple( + index + for index, test in enumerate(tests) + if isinstance(test.payload, CompiledModelSqlTestPayload) + and declares_cursor_window(test=test) + ) + chains: tuple[tuple[str, ...], ...] = ( + resolve_sql_test_model_chains( + project=project, tests=tuple(tests[index] for index in windowed_indexes) + ) + if windowed_indexes + else () + ) + chains_by_index: dict[int, tuple[str, ...]] = dict(zip(windowed_indexes, chains, strict=True)) + rendered: dict[int, dict[str, str]] = {} + for index, test in enumerate(tests): + payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload + if not isinstance(payload, CompiledModelSqlTestPayload): + continue + chain: tuple[str, ...] | None = chains_by_index.get(index) + if chain is None and not any( + _mentions_cursor_intrinsic(sql) for sql in payload.model_query_overrides.values() + ): + continue + overrides: dict[str, str] | None = _windowed_model_query_overrides( + test=test, models_by_name=models_by_name, adapter=adapter, chain=chain + ) + if overrides is not None: + rendered[index] = overrides + return rendered + + +def _record_planning_timing(*, start_ns: int, planning_ns: int, rendering_ns: int) -> None: + elapsed_ns: int = time.perf_counter_ns() - start_ns + boundary_overhead_ns: int = max(0, elapsed_ns - planning_ns - rendering_ns) + collector: CompileTimingCollector | None = CompileTimingContext.active.get() + if collector is not None: + collector.add( + phase="test_planning_ms", + elapsed_ns=planning_ns + boundary_overhead_ns, + ) + collector.add(phase="comparison_render_ms", elapsed_ns=rendering_ns) + + +def _plan_sql_tests_from_json( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, + render_sql: bool, + include_plan: bool, +) -> tuple[NativeSqlTestPlan, ...]: if not tests: return () start_ns: int = time.perf_counter_ns() @@ -143,15 +362,7 @@ def plan_sql_tests_natively( plans: tuple[NativeSqlTestPlan, ...] = tuple( _plan_from_payload(value=value) for value in response ) - elapsed_ns: int = time.perf_counter_ns() - start_ns - boundary_overhead_ns: int = max(0, elapsed_ns - planning_ns - rendering_ns) - collector: CompileTimingCollector | None = CompileTimingContext.active.get() - if collector is not None: - collector.add( - phase="test_planning_ms", - elapsed_ns=planning_ns + boundary_overhead_ns, - ) - collector.add(phase="comparison_render_ms", elapsed_ns=rendering_ns) + _record_planning_timing(start_ns=start_ns, planning_ns=planning_ns, rendering_ns=rendering_ns) return plans @@ -162,6 +373,18 @@ def resolve_sql_test_model_chains( if not tests: return () + if native_stage_enabled(NativeStage.SQL_TEST_GLUE): + try: + native_chains: list[list[str]] = _native.resolve_compiled_sql_test_chains( + NativeSqlTestChainRequest( + models=project.models, + tests=tests, + lexical_syntax=project.sql_lexical_syntax.native_mapping, + ) + ) + except ValueError as error: + raise _planning_error(error=error, tests=tests) from None + return tuple(tuple(chain) for chain in native_chains) request: dict[str, object] = { "models": _chain_model_requests(project=project), "tests": [_test_request(test=test) for test in tests], @@ -240,11 +463,13 @@ def _plan_from_payload(*, value: object) -> NativeSqlTestPlan: or not isinstance(warnings, list) ): raise NativeSqlTestPlanningError("native SQL-test planning returned an invalid result") + plan_warnings: tuple[PlanWarning, ...] = tuple(_warning(value=warning) for warning in warnings) return NativeSqlTestPlan( chain=tuple(_chain_step(value=step) for step in chain), assertions=tuple(_assertion_step(value=step) for step in assertions), model_names=_string_tuple(value=payload.get("modelNames"), context="model names"), - warnings=tuple(_warning(value=warning) for warning in warnings), + warnings=plan_warnings, + error_messages=sql_test_plan_error_messages(warnings=plan_warnings), sql=sql if isinstance(sql, str) else None, ) @@ -330,7 +555,30 @@ def _string_tuple(*, value: object, context: str) -> tuple[str, ...]: def _function_requests( *, project: CompiledProject, adapter: BaseAdapter ) -> list[dict[str, object]]: - functions: list[dict[str, object]] = [] + return [ + { + "name": name, + "udfPrefix": udf_prefix, + "udfSuffix": udf_suffix, + "tableFunctionPrefix": table_function_prefix, + "tableFunctionSuffix": table_function_suffix, + } + for ( + name, + udf_prefix, + udf_suffix, + table_function_prefix, + table_function_suffix, + ) in _function_templates(project=project, adapter=adapter) + ] + + +def _function_templates( + *, project: CompiledProject, adapter: BaseAdapter +) -> tuple[tuple[str, str, str, str, str], ...]: + """Each function's name and the adapter's UDF and table-function call templates.""" + + functions: list[tuple[str, str, str, str, str]] = [] for function in project.functions: target: str | None = function.destination.qualified_name if target is None: @@ -348,15 +596,9 @@ def _function_requests( ) ) functions.append( - { - "name": function.name, - "udfPrefix": udf_prefix, - "udfSuffix": udf_suffix, - "tableFunctionPrefix": table_function_prefix, - "tableFunctionSuffix": table_function_suffix, - } + (function.name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix) ) - return functions + return tuple(functions) def _planning_test_requests( diff --git a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py new file mode 100644 index 0000000000..b3cfbee3fa --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py @@ -0,0 +1,25 @@ +"""Planner entrypoint for SQL-test artifacts planned natively from the compiled objects.""" + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( + plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, +) +from sqlbuild.compiler.planner.models import NativeSqlTestArtifact + + +def plan_compiled_sql_test_artifacts( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, +) -> tuple[NativeSqlTestArtifact, ...]: + """Plan and render SQL tests natively from the compiled objects, without a JSON request.""" + + return _plan_compiled_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index b726b8ff07..a3959b72e0 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -1426,6 +1426,7 @@ class NativeSqlTestPlan: assertions: tuple[SqlTestAssertionStep, ...] model_names: tuple[str, ...] warnings: tuple[PlanWarning, ...] + error_messages: tuple[str, ...] sql: str | None = None diff --git a/src/sqlbuild/compiler/sql_test_glue/constants.py b/src/sqlbuild/compiler/sql_test_glue/constants.py new file mode 100644 index 0000000000..727ece2d99 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/constants.py @@ -0,0 +1,5 @@ +"""Constants for the native SQL-test planning glue.""" + +from __future__ import annotations + +CURSOR_INTRINSIC_NAMES: tuple[str, ...] = ("__cursor_start", "__cursor_end") diff --git a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py index 7b5861fead..0304327f4f 100644 --- a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py +++ b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py @@ -4,6 +4,9 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.planner.main.execution._plan_compiled_sql_test_artifacts import ( + plan_compiled_sql_test_artifacts, +) from sqlbuild.compiler.planner.models import NativeSqlTestArtifact @@ -13,7 +16,12 @@ def plan_native_sql_test_artifacts( tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter, sql_analysis_enabled: bool, -) -> tuple[NativeSqlTestArtifact, ...] | None: - """Return one artifact per test, or None where Python must plan the tests.""" +) -> tuple[NativeSqlTestArtifact, ...]: + """Return one artifact per test, planned from the compiled objects without a JSON request.""" - return None + return plan_compiled_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + ) diff --git a/src/sqlbuild/compiler/sql_test_glue/models.py b/src/sqlbuild/compiler/sql_test_glue/models.py new file mode 100644 index 0000000000..0883787198 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/models.py @@ -0,0 +1,34 @@ +"""Requests the native SQL-test planning glue reads from compiled objects.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.models import CompiledModel, CompiledSqlTest + + +@dataclass(frozen=True) +class NativeSqlTestPlanningRequest: + """Compiled models and tests to plan, with the adapter facts native planning cannot derive.""" + + models: tuple[CompiledModel, ...] + rendered_model_sql: dict[int, str] + functions: tuple[tuple[str, str, str, str, str], ...] + tests: tuple[CompiledSqlTest, ...] + rendered_test_overrides: dict[int, dict[str, str]] + sql_analysis_enabled: bool + sql_analysis_dialect: str | None + set_difference_operator: str + requires_derived_table_aliases: bool + lexical_syntax: dict[str, object] + render_sql: bool + include_plan: bool + + +@dataclass(frozen=True) +class NativeSqlTestChainRequest: + """Compiled models and the tests whose unmocked model chains to resolve.""" + + models: tuple[CompiledModel, ...] + tests: tuple[CompiledSqlTest, ...] + lexical_syntax: dict[str, object] diff --git a/src/sqlbuild/compiler/sql_test_glue/types.py b/src/sqlbuild/compiler/sql_test_glue/types.py new file mode 100644 index 0000000000..9a3fdf65c4 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/types.py @@ -0,0 +1,24 @@ +"""Row shapes the native SQL-test planning glue returns.""" + +from __future__ import annotations + +type CtePairRow = tuple[str, str] +type NativeChainStepRow = tuple[ + str, + str, + str | None, + list[CtePairRow], + str | None, + list[str] | None, + list[CtePairRow], +] +type NativeAssertionStepRow = tuple[str, str, list[CtePairRow], str | None] +type NativePlanWarningRow = tuple[str | None, str, str] +type NativeSqlTestPlanRow = tuple[ + str | None, + list[NativeChainStepRow], + list[NativeAssertionStepRow], + list[str], + list[NativePlanWarningRow], + list[str], +] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 7cd113a127..cf88cb5277 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -3,13 +3,15 @@ from __future__ import annotations from pathlib import Path -from unittest.mock import ANY +from unittest.mock import ANY, Mock import pytest +import sqlbuild._native as native_module from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.models import ProjectColumnLineage +from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, ) @@ -38,7 +40,7 @@ "evaluate_native_model_contracts": [ANY], "native_promotion_conflict_diagnostics": [()], "build_native_column_lineage": [ANY], - "plan_native_sql_test_artifacts": [None], + "plan_native_sql_test_artifacts": [ANY], }, ) ], @@ -52,6 +54,8 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ ) -> None: prepared_project: Path = tmp_path / "orders" prepare_compile_reuse_project(project_dir=prepared_project) + native_sql_test_planning: Mock = Mock(wraps=native_module.plan_compiled_sql_tests) + monkeypatch.setattr(native_module, "plan_compiled_sql_tests", native_sql_test_planning) outcomes: dict[str, tuple[CompileReuseRun, dict[str, list[object]]]] = { engine: engine_in_process_compile( project_dir=copy_compile_project( @@ -77,6 +81,13 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ assert isinstance(preview_seams["build_native_column_lineage"][0], ProjectColumnLineage) assert isinstance(preview_seams["analyze_native_model_sql"][0], dict) assert isinstance(preview_seams["infer_native_expression_source_shapes"][0], tuple) + planned_artifacts: object = preview_seams["plan_native_sql_test_artifacts"][0] + assert isinstance(planned_artifacts, tuple) + assert planned_artifacts + assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) + assert [len(call.args[0].tests) for call in native_sql_test_planning.call_args_list] == [ + len(planned_artifacts) + ] assert isinstance(preview_seams["complete_native_semantic_diagnostics"][0], CompiledProject) assert isinstance(preview_seams["evaluate_native_model_contracts"][0], ContractValidationResult) assert (native_run.returncode, preview_run.returncode) == (0, 0) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 403c5373fb..24d8b32133 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -48,6 +48,7 @@ class SqlTestPlanInspectionE2ETestCase: expected_stdout_fragments: tuple[str, ...] expected_exit_code: int = 0 unexpected_stdout_fragments: tuple[str, ...] = () + engine_env: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) @@ -178,6 +179,7 @@ class CursorWindowE2ETestCase: expected_exit_code: int expected_output_fragments: tuple[str, ...] expected_compiled_fragments: tuple[str, ...] = () + engine_env: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py index cac1a1ad82..ed17a867e3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py @@ -19,6 +19,8 @@ "SQL test 'declared_window_case' in 'tests/unit/declared_window_case.sql'" ) +_PREVIEW_ENGINE: tuple[tuple[str, str], ...] = (("SQLBUILD_COMPILER_ENGINE", "native-preview"),) + @pytest.mark.parametrize( "test_case", @@ -33,6 +35,17 @@ "TIMESTAMP '2999-12-31 00:00:00'", ), ), + CursorWindowE2ETestCase( + description="windows planned natively on the native-preview engine", + tests=CURSOR_WINDOW_PASSING_TESTS, + expected_exit_code=0, + expected_output_fragments=("PASS=6 FAIL=0 TOTAL=6",), + expected_compiled_fragments=( + "TIMESTAMP '1900-01-01 00:00:00'", + "TIMESTAMP '2999-12-31 00:00:00'", + ), + engine_env=_PREVIEW_ENGINE, + ), ), ids=lambda case: case.description, ) @@ -45,11 +58,12 @@ def test_given_models_using_cursor_intrinsics_when_testing_then_window_is_render project_name="cursor_window_project", repo_files=build_cursor_window_project_files(tests=test_case.tests), ) + engine_env: dict[str, str] = dict(test_case.engine_env) test_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "test"), project_dir=project_dir + command=("--no-color", "test"), project_dir=project_dir, env=engine_env ) compile_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "compile"), project_dir=project_dir + command=("--no-color", "compile"), project_dir=project_dir, env=engine_env ) output: str = test_result.stdout + test_result.stderr @@ -94,6 +108,22 @@ def test_given_models_using_cursor_intrinsics_when_testing_then_window_is_render ), ), ), + CursorWindowE2ETestCase( + description="inverted declared window on the native-preview engine", + tests=build_declared_window_test( + window='cursor_start "2026-02-03", cursor_end "2026-02-01"', + model_name="daily_orders", + ), + expected_exit_code=1, + expected_output_fragments=( + ( + _DECLARED_ERROR_PREFIX + + ": cursor_start '2026-02-03' must be before the exclusive cursor_end " + "'2026-02-01' for model 'daily_orders'" + ), + ), + engine_env=_PREVIEW_ENGINE, + ), CursorWindowE2ETestCase( description="invalid timestamp bound", tests=build_declared_window_test( @@ -167,7 +197,11 @@ def test_given_invalid_declared_cursor_window_when_testing_and_compiling_then_er ) results: tuple[subprocess.CompletedProcess[str], ...] = tuple( - run_sqb(command=command, project_dir=project_dir) + run_sqb( + command=command, + project_dir=project_dir, + env=dict(test_case.engine_env), + ) for command in (("--no-color", "test"), ("--no-color", "compile")) ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py index e03de5d349..ce59820a8f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py @@ -116,6 +116,8 @@ "expected output 'orders' lists columns that model 'orders' does not output: region" ) +_PREVIEW_ENGINE: tuple[tuple[str, str], ...] = (("SQLBUILD_COMPILER_ENGINE", "native-preview"),) + @pytest.mark.parametrize( "test_case", @@ -536,6 +538,16 @@ def test_given_invalid_fixture_when_testing_then_static_diagnostics_prevent_conn "boundary: stg_orders is replaced by __ref__stg_orders", ), ), + SqlTestPlanInspectionE2ETestCase( + description="ref mock boundary planned natively on the native-preview engine", + repo_files=build_mock_boundary_test_project_files(), + expected_stdout_fragments=( + "mocked refs: stg_orders", + "real models: int_orders, fact_orders", + "boundary: stg_orders is replaced by __ref__stg_orders", + ), + engine_env=_PREVIEW_ENGINE, + ), SqlTestPlanInspectionE2ETestCase( description="recursive derived columns are not required from the mocked input", repo_files=build_recursive_ref_fixture_project_files(), @@ -570,6 +582,7 @@ def test_given_sql_test_when_inspecting_then_fixture_boundaries_and_chain_are_sh result: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "test", "--inspect"), project_dir=project_dir, + env=dict(test_case.engine_env), ) assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/__init__.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py new file mode 100644 index 0000000000..1ead44a814 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from tests.integration.src.sqlbuild.compiler.sql_test_glue.helpers import SqlTestCorpusShape + + +@dataclass(frozen=True) +class GeneratedSqlTestPlanningParityTestCase: + """Seeded SQL-test projects the native planning glue and the JSON request must agree on.""" + + description: str + seed: int + count: int + test_count: int + shape: SqlTestCorpusShape + adapter_names: tuple[str, ...] + expected_minimum_native_planned: int + expected_minimum_native_with_errors: int + expected_minimum_native_chains: int + expected_minimum_native_raised: int + expected_native_raised_kinds: frozenset[str] + expected_minimum_answered_batches: int + expected_minimum_raised_outcomes: int diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py new file mode 100644 index 0000000000..cfbadc548b --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -0,0 +1,454 @@ +"""Seeded SQL-test projects and both planning paths for the native planning glue tests.""" + +from __future__ import annotations + +import random +from collections import Counter +from collections.abc import Callable +from dataclasses import dataclass, replace +from itertools import chain, compress +from pathlib import Path +from typing import cast + +import pytest + +import sqlbuild._native as native_module +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import ( + CompiledModelSqlTestPayload, + CompiledProject, + CompiledSqlTest, + CompileSqlTestCte, +) +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.pipeline.main.graph import build_project_graph +from sqlbuild.compiler.planner._helpers.sql_tests import native_planning +from sqlbuild.compiler.planner.models import NativeSqlTestArtifact +from sqlbuild.compiler.sql_test_glue.models import ( + NativeSqlTestChainRequest, + NativeSqlTestPlanningRequest, +) +from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow + + +@dataclass(frozen=True) +class PlanningCallOutcome: + """What one planning call returned, or the type and text of what it raised.""" + + value: object + raised: tuple[str, str] | None + + +@dataclass(frozen=True) +class SqlTestCorpusShape: + """The cursor windows and failing shapes a generated SQL-test project draws from.""" + + windows: tuple[str, ...] + stray_window_share: float + helper_redefinition_share: float + assertion_share: float + unflattenable_assertion_share: float + + +_PROJECT_TOML: str = ( + 'name = "orders_testing"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +_SOURCES: str = """sources: + - name: raw_orders + description: Orders feed. + schema: main + table: raw_orders + columns: + - name: order_id + type: INTEGER + - name: customer_id + type: INTEGER + - name: amount + type: INTEGER + - name: status + type: VARCHAR + - name: ordered_at + type: TIMESTAMP + - name: raw_customers + description: Customers feed. + schema: main + table: raw_customers + columns: + - name: customer_id + type: INTEGER + - name: region + type: VARCHAR +""" +_SEEDS: str = """seeds: + - name: regions + description: Region labels. + columns: + - name: region + type: VARCHAR + - name: label + type: VARCHAR +""" +_MACROS: str = ( + "def cents(column: str) -> str:\n" + ' return f"({column}) * 100"\n\n' + "def known_region() -> str:\n" + ' return "region IS NOT NULL"\n' +) +_UDF: str = ( + 'FUNCTION (\n description "Whether an amount is large",\n' + " arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount > 100\n" +) +_TABLE_FUNCTION: str = ( + 'FUNCTION (\n description "Orders of one customer",\n' + " arguments (p_customer_id INTEGER),\n" + " returns table (order_id INTEGER, amount INTEGER)\n);\n\n" + 'SELECT order_id, amount FROM __ref("stg_orders") WHERE customer_id = p_customer_id\n' +) +_BASE_MODELS: dict[str, str] = { + "stg_orders": ( + 'SELECT order_id, customer_id, amount, status, ordered_at\nFROM __source("raw_orders")' + ), + "stg_customers": ( + 'SELECT customer_id, region\nFROM __source("raw_customers")\nWHERE @known_region()' + ), + "int_orders": ( + 'SELECT o.order_id, o.customer_id, c.region, @cents("o.amount") AS amount_cents,\n' + ' __udf("udf__is_big")(o.amount) AS is_big\n' + 'FROM __ref("stg_orders") o\nJOIN __ref("stg_customers") c ON o.customer_id = c.customer_id' + ), + "fct_regions": ( + "WITH totals AS (\n" + ' SELECT region, SUM(amount_cents) AS total_cents FROM __ref("int_orders") GROUP BY 1\n' + ")\nSELECT t.region, r.label, t.total_cents\n" + 'FROM totals t\nLEFT JOIN __seed("regions") r ON t.region = r.region' + ), + "customer_orders": ('SELECT order_id, amount\nFROM __table_fn("table_fn__customer_orders")(1)'), +} +_DAILY_ORDERS: str = ( + "MODEL (description 'Daily orders.',\n materialized incremental,\n" + " incremental_strategy delete_insert,\n cursor ordered_at,\n cursor_type timestamp,\n" + " cursor_grain day,\n cursor_inputs (raw_orders ordered_at,),\n);\n\n" + "SELECT ordered_at, amount\n" + 'FROM __source("raw_orders")\n' + "WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end()\n" +) +_FIXTURES: dict[str, str] = { + "__source__raw_orders": ( + "SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount, 'placed' AS status, " + "TIMESTAMP '2026-02-01 10:00:00' AS ordered_at" + ), + "__source__raw_customers": "SELECT 10 AS customer_id, 'east' AS region", + "__seed__regions": "SELECT 'east' AS region, 'East' AS label", + "__ref__stg_orders": ( + "SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount, 'placed' AS status, " + "TIMESTAMP '2026-02-01 10:00:00' AS ordered_at" + ), + "__ref__stg_customers": "SELECT 10 AS customer_id, 'east' AS region", + "__ref__int_orders": ( + "SELECT 1 AS order_id, 10 AS customer_id, 'east' AS region, 15000 AS amount_cents, " + "TRUE AS is_big" + ), + "__table_fn__table_fn__customer_orders": "SELECT 1 AS order_id, 150 AS amount", + "__macro__cents": "SELECT 'amount'", +} +_UPSTREAM: dict[str, tuple[str, ...]] = { + "stg_orders": ("__source__raw_orders",), + "stg_customers": ("__source__raw_customers",), + "int_orders": ("__ref__stg_orders", "__ref__stg_customers", "__source__raw_orders"), + "fct_regions": ("__ref__int_orders", "__seed__regions", "__source__raw_customers"), + "customer_orders": ("__table_fn__table_fn__customer_orders",), + "daily_orders": ("__source__raw_orders",), +} +_DIRECT_TESTS: tuple[str, ...] = ( + 'TEST (mode macro, name "doubles_cents");\n\nWITH\n' + "amounts AS (SELECT 2 AS amount),\n" + '__macro_actual__ AS (SELECT @cents("amount") AS cents FROM amounts),\n' + "__macro_expected__ AS (SELECT 200 AS cents)\nSELECT 1\n", + 'TEST (mode udf, name "flags_big_amounts");\n\nWITH\n' + '__udf_actual__ AS (SELECT __udf("udf__is_big")(150) AS is_big),\n' + "__udf_expected__ AS (SELECT TRUE AS is_big)\nSELECT 1\n", + 'TEST (mode table_fn, name "lists_customer_orders");\n\nWITH\n' + "__table_fn_actual__ AS (\n" + ' SELECT order_id, amount FROM __table_fn("table_fn__customer_orders")(10)\n),\n' + "__table_fn_expected__ AS (SELECT 1 AS order_id, 150 AS amount)\nSELECT 1\n", +) +_DIRECT_TEST_SHARE: float = 0.8 +_MOCK_SHARE: float = 0.85 +_MACRO_MOCK_SHARE: float = 0.5 +_HELPER_SHARE: float = 0.4 +_UNMOCKED_ASSERTION: CompileSqlTestCte = CompileSqlTestCte( + name="__assert__no_returns", sql_body='SELECT order_id FROM __source("raw_returns") WHERE 1 = 0' +) +_REGIONS_ASSERTION: str = 'SELECT * FROM __ref("fct_regions") WHERE 1 = 0' +_NATIVE_ERROR_KIND_SEPARATOR: str = ":" +_NATIVE_RAISED_PREFIX: str = "native_raised:" +NO_WINDOW: str = "" +VALID_WINDOW: str = ', cursor_start "2026-02-01", cursor_end "2026-02-03"' +INVERTED_WINDOW: str = ', cursor_start "2026-02-03", cursor_end "2026-02-01"' + + +def generated_sql_test_files( + *, rng: random.Random, test_count: int, shape: SqlTestCorpusShape +) -> dict[str, str]: + """A project of mocked, expected, asserted, helper, direct and failing SQL tests.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": _SOURCES, + "seeds/regions.yml": _SEEDS, + "seeds/regions.csv": "region,label\neast,East\n", + "macros/amounts.py": _MACROS, + "functions/sql/udf__is_big.sql": _UDF, + "functions/sql/table_fn__customer_orders.sql": _TABLE_FUNCTION, + "models/daily_orders.sql": _DAILY_ORDERS, + } + files.update( + (f"models/{name}.sql", f'MODEL (description "Model {name}.");\n\n{sql}\n') + for name, sql in _BASE_MODELS.items() + ) + files.update( + (f"tests/unit/test_generated_{index}.sql", _model_test(rng=rng, index=index, shape=shape)) + for index in range(test_count) + ) + files.update( + compress( + ( + (f"tests/unit/test_direct_{index}.sql", sql) + for index, sql in enumerate(_DIRECT_TESTS) + ), + [rng.random() < _DIRECT_TEST_SHARE for _ in _DIRECT_TESTS], + ) + ) + return files + + +def _model_test(*, rng: random.Random, index: int, shape: SqlTestCorpusShape) -> str: + """One model test: the first target's first mock, then random mocks, helper and checks. + + A helper may be read through a CTE whose nested WITH redefines it, which native planning + rejects; an assertion on a model beginning with WITH cannot be flattened for SQL Server. + """ + + targets: list[str] = rng.sample(sorted(_UPSTREAM), k=rng.choice((1, 1, 2))) + windowed: bool = "daily_orders" in targets or rng.random() < shape.stray_window_share + window: str = rng.choice(((NO_WINDOW,), shape.windows)[windowed]) + fixtures: list[str] = list(chain.from_iterable(_UPSTREAM[target] for target in targets)) + mocks: list[str] = [ + fixtures[0], + *compress(fixtures, [rng.random() < _MOCK_SHARE for _ in fixtures]), + *rng.sample(sorted(_FIXTURES), k=rng.choice((0, 0, 0, 1))), + *(("__macro__cents",) * ("int_orders" in targets and rng.random() < _MACRO_MOCK_SHARE)), + ] + has_helper: bool = rng.random() < _HELPER_SHARE + helper: str = f"wanted_{index}" + shadow: str = f"shadowing_{index}" + redefined: bool = rng.random() < shape.helper_redefinition_share + helper_ctes: list[tuple[str, str]] = [ + ( + shadow, + f"WITH {helper} AS (SELECT 1 AS order_id) SELECT order_id FROM {helper}", + ), + ( + helper, + ( + rng.choice( + ( + "SELECT 1 AS order_id", + f'SELECT order_id FROM __ref("{rng.choice(targets)}")', + ) + ), + f"SELECT order_id FROM {shadow}", + )[redefined], + ), + ][int(not redefined) : 2 * int(has_helper)] + expected_sql: str = ("SELECT 1 AS order_id", f"SELECT order_id FROM {helper}")[has_helper] + checks: list[tuple[str, str]] = [ + ( + (f"__expected__{target}", expected_sql), + (f"__assert__{target}_has_rows", f'SELECT * FROM __ref("{target}") WHERE 1 = 0'), + )[rng.random() < shape.assertion_share] + for target in targets + ] + unflattenable: list[tuple[str, str]] = [(f"__assert__regions_{index}", _REGIONS_ASSERTION)][ + : int(rng.random() < shape.unflattenable_assertion_share) + ] + ctes: dict[str, str] = { + **{mock: _FIXTURES[mock] for mock in mocks}, + **dict(helper_ctes), + **dict(checks), + **dict(unflattenable), + } + body: str = ",\n".join(f"{name} AS (\n {sql}\n)" for name, sql in ctes.items()) + return f'TEST (name "generated_{index}"{window});\n\nWITH\n{body}\nSELECT 1\n' + + +def compiled_project(*, project_dir: Path, files: dict[str, str]) -> CompiledProject: + """Write and compile a project, keeping any compile diagnostics on it.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + return build_project_graph( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + ).project + + +def with_unmocked_assertion(*, project: CompiledProject) -> CompiledSqlTest: + """The first model test with an assertion calling a source nothing mocks. + + The compiler reports such a call itself, so this stands in for a reference it missed and + reaches the planner's own unresolved-reference error. + """ + + test: CompiledSqlTest = next( + filter( + lambda test: isinstance(test.payload, CompiledModelSqlTestPayload), project.sql_tests + ) + ) + payload: CompiledModelSqlTestPayload = cast(CompiledModelSqlTestPayload, test.payload) + return replace( + test, + payload=replace( + payload, + assertion_ctes=(*payload.assertion_ctes, _UNMOCKED_ASSERTION), + authored_ctes=(*payload.authored_ctes, _UNMOCKED_ASSERTION), + ), + ) + + +def use_sql_test_glue(*, monkeypatch: pytest.MonkeyPatch, enabled: bool) -> None: + """Route SQL-test planning through the native glue, or through the JSON request.""" + + monkeypatch.setattr( + native_planning, + "native_stage_enabled", + lambda stage: enabled and stage is NativeStage.SQL_TEST_GLUE, + ) + + +def planning_outcome( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + render_sql: bool, +) -> PlanningCallOutcome: + """The plans `plan_sql_tests_natively` returns, or what it raises.""" + + return _outcome( + lambda: native_planning.plan_sql_tests_natively( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + render_sql=render_sql, + include_plan=not render_sql, + ) + ) + + +def artifact_outcome( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + glue: bool, +) -> PlanningCallOutcome: + """Rendered artifacts from the glue or from the JSON request, or what planning raises.""" + + plan: Callable[..., tuple[NativeSqlTestArtifact, ...]] = ( + native_planning.plan_and_render_sql_test_artifacts, + native_planning.plan_compiled_sql_test_artifacts, + )[glue] + return _outcome( + lambda: plan( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + ) + ) + + +def chain_outcome( + *, project: CompiledProject, tests: tuple[CompiledSqlTest, ...] +) -> PlanningCallOutcome: + """Each test's model chain, or what chain resolution raises.""" + + return _outcome( + lambda: native_planning.resolve_sql_test_model_chains(project=project, tests=tests) + ) + + +def outcome_kind(outcome: object) -> str: + """`raised` when the call raised, else `answered`; only native rows count as native work.""" + + called: PlanningCallOutcome = cast(PlanningCallOutcome, outcome) + return ("answered", "raised")[called.raised is not None] + + +def record_native_answers(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count the plans, chains and errors the native glue itself returns for the rest of the test. + + A plan with native error messages counts as `native_with_errors`, any other as + `native_planned`; an error a glue binding raises counts as `native_raised` and under + `native_raised:`. JSON-path planning never reaches these bindings, and errors Python + raises before calling them are never counted. + """ + + answers: Counter[str] = Counter() + plan: Callable[[NativeSqlTestPlanningRequest], tuple[list[NativeSqlTestPlanRow], int, int]] = ( + native_module.plan_compiled_sql_tests + ) + resolve: Callable[[NativeSqlTestChainRequest], list[list[str]]] = ( + native_module.resolve_compiled_sql_test_chains + ) + + def counted_plans( + request: NativeSqlTestPlanningRequest, + ) -> tuple[list[NativeSqlTestPlanRow], int, int]: + response: tuple[list[NativeSqlTestPlanRow], int, int] = _counting_raises( + call=lambda: plan(request), answers=answers + ) + answers.update( + ("native_planned", "native_with_errors")[bool(row[5])] for row in response[0] + ) + return response + + def counted_chains(request: NativeSqlTestChainRequest) -> list[list[str]]: + chains: list[list[str]] = _counting_raises(call=lambda: resolve(request), answers=answers) + answers["native_chains"] += len(chains) + return chains + + monkeypatch.setattr(native_module, "plan_compiled_sql_tests", counted_plans) + monkeypatch.setattr(native_module, "resolve_compiled_sql_test_chains", counted_chains) + return answers + + +def native_raised_kinds(*, answers: Counter[str]) -> frozenset[str]: + """The kinds (message prefixes) of the errors the native glue bindings raised.""" + + return frozenset( + kind.removeprefix(_NATIVE_RAISED_PREFIX) + for kind in filter(lambda kind: kind.startswith(_NATIVE_RAISED_PREFIX), answers) + ) + + +def _counting_raises[T](*, call: Callable[[], T], answers: Counter[str]) -> T: + try: + return call() + except ValueError as error: + answers["native_raised"] += 1 + answers[_NATIVE_RAISED_PREFIX + str(error).partition(_NATIVE_ERROR_KIND_SEPARATOR)[0]] += 1 + raise + + +def _outcome(call: Callable[[], object]) -> PlanningCallOutcome: + try: + return PlanningCallOutcome(value=call(), raised=None) + except Exception as error: # noqa: BLE001 - both paths must raise the same error + return PlanningCallOutcome(value=None, raised=(type(error).__name__, str(error))) diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py new file mode 100644 index 0000000000..c24cf34efd --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py @@ -0,0 +1,173 @@ +"""SQL tests planned natively from compiled objects equal the JSON-request plans, test by test.""" + +from __future__ import annotations + +import random +from collections import Counter +from pathlib import Path + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from tests.integration.src.sqlbuild.compiler.sql_test_glue._test_types import ( + GeneratedSqlTestPlanningParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.sql_test_glue.helpers import ( + INVERTED_WINDOW, + NO_WINDOW, + VALID_WINDOW, + PlanningCallOutcome, + SqlTestCorpusShape, + artifact_outcome, + chain_outcome, + compiled_project, + generated_sql_test_files, + native_raised_kinds, + outcome_kind, + planning_outcome, + record_native_answers, + use_sql_test_glue, + with_unmocked_assertion, +) + +_ADAPTERS: dict[str, BaseAdapter] = { + "duckdb": DuckDbAdapter(), + "postgres": PostgresAdapter(), + "snowflake": SnowflakeAdapter(), + "bigquery": BigQueryAdapter(), + "sqlserver": SqlServerAdapter(), +} + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedSqlTestPlanningParityTestCase( + description="mocks, expected outputs, assertions, helpers, windows and failing tests", + seed=20261009, + count=4, + test_count=12, + shape=SqlTestCorpusShape( + windows=(NO_WINDOW, VALID_WINDOW, INVERTED_WINDOW), + stray_window_share=0.1, + helper_redefinition_share=0.15, + assertion_share=0.2, + unflattenable_assertion_share=0.1, + ), + adapter_names=("duckdb", "postgres", "snowflake", "bigquery", "sqlserver"), + expected_minimum_native_planned=624, + expected_minimum_native_with_errors=144, + expected_minimum_native_chains=398, + expected_minimum_native_raised=102, + expected_native_raised_kinds=frozenset( + {"compile_input", "planner_input", "sql_test_reference"} + ), + expected_minimum_answered_batches=0, + expected_minimum_raised_outcomes=108, + ), + GeneratedSqlTestPlanningParityTestCase( + description="multi-test batches that plan without window or helper errors", + seed=20261010, + count=3, + test_count=12, + shape=SqlTestCorpusShape( + windows=(NO_WINDOW, VALID_WINDOW), + stray_window_share=0.0, + helper_redefinition_share=0.0, + assertion_share=0.3, + unflattenable_assertion_share=0.0, + ), + adapter_names=("duckdb", "postgres", "snowflake", "bigquery", "sqlserver"), + expected_minimum_native_planned=1068, + expected_minimum_native_with_errors=192, + expected_minimum_native_chains=122, + expected_minimum_native_raised=45, + expected_native_raised_kinds=frozenset({"sql_test_reference"}), + expected_minimum_answered_batches=30, + expected_minimum_raised_outcomes=30, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_sql_tests_when_planning_natively_then_plans_match_the_json_request( + test_case: GeneratedSqlTestPlanningParityTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rng: random.Random = random.Random(test_case.seed) + native_answers: Counter[str] = record_native_answers(monkeypatch=monkeypatch) + outcomes: Counter[str] = Counter() + differences: list[tuple[object, object, object]] = [] + for index in range(test_case.count): + project: CompiledProject = compiled_project( + project_dir=tmp_path / f"project_{index}", + files=generated_sql_test_files( + rng=rng, test_count=test_case.test_count, shape=test_case.shape + ), + ) + batches: tuple[tuple[CompiledSqlTest, ...], ...] = ( + project.sql_tests, + *((test,) for test in project.sql_tests), + (with_unmocked_assertion(project=project),), + ) + for adapter_name in test_case.adapter_names: + adapter: BaseAdapter = _ADAPTERS[adapter_name] + for render_sql in (False, True): + inputs: list[object] = [] + expected: list[object] = [] + actual: list[object] = [] + for tests in batches: + use_sql_test_glue(monkeypatch=monkeypatch, enabled=False) + expected.append( + planning_outcome( + project=project, tests=tests, adapter=adapter, render_sql=render_sql + ) + ) + use_sql_test_glue(monkeypatch=monkeypatch, enabled=True) + actual.append( + planning_outcome( + project=project, tests=tests, adapter=adapter, render_sql=render_sql + ) + ) + inputs.append((adapter_name, render_sql, tuple(test.name for test in tests))) + differences.extend(mismatches(inputs=inputs, expected=expected, actual=actual)) + outcomes.update(map(outcome_kind, actual[1:])) + outcomes[f"batch_{outcome_kind(actual[0])}"] += 1 + differences.extend( + mismatches( + inputs=[(adapter_name, "artifacts", tests) for tests in batches], + expected=[ + artifact_outcome(project=project, tests=tests, adapter=adapter, glue=False) + for tests in batches + ], + actual=[ + artifact_outcome(project=project, tests=tests, adapter=adapter, glue=True) + for tests in batches + ], + ) + ) + use_sql_test_glue(monkeypatch=monkeypatch, enabled=False) + python_chains: PlanningCallOutcome = chain_outcome(project=project, tests=project.sql_tests) + use_sql_test_glue(monkeypatch=monkeypatch, enabled=True) + native_chains: PlanningCallOutcome = chain_outcome(project=project, tests=project.sql_tests) + differences.extend( + mismatches(inputs=["chains"], expected=[python_chains], actual=[native_chains]) + ) + + assert ( + differences, + native_answers["native_planned"] >= test_case.expected_minimum_native_planned, + native_answers["native_with_errors"] >= test_case.expected_minimum_native_with_errors, + native_answers["native_chains"] >= test_case.expected_minimum_native_chains, + native_answers["native_raised"] >= test_case.expected_minimum_native_raised, + native_raised_kinds(answers=native_answers) >= test_case.expected_native_raised_kinds, + outcomes["batch_answered"] >= test_case.expected_minimum_answered_batches, + outcomes["raised"] >= test_case.expected_minimum_raised_outcomes, + ) == ([], True, True, True, True, True, True, True), (native_answers, outcomes)