From a2d184b5ae9de99511e43ccc0791d782816b9c63 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:10:08 +0100 Subject: [PATCH 1/5] feat(native): plan SQL tests natively behind the preview engine --- .../src/compiler/_helpers/sql_tests/glue.rs | 31 ++ .../src/compiler/_helpers/sql_tests/mod.rs | 1 + .../compiler/_helpers/sql_tests/planning.rs | 276 ++++---------- .../compiler/_helpers/sql_tests/rendering.rs | 50 +-- .../src/compiler/main/mod.rs | 3 + .../src/compiler/main/sql_test_chains.rs | 9 + .../src/compiler/main/sql_test_glue.rs | 9 + .../src/compiler/main/sql_test_plan_errors.rs | 10 + .../sqlbuild-analysis/src/compiler/models.rs | 179 ++++++++++ .../sqlbuild-analysis/src/compiler/tests.rs | 2 + .../src/compiler/tests/test_sql_test_glue.rs | 217 +++++++++++ .../src/compiler/tests/test_types.rs | 20 ++ .../bindings/_helpers/sql_test_glue/mod.rs | 1 + .../_helpers/sql_test_glue/planning.rs | 304 ++++++++++++++++ .../_helpers/sql_test_glue/registration.rs | 6 +- .../_helpers/analysis_corpus/sql_test_glue.py | 71 +++- src/sqlbuild/_native.pyi | 10 + .../_helpers/sql_tests/native_planning.py | 281 ++++++++++++++- .../_plan_compiled_sql_test_artifacts.py | 25 ++ .../compiler/sql_test_glue/constants.py | 5 + .../main/plan_native_sql_test_artifacts.py | 12 +- src/sqlbuild/compiler/sql_test_glue/models.py | 34 ++ src/sqlbuild/compiler/sql_test_glue/types.py | 24 ++ .../main/compile/sql_test_glue/__init__.py | 0 .../main/compile/sql_test_glue/_test_types.py | 14 + .../main/compile/sql_test_glue/helpers.py | 77 ++++ .../test_native_sql_test_glue_cli.py | 172 +++++++++ .../compile/test_native_analysis_seams.py | 7 +- .../compiler/sql_test_glue/__init__.py | 0 .../compiler/sql_test_glue/_test_types.py | 17 + .../compiler/sql_test_glue/helpers.py | 337 ++++++++++++++++++ .../test_native_sql_test_planning_glue.py | 123 +++++++ 32 files changed, 2061 insertions(+), 266 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/glue.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/main/sql_test_chains.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/main/sql_test_glue.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/main/sql_test_plan_errors.rs create mode 100644 crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_glue.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs create mode 100644 src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py create mode 100644 src/sqlbuild/compiler/sql_test_glue/constants.py create mode 100644 src/sqlbuild/compiler/sql_test_glue/models.py create mode 100644 src/sqlbuild/compiler/sql_test_glue/types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/__init__.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/helpers.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/test_native_sql_test_glue_cli.py create mode 100644 tests/integration/src/sqlbuild/compiler/sql_test_glue/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/glue.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/glue.rs new file mode 100644 index 0000000000..d776c9896b --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/glue.rs @@ -0,0 +1,31 @@ +//! Projections of planned SQL tests into the facts the compiler reports. + +use std::collections::HashSet; + +use crate::compiler::models::SqlTestPlan; + +const ERROR_SEVERITY: &str = "error"; +const DEFAULT_WORKERS: usize = 4; + +/// Each distinct error-severity warning message of one planned test, in warning order. +pub(crate) fn error_messages(plan: &SqlTestPlan) -> Vec { + let mut seen: HashSet<&str> = HashSet::new(); + plan.warnings + .iter() + .filter(|warning| warning.severity == ERROR_SEVERITY) + .filter(|warning| seen.insert(warning.message.as_str())) + .map(|warning| warning.message.clone()) + .collect() +} + +pub(crate) fn default_workers() -> usize { + DEFAULT_WORKERS +} + +pub(crate) fn default_true() -> bool { + true +} + +pub(crate) fn default_set_difference() -> String { + "EXCEPT".to_string() +} diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs index de20c4f5ee..364bfe13c4 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs @@ -4,6 +4,7 @@ pub mod cte_slices; pub mod cte_sql; pub mod expected_columns; pub mod extraction; +pub mod glue; pub mod helper_names; pub mod helper_scope; pub mod markers; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index 3bf4a939ff..e879c9da90 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -7,7 +7,7 @@ use std::time::Instant; use polyglot_sql::{Dialect, Expression}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use regex::Regex; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::{SliceDialect, strip_statement_terminators}; @@ -29,8 +29,12 @@ use crate::compiler::_helpers::sql_tests::reader_references::{ }; use crate::compiler::_helpers::sql_tests::relation_markers::relation_marker_calls; use crate::compiler::_helpers::sql_tests::rendering::{ - AssertionStep, ChainStep, RenderRequest, render_comparison_sql, render_dialect, - rendered_chain_steps, + RenderRequest, render_comparison_sql, render_dialect, rendered_chain_steps, +}; +use crate::compiler::models::{ + SqlTestAssertionStep, SqlTestChainBatch, SqlTestChainStep, SqlTestCte, SqlTestPlan, + SqlTestPlanBatch, SqlTestPlanBatchOutcome, SqlTestPlanFunction, SqlTestPlanPayload, + SqlTestPlanTest, SqlTestPlanWarning, }; use crate::constants::{ SQL_TEST_ACTUAL_CTE, SQL_TEST_ACTUAL_CTE_PREFIX, SQL_TEST_EXPECTED_CTE, @@ -38,7 +42,6 @@ use crate::constants::{ }; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -const DEFAULT_WORKERS: usize = 4; const MAX_WORKERS: usize = 4; const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; pub(crate) const REF_PREFIX: &str = "__ref__"; @@ -62,160 +65,22 @@ pub(crate) const DBT_REF_FUNCTION: &str = "__dbt_ref"; const UDF_FUNCTION: &str = "__udf"; const TABLE_FUNCTION: &str = "__table_fn"; -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct PlanBatchRequest { - models: Vec, - #[serde(default)] - functions: Vec, - tests: Vec, - #[serde(default = "default_true")] - sql_analysis_enabled: bool, - #[serde(default)] - sql_analysis_dialect: Option, - #[serde(default = "default_set_difference")] - set_difference_operator: String, - #[serde(default)] - requires_derived_table_aliases: bool, - #[serde(default = "default_workers")] - workers: usize, - #[serde(default = "default_true")] - render_sql: bool, - #[serde(default = "default_true")] - include_plan: bool, - lexical_syntax: LexicalSyntax, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ChainBatchRequest { - models: Vec, - tests: Vec, - lexical_syntax: LexicalSyntax, -} - -/// Chain ordering reads only declared dependencies, so chain requests omit model SQL. -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ChainModelInput { - name: String, - #[serde(default)] - model_dependencies: Vec, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct ChainBatchResponse { chains: Vec>, } -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ModelInput { - name: String, - query_sql: String, - #[serde(default)] - model_dependencies: Vec, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct FunctionInput { - name: String, - #[serde(default)] - udf_prefix: Option, - #[serde(default)] - udf_suffix: Option, - #[serde(default)] - table_function_prefix: Option, - #[serde(default)] - table_function_suffix: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct TestInput { - name: String, - file_label: String, - payload: TestPayload, -} - -#[derive(Debug, Deserialize)] -#[serde( - tag = "kind", - rename_all = "camelCase", - rename_all_fields = "camelCase" -)] -enum TestPayload { - Model { - #[serde(default)] - authored_ctes: Vec, - #[serde(default)] - model_query_overrides: BTreeMap, - #[serde(default)] - expected_ctes: Vec, - #[serde(default)] - expected_model_names: Vec, - #[serde(default)] - assertion_ctes: Vec, - #[serde(default)] - read_helper_names: Option>, - #[serde(default)] - reference_target_model_names: Option>, - }, - Direct { - mode: String, - actual_cte: CteInput, - expected_cte: CteInput, - #[serde(default)] - helper_ctes: Vec, - }, -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct CteInput { - pub(crate) name: String, - pub(crate) sql_body: String, -} - -/// One planned SQL test: the executable chain and assertion steps plus optional rendered SQL. -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct PlanResponse { - sql: Option, - chain: Vec, - assertions: Vec, - model_names: Vec, - warnings: Vec, -} - struct PlannedResponse { request: RenderRequest, model_names: Vec, - warnings: Vec, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct PlanBatchResponse { - artifacts: Vec, - planning_ns: u64, - rendering_ns: u64, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct PlanWarning { - model_name: Option, - severity: &'static str, - message: String, + warnings: Vec, } #[derive(Clone)] struct ProjectContext { models: HashMap, - functions: HashMap, + functions: HashMap, sql_analysis_enabled: bool, dialect: String, set_difference_operator: String, @@ -286,7 +151,7 @@ pub(crate) struct TestFixtures { pub(crate) mock_seeds: BTreeMap, pub(crate) mock_dbt_refs: BTreeMap, pub(crate) mock_table_functions: BTreeMap, - pub(crate) helpers: Vec, + pub(crate) helpers: Vec, pub(crate) scope: ScopeGraph, pub(crate) expected: BTreeMap, pub(crate) assertions: Vec<(String, String)>, @@ -295,19 +160,19 @@ pub(crate) struct TestFixtures { struct DirectTestPlan { name: String, mode: String, - actual_cte: CteInput, - expected_cte: CteInput, - helpers: Vec, + actual_cte: SqlTestCte, + expected_cte: SqlTestCte, + helpers: Vec, } struct ModelTestPlan { test_name: String, file_label: String, - authored_ctes: Vec, + authored_ctes: Vec, model_query_overrides: BTreeMap, - expected_ctes: Vec, + expected_ctes: Vec, expected_model_names: Vec, - assertion_ctes: Vec, + assertion_ctes: Vec, reads: CompilerReads, } @@ -441,7 +306,7 @@ struct AssertionResolutionRequest<'a> { assertion_sql: &'a str, fixtures: &'a TestFixtures, chain: &'a TextualChain, - functions: &'a HashMap, + functions: &'a HashMap, requires_flat_ctes: bool, patterns: &'a SqlTestPatterns, } @@ -451,7 +316,7 @@ struct AnalysisResolutionRequest<'a> { fixture_ctes: &'a [(String, String)], fixtures: &'a TestFixtures, resolved_chain: &'a HashMap, - functions: &'a HashMap, + functions: &'a HashMap, file_label: &'a str, dialect_name: &'a str, templates: &'a AnalysisTemplateCache, @@ -463,7 +328,7 @@ struct TextualResolutionRequest<'a> { fixtures: &'a TestFixtures, resolved_chain: &'a HashMap, chain_references: Option<&'a mut Vec>, - functions: &'a HashMap, + functions: &'a HashMap, patterns: &'a SqlTestPatterns, } @@ -562,8 +427,14 @@ impl GeneratedCteState { /// Return each test's topologically ordered unmocked model chain without planning SQL. pub(crate) fn resolve_chains_json(request_json: &str) -> Result { - let request: ChainBatchRequest = + let request: SqlTestChainBatch = serde_json::from_str(request_json).map_err(|error| error.to_string())?; + let chains: Vec> = resolve_chains(request)?; + serde_json::to_string(&ChainBatchResponse { chains }).map_err(|error| error.to_string()) +} + +/// Each test's topologically ordered unmocked model chain, in request order. +pub(crate) fn resolve_chains(request: SqlTestChainBatch) -> Result>, String> { let patterns = SqlTestPatterns::new(request.lexical_syntax)?; let models: HashMap = request .models @@ -582,8 +453,8 @@ pub(crate) fn resolve_chains_json(request_json: &str) -> Result .tests .into_iter() .map(|test| match test.payload { - TestPayload::Direct { .. } => Ok(Vec::new()), - TestPayload::Model { + SqlTestPlanPayload::Direct { .. } => Ok(Vec::new()), + SqlTestPlanPayload::Model { authored_ctes, model_query_overrides, expected_ctes, @@ -610,7 +481,7 @@ pub(crate) fn resolve_chains_json(request_json: &str) -> Result } }) .collect::>()?; - serde_json::to_string(&ChainBatchResponse { chains }).map_err(|error| error.to_string()) + Ok(chains) } /// Models the test runs: expected models, assertion targets and the compiler's reference targets. @@ -626,8 +497,13 @@ fn chain_root_names( } pub(crate) fn plan_and_render_json(request_json: &str) -> Result { - let request: PlanBatchRequest = + let request: SqlTestPlanBatch = serde_json::from_str(request_json).map_err(|error| error.to_string())?; + serde_json::to_string(&plan_batch(request)?).map_err(|error| error.to_string()) +} + +/// Plan every test of `request` on a bounded pool, keeping request order. +pub(crate) fn plan_batch(request: SqlTestPlanBatch) -> Result { let render_sql = request.render_sql; let include_plan = request.include_plan; let render_dialect = Arc::new(render_dialect(request.sql_analysis_dialect.as_deref())); @@ -670,7 +546,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result .build() .map_err(|error| error.to_string())?; let batch_start = Instant::now(); - let responses: Vec> = pool.install(|| { + let responses: Vec> = pool.install(|| { request .tests .into_par_iter() @@ -685,7 +561,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result let sql = match rendered { Some(Ok(sql)) => render_sql.then_some(sql), Some(Err(message)) => { - planned.warnings.push(PlanWarning { + planned.warnings.push(SqlTestPlanWarning { model_name: None, severity: "error", message: format!("test '{test_name}' cannot be rendered: {message}"), @@ -698,7 +574,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result planned.request.chain.clear(); planned.request.assertions.clear(); } - let response = PlanResponse { + let response = SqlTestPlan { sql, chain: planned.request.chain, assertions: planned.request.assertions, @@ -709,7 +585,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result }) .collect() }); - let responses: Vec<(PlanResponse, u128, u128)> = + let responses: Vec<(SqlTestPlan, u128, u128)> = responses.into_iter().collect::>()?; let batch_ns = batch_start.elapsed().as_nanos(); let planning_cpu_ns: u128 = responses.iter().map(|(_, value, _)| *value).sum(); @@ -720,21 +596,20 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result .checked_div(measured_cpu_ns) .unwrap_or(batch_ns); let rendering_ns = batch_ns.saturating_sub(planning_ns); - let artifacts: Vec = responses + let artifacts: Vec = responses .into_iter() .map(|(response, _, _)| response) .collect(); - serde_json::to_string(&PlanBatchResponse { - artifacts, + Ok(SqlTestPlanBatchOutcome { + plans: artifacts, planning_ns: planning_ns.min(u128::from(u64::MAX)) as u64, rendering_ns: rendering_ns.min(u128::from(u64::MAX)) as u64, }) - .map_err(|error| error.to_string()) } -fn plan_test(test: TestInput, context: &ProjectContext) -> Result { +fn plan_test(test: SqlTestPlanTest, context: &ProjectContext) -> Result { match test.payload { - TestPayload::Direct { + SqlTestPlanPayload::Direct { mode, actual_cte, expected_cte, @@ -749,7 +624,7 @@ fn plan_test(test: TestInput, context: &ProjectContext) -> Result = Vec::new(); + let mut warnings: Vec = Vec::new(); let mut reported_missing_mocks: HashSet<(&'static str, String)> = HashSet::new(); let mut reachable_mocks: HashSet = HashSet::new(); let mut analysis_resolved: HashMap = HashMap::new(); let mut textual_chain: TextualChain = TextualChain::default(); - let mut chain: Vec = Vec::new(); + let mut chain: Vec = Vec::new(); for (model_index, model_name) in ordered_names.iter().enumerate() { let Some(model) = context.models.get(model_name) else { - warnings.push(PlanWarning { + warnings.push(SqlTestPlanWarning { model_name: None, severity: "error", message: format!( @@ -953,7 +828,7 @@ fn plan_model_test( } None => Vec::new(), }; - chain.push(ChainStep { + chain.push(SqlTestChainStep { model_name: model_name.clone(), resolved_sql, expected_columns: authored_expected @@ -983,7 +858,7 @@ fn plan_model_test( reachable_mocks: &mut reachable_mocks, })?; - let mut assertions: Vec = Vec::new(); + let mut assertions: Vec = Vec::new(); for (assertion_name, assertion_sql) in &fixtures.assertions { let placed_sql = fixtures.scope.reader_sql(assertion_sql, &context.patterns); let resolved = readers.resolve( @@ -1013,7 +888,7 @@ fn plan_model_test( comparison_body_sql.as_deref().unwrap_or(&resolved_sql), ); } - assertions.push(AssertionStep { + assertions.push(SqlTestAssertionStep { name: assertion_name.clone(), resolved_sql, lifted_ctes, @@ -1047,7 +922,7 @@ struct HelperResolution<'r, 'a> { readers: &'r mut ReaderContext<'a>, fixtures: &'r mut TestFixtures, read_helper_names: &'r [String], - chain: &'r mut [ChainStep], + chain: &'r mut [SqlTestChainStep], reachable_mocks: &'r mut HashSet, } @@ -1237,7 +1112,10 @@ impl ReaderContext<'_> { } /// Drop SQL from chain steps the renderer never emits, keeping plan output linear in chain length. -fn omit_unrendered_step_sql(chain: Vec, assertions: &[AssertionStep]) -> Vec { +fn omit_unrendered_step_sql( + chain: Vec, + assertions: &[SqlTestAssertionStep], +) -> Vec { let rendered_steps = rendered_chain_steps(&chain, assertions); chain .into_iter() @@ -1246,7 +1124,7 @@ fn omit_unrendered_step_sql(chain: Vec, assertions: &[AssertionStep]) if rendered { step } else { - ChainStep { + SqlTestChainStep { model_name: step.model_name, resolved_sql: String::new(), expected_cte_sql: step.expected_cte_sql, @@ -1350,9 +1228,9 @@ fn resolve_assertion_textual_sql( } fn classify_fixtures( - authored: Vec, - expected: Vec, - assertions: Vec, + authored: Vec, + expected: Vec, + assertions: Vec, ) -> TestFixtures { let mut fixtures = TestFixtures { mock_refs: BTreeMap::new(), @@ -1760,7 +1638,7 @@ fn resolve_table_function_fixtures( fn resolve_function_calls( sql: &str, - functions: &HashMap, + functions: &HashMap, table_function: bool, patterns: &SqlTestPatterns, ) -> Result { @@ -1836,7 +1714,9 @@ struct UnresolvedReferenceRequest<'a> { } /// Report each unmocked reference once per test, attributed to the first step reaching it. -fn unresolved_reference_warnings(request: UnresolvedReferenceRequest<'_>) -> Vec { +fn unresolved_reference_warnings( + request: UnresolvedReferenceRequest<'_>, +) -> Vec { let UnresolvedReferenceRequest { sql, test_name, @@ -1847,10 +1727,10 @@ fn unresolved_reference_warnings(request: UnresolvedReferenceRequest<'_>) -> Vec if !patterns.test_reference.is_match(sql) { return Vec::new(); } - let mut warnings: Vec = Vec::new(); + let mut warnings: Vec = Vec::new(); let mut warn = |kind: &'static str, name: String, message: String| { if reported.insert((kind, name)) { - warnings.push(PlanWarning { + warnings.push(SqlTestPlanWarning { model_name: Some(model_name.to_string()), severity: "error", message, @@ -1901,7 +1781,7 @@ fn unreachable_mock_warnings( test_name: &str, reachable: &HashSet, fixtures: &TestFixtures, -) -> Vec { +) -> Vec { let groups = [ ( &fixtures.mock_refs, @@ -1917,10 +1797,10 @@ fn unreachable_mock_warnings( " is unreachable", ), ]; - let mut warnings: Vec = Vec::new(); + let mut warnings: Vec = Vec::new(); for (mocks, prefix, suffix) in groups { for name in mocks.keys().filter(|name| !reachable.contains(*name)) { - warnings.push(PlanWarning { + warnings.push(SqlTestPlanWarning { model_name: None, severity: "warning", message: format!("test '{test_name}' mock {prefix}{name}{suffix}"), @@ -1941,7 +1821,7 @@ fn has_unresolved_test_reference(sql: &str, patterns: &SqlTestPatterns) -> bool || patterns.table_function.is_match(sql) } -fn helper_with_clause(helpers: &[CteInput]) -> String { +fn helper_with_clause(helpers: &[SqlTestCte]) -> String { if helpers.is_empty() { return String::new(); } @@ -1956,7 +1836,7 @@ fn helper_with_clause(helpers: &[CteInput]) -> String { } /// Prefix every helper CTE to a direct test's actual and expected queries. -fn with_helper_ctes(sql: &str, helpers: &[CteInput]) -> String { +fn with_helper_ctes(sql: &str, helpers: &[SqlTestCte]) -> String { if helpers.is_empty() { sql.to_string() } else { @@ -1975,18 +1855,6 @@ fn compile_pattern(pattern: &str) -> Result { .map_err(|error| planner_error(&format!("invalid SQL-test pattern: {error}"))) } -fn default_workers() -> usize { - DEFAULT_WORKERS -} - -fn default_true() -> bool { - true -} - -fn default_set_difference() -> String { - "EXCEPT".to_string() -} - pub(crate) fn compile_error(message: &str) -> String { format!("compile_input:{message}") } diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs index 72a24956ab..efc1b7d5f2 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/rendering.rs @@ -13,6 +13,7 @@ use crate::compiler::_helpers::sql_tests::cte_slices::{ use crate::compiler::_helpers::sql_tests::cte_sql::{cte_definition_sql, leading_with_prefix_end}; use crate::compiler::_helpers::sql_tests::helper_names::HELPER_PREFIX; use crate::compiler::_helpers::sql_tests::planning::REF_PREFIX; +use crate::compiler::models::{SqlTestAssertionStep, SqlTestChainStep}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; @@ -32,7 +33,7 @@ struct RenderBatchRequest { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct DifferenceSampleRequest { - step: ChainStep, + step: SqlTestChainStep, #[serde(default = "default_true")] sql_analysis_enabled: bool, #[serde(default = "default_set_difference")] @@ -56,9 +57,9 @@ enum DifferenceDirection { #[serde(rename_all = "camelCase")] pub(crate) struct RenderRequest { #[serde(default)] - pub(crate) chain: Vec, + pub(crate) chain: Vec, #[serde(default)] - pub(crate) assertions: Vec, + pub(crate) assertions: Vec, #[serde(default = "default_true")] pub(crate) sql_analysis_enabled: bool, #[serde(default = "default_set_difference")] @@ -69,34 +70,6 @@ pub(crate) struct RenderRequest { pub(crate) probe_step_index: Option, } -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct ChainStep { - pub(crate) model_name: String, - pub(crate) resolved_sql: String, - #[serde(default)] - pub(crate) expected_cte_sql: Option, - #[serde(default)] - pub(crate) expected_lifted_ctes: Vec<(String, String)>, - #[serde(default)] - pub(crate) lifted_ctes: Vec<(String, String)>, - #[serde(default)] - pub(crate) comparison_body_sql: Option, - #[serde(default)] - pub(crate) expected_columns: Option>, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub(crate) struct AssertionStep { - pub(crate) name: String, - pub(crate) resolved_sql: String, - #[serde(default)] - pub(crate) lifted_ctes: Vec<(String, String)>, - #[serde(default)] - pub(crate) comparison_body_sql: Option, -} - #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct RenderResponse { @@ -202,7 +175,7 @@ impl RenderCteState { /// Define a step's inputs and its model once at top level, the model as a nested `__ref__` CTE. fn actual_step_sql( &mut self, - step: &ChainStep, + step: &SqlTestChainStep, suffix: &str, enabled: bool, ) -> Result { @@ -225,7 +198,7 @@ impl RenderCteState { /// Place an expected step's helper CTEs at top level and return its comparison body. fn expected_step_sql( &mut self, - step: &ChainStep, + step: &SqlTestChainStep, expected_sql: &str, enabled: bool, ) -> Result { @@ -237,7 +210,7 @@ impl RenderCteState { /// Place an assertion's helper CTEs at top level and return its comparison body. fn assertion_sql( &mut self, - assertion: &AssertionStep, + assertion: &SqlTestAssertionStep, enabled: bool, ) -> Result { let origin = format!("assertion '{}'", assertion.name); @@ -575,7 +548,10 @@ impl CteSuffixCounter { } /// Mark the chain steps whose actual SQL the comparison renderer emits. -pub(crate) fn rendered_chain_steps(chain: &[ChainStep], assertions: &[AssertionStep]) -> Vec { +pub(crate) fn rendered_chain_steps( + chain: &[SqlTestChainStep], + assertions: &[SqlTestAssertionStep], +) -> Vec { let mut suffixes = CteSuffixCounter::default(); chain .iter() @@ -666,7 +642,7 @@ fn render_difference_sample_sql(request: &DifferenceSampleRequest) -> Result String { +fn compared_projection(step: &SqlTestChainStep) -> String { step.expected_columns .as_ref() .map_or_else(|| "*".to_string(), |columns| columns.join(", ")) @@ -789,7 +765,7 @@ fn sanitize_cte_suffix(model_name: &str) -> String { } } -fn assertions_use_actual(assertions: &[AssertionStep], actual_cte: &str) -> bool { +fn assertions_use_actual(assertions: &[SqlTestAssertionStep], actual_cte: &str) -> bool { let needle = actual_cte.to_ascii_lowercase(); assertions.iter().any(|assertion| { assertion diff --git a/crates/sqlbuild-analysis/src/compiler/main/mod.rs b/crates/sqlbuild-analysis/src/compiler/main/mod.rs index 0267d8f76c..357fadc9d0 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/mod.rs @@ -1,7 +1,10 @@ pub mod sql_scenario_extraction; pub mod sql_test_chain_resolution; +pub mod sql_test_chains; pub mod sql_test_difference_sampling; pub mod sql_test_extraction; pub mod sql_test_fixture_facts; +pub mod sql_test_glue; +pub mod sql_test_plan_errors; pub mod sql_test_planning; pub mod sql_test_rendering; diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_test_chains.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_chains.rs new file mode 100644 index 0000000000..c54b8b3e32 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_chains.rs @@ -0,0 +1,9 @@ +//! SQL-test model chains for compiled projects, from a typed request. + +use crate::compiler::_helpers::sql_tests::planning; +use crate::compiler::models::SqlTestChainBatch; + +/// Each test's ordered unmocked model chain; direct-logic tests have none. +pub fn resolve_sql_test_chains(request: SqlTestChainBatch) -> Result>, String> { + planning::resolve_chains(request) +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_test_glue.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_glue.rs new file mode 100644 index 0000000000..972ab2a837 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_glue.rs @@ -0,0 +1,9 @@ +//! SQL-test planning for compiled projects, from a typed request. + +use crate::compiler::_helpers::sql_tests::planning; +use crate::compiler::models::{SqlTestPlanBatch, SqlTestPlanBatchOutcome}; + +/// Plan every test of `request` in request order; errors carry the planner's message prefixes. +pub fn plan_sql_test_batch(request: SqlTestPlanBatch) -> Result { + planning::plan_batch(request) +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_test_plan_errors.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_test_plan_errors.rs new file mode 100644 index 0000000000..115be96298 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_test_plan_errors.rs @@ -0,0 +1,10 @@ +//! The errors a planned SQL test reports. + +use crate::compiler::_helpers::sql_tests::glue; +use crate::compiler::models::SqlTestPlan; + +/// Each distinct error message of one planned test, in the order its warnings report them. +#[must_use] +pub fn sql_test_plan_error_messages(plan: &SqlTestPlan) -> Vec { + glue::error_messages(plan) +} diff --git a/crates/sqlbuild-analysis/src/compiler/models.rs b/crates/sqlbuild-analysis/src/compiler/models.rs index bef142348d..03e40d082d 100644 --- a/crates/sqlbuild-analysis/src/compiler/models.rs +++ b/crates/sqlbuild-analysis/src/compiler/models.rs @@ -1,5 +1,184 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct SqlTestFixtureFacts { pub mock: bool, pub empty_fixture_marker: bool, } + +/// One batch of SQL tests to plan against the project's models and functions. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanBatch { + pub models: Vec, + #[serde(default)] + pub functions: Vec, + pub tests: Vec, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_true")] + pub sql_analysis_enabled: bool, + #[serde(default)] + pub sql_analysis_dialect: Option, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_set_difference")] + pub set_difference_operator: String, + #[serde(default)] + pub requires_derived_table_aliases: bool, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_workers")] + pub workers: usize, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_true")] + pub render_sql: bool, + #[serde(default = "crate::compiler::_helpers::sql_tests::glue::default_true")] + pub include_plan: bool, + pub lexical_syntax: LexicalSyntax, +} + +/// The tests whose model chains to resolve; chain ordering reads only declared dependencies. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestChainBatch { + pub models: Vec, + pub tests: Vec, + pub lexical_syntax: LexicalSyntax, +} + +/// One project model's name and the models it depends on. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestChainModel { + pub name: String, + #[serde(default)] + pub model_dependencies: Vec, +} + +/// One project model as SQL tests read it: its SQL and the models it depends on. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanModel { + pub name: String, + pub query_sql: String, + #[serde(default)] + pub model_dependencies: Vec, +} + +/// One project function and the adapter's call templates around its arguments. +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanFunction { + pub name: String, + #[serde(default)] + pub udf_prefix: Option, + #[serde(default)] + pub udf_suffix: Option, + #[serde(default)] + pub table_function_prefix: Option, + #[serde(default)] + pub table_function_suffix: Option, +} + +/// One compiled SQL test to plan. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanTest { + pub name: String, + pub file_label: String, + pub payload: SqlTestPlanPayload, +} + +/// A model-mode test's CTEs and overrides, or a direct-logic test's actual and expected CTEs. +#[derive(Debug, Deserialize)] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum SqlTestPlanPayload { + Model { + #[serde(default)] + authored_ctes: Vec, + #[serde(default)] + model_query_overrides: BTreeMap, + #[serde(default)] + expected_ctes: Vec, + #[serde(default)] + expected_model_names: Vec, + #[serde(default)] + assertion_ctes: Vec, + #[serde(default)] + read_helper_names: Option>, + #[serde(default)] + reference_target_model_names: Option>, + }, + Direct { + mode: String, + actual_cte: SqlTestCte, + expected_cte: SqlTestCte, + #[serde(default)] + helper_ctes: Vec, + }, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestCte { + pub name: String, + pub sql_body: String, +} + +/// One planned SQL test: the executable chain and assertion steps plus optional rendered SQL. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlan { + pub sql: Option, + pub chain: Vec, + pub assertions: Vec, + pub model_names: Vec, + pub warnings: Vec, +} + +/// Every planned test in request order, with the batch's wall time split by phase. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanBatchOutcome { + #[serde(rename = "artifacts")] + pub plans: Vec, + pub planning_ns: u64, + pub rendering_ns: u64, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestPlanWarning { + pub model_name: Option, + pub severity: &'static str, + pub message: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestChainStep { + pub model_name: String, + pub resolved_sql: String, + #[serde(default)] + pub expected_cte_sql: Option, + #[serde(default)] + pub expected_lifted_ctes: Vec<(String, String)>, + #[serde(default)] + pub lifted_ctes: Vec<(String, String)>, + #[serde(default)] + pub comparison_body_sql: Option, + #[serde(default)] + pub expected_columns: Option>, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SqlTestAssertionStep { + pub name: String, + pub resolved_sql: String, + #[serde(default)] + pub lifted_ctes: Vec<(String, String)>, + #[serde(default)] + pub comparison_body_sql: Option, +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index cc4183b024..ea9f705748 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -12,6 +12,8 @@ mod sql_test_expected_columns; mod sql_test_extraction; #[path = "tests/test_sql_test_extraction_rules.rs"] mod sql_test_extraction_rules; +#[path = "tests/test_sql_test_glue.rs"] +mod sql_test_glue; #[path = "tests/test_sql_test_helper_references.rs"] mod sql_test_helper_references; #[path = "tests/test_sql_test_markers.rs"] diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_glue.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_glue.rs new file mode 100644 index 0000000000..3976f6d8e0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_glue.rs @@ -0,0 +1,217 @@ +use serde_json::{Value, json}; + +use crate::compiler::main::sql_test_chain_resolution::resolve_chains_json; +use crate::compiler::main::sql_test_chains::resolve_sql_test_chains; +use crate::compiler::main::sql_test_glue::plan_sql_test_batch; +use crate::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; +use crate::compiler::main::sql_test_planning::plan_and_render_json; +use crate::compiler::models::{SqlTestPlan, SqlTestPlanWarning}; +use crate::compiler::tests::test_types::{ + GlueErrorProjectionTestCase, GluePlanParityTestCase, GluePlannerErrorTestCase, +}; + +/// A mocked model test, a test missing its source mock and a UDF test over two models. +const GLUE_REQUEST: &str = r#"{ + "lexicalSyntax": { + "backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, + "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"] + }, + "models": [ + {"name": "stg_orders", "querySql": "SELECT order_id, amount FROM __source(\"raw_orders\")"}, + { + "name": "orders", + "querySql": "WITH big AS (SELECT * FROM __ref(\"stg_orders\") WHERE __udf(\"udf__is_big\")(amount)) SELECT order_id FROM big", + "modelDependencies": ["stg_orders"] + } + ], + "functions": [{ + "name": "udf__is_big", "udfPrefix": "main.udf__is_big(", "udfSuffix": ")", + "tableFunctionPrefix": "main.udf__is_big(", "tableFunctionSuffix": ")" + }], + "tests": [ + { + "name": "mocked", "fileLabel": "tests/mocked.sql", + "payload": { + "kind": "model", + "authoredCtes": [ + {"name": "__source__raw_orders", "sqlBody": "SELECT 1 AS order_id, 5 AS amount"}, + {"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"} + ], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], + "expectedModelNames": ["orders"], + "assertionCtes": [{ + "name": "__assert__orders_exist", + "sqlBody": "SELECT * FROM __ref(\"orders\") WHERE 1 = 0" + }], + "readHelperNames": [], "referenceTargetModelNames": [] + } + }, + { + "name": "missing_mock", "fileLabel": "tests/missing_mock.sql", + "payload": { + "kind": "model", + "authoredCtes": [{"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"}], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], + "expectedModelNames": ["orders"], + "readHelperNames": [], "referenceTargetModelNames": [] + } + }, + { + "name": "udf_case", "fileLabel": "tests/udf_case.sql", + "payload": { + "kind": "direct", "mode": "udf", + "actualCte": { + "name": "__udf_actual__", "sqlBody": "SELECT __udf(\"udf__is_big\")(150) AS is_big" + }, + "expectedCte": {"name": "__udf_expected__", "sqlBody": "SELECT TRUE AS is_big"} + } + } + ] +}"#; + +#[test] +fn given_typed_requests_when_planning_then_plans_and_chains_equal_the_json_entry_points() { + let test_cases = [ + GluePlanParityTestCase { + description: "generic SQL, planned without rendering", + dialect: None, + render_sql: false, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "DuckDB, rendered", + dialect: Some("duckdb"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "Snowflake, rendered", + dialect: Some("snowflake"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "SQL Server rejects nested WITH and renders anyway", + dialect: Some("tsql"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + ]; + let without_timings = |mut value: Value| { + for key in ["planningNs", "renderingNs"] { + let _ = value.as_object_mut().map(|object| object.remove(key)); + } + value + }; + for test_case in test_cases { + let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); + request["sqlAnalysisDialect"] = json!(test_case.dialect); + request["renderSql"] = json!(test_case.render_sql); + request["includePlan"] = json!(!test_case.render_sql); + let typed = plan_sql_test_batch(serde_json::from_value(request.clone()).expect("typed")) + .expect("typed plans"); + let typed: Value = without_timings(serde_json::to_value(&typed).expect("serializable")); + let json: Value = without_timings( + serde_json::from_str(&plan_and_render_json(&request.to_string()).expect("plans")) + .expect("JSON plans"), + ); + let typed_chains = resolve_sql_test_chains( + serde_json::from_value(request.clone()).expect("typed chains request"), + ) + .expect("typed chains"); + let json_chains: Value = + serde_json::from_str(&resolve_chains_json(&request.to_string()).expect("chains")) + .expect("JSON chains"); + assert_eq!( + ( + typed["artifacts"].as_array().map(Vec::len), + typed["artifacts"][1]["warnings"][0]["severity"].clone(), + typed == json, + json!({"chains": typed_chains}) == json_chains, + ), + ( + Some(test_case.expected_plan_count), + json!(test_case.expected_missing_mock_severity), + true, + true, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_test_without_compiler_reads_when_planning_then_the_typed_batch_returns_its_error() { + let test_cases = [ + GluePlannerErrorTestCase { + description: "helper reads are required for model tests", + payload_field: "readHelperNames", + expected_error_prefix: "planner_input:", + }, + GluePlannerErrorTestCase { + description: "reference targets are required for model tests", + payload_field: "referenceTargetModelNames", + expected_error_prefix: "planner_input:", + }, + ]; + for test_case in test_cases { + let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); + request["tests"][0]["payload"][test_case.payload_field] = json!(null); + let error = plan_sql_test_batch(serde_json::from_value(request).expect("typed")).err(); + assert_eq!( + error.map(|message| message.starts_with(test_case.expected_error_prefix)), + Some(true), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_plan_warnings_when_projecting_errors_then_each_distinct_error_message_is_kept_in_order() { + let test_cases = [ + GlueErrorProjectionTestCase { + description: "warnings are not errors", + warnings: &[("warning", "mock is unreachable"), ("info", "note")], + expected_messages: &[], + }, + GlueErrorProjectionTestCase { + description: "repeated errors are reported once, in first-seen order", + warnings: &[ + ("error", "b has no mock"), + ("warning", "a is unreachable"), + ("error", "a has no mock"), + ("error", "b has no mock"), + ], + expected_messages: &["b has no mock", "a has no mock"], + }, + ]; + for test_case in test_cases { + let plan = SqlTestPlan { + sql: None, + chain: Vec::new(), + assertions: Vec::new(), + model_names: Vec::new(), + warnings: test_case + .warnings + .iter() + .map(|(severity, message)| SqlTestPlanWarning { + model_name: None, + severity, + message: (*message).to_owned(), + }) + .collect(), + }; + assert_eq!( + sql_test_plan_error_messages(&plan), + test_case.expected_messages.to_vec(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index d83ddeb0f8..72cd3687f2 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -165,3 +165,23 @@ pub(crate) struct AuthoredCtesTestCase { pub(crate) sql: &'static str, pub(crate) expected_response: serde_json::Value, } + +pub(crate) struct GluePlanParityTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: Option<&'static str>, + pub(crate) render_sql: bool, + pub(crate) expected_plan_count: usize, + pub(crate) expected_missing_mock_severity: &'static str, +} + +pub(crate) struct GluePlannerErrorTestCase { + pub(crate) description: &'static str, + pub(crate) payload_field: &'static str, + pub(crate) expected_error_prefix: &'static str, +} + +pub(crate) struct GlueErrorProjectionTestCase { + pub(crate) description: &'static str, + pub(crate) warnings: &'static [(&'static str, &'static str)], + pub(crate) expected_messages: &'static [&'static str], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs index 7444bd0374..8be67c2871 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -1 +1,2 @@ +pub(crate) mod planning; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs new file mode 100644 index 0000000000..43b10157f0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -0,0 +1,304 @@ +//! Plan compiled SQL tests from the compiled project's own objects, without a JSON request. + +use std::collections::{BTreeMap, HashMap}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiler::main::sql_test_chains::resolve_sql_test_chains; +use sqlbuild_analysis::compiler::main::sql_test_glue::plan_sql_test_batch; +use sqlbuild_analysis::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; +use sqlbuild_analysis::compiler::models::{ + SqlTestAssertionStep, SqlTestChainBatch, SqlTestChainModel, SqlTestChainStep, SqlTestCte, + SqlTestPlan, SqlTestPlanBatch, SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, + SqlTestPlanTest, +}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; +use crate::bindings::types::CompilerDetach; + +const MODEL_RESOURCE_TYPE: &str = "model"; +const PLANNING_WORKERS: usize = 4; + +type CtePair = (String, String); +/// `(model, SQL, expected SQL, lifted CTEs, body, expected columns, expected lifted CTEs)`. +type ChainRow = ( + String, + String, + Option, + Vec, + Option, + Option>, + Vec, +); +/// `(name, resolved SQL, lifted CTEs, comparison body)`. +type AssertionRow = (String, String, Vec, Option); +/// `(model, severity, message)`. +type WarningRow = (Option, &'static str, String); +/// `(SQL, chain, assertions, model names, warnings, distinct error messages)`. +type PlanRow = ( + Option, + Vec, + Vec, + Vec, + Vec, + Vec, +); + +/// The Python planning request: compiled models and tests, with the adapter's facts. +#[derive(FromPyObject)] +struct PlanningRequest<'py> { + #[pyo3(attribute)] + models: Vec>, + /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. + #[pyo3(attribute)] + rendered_model_sql: HashMap, + #[pyo3(attribute)] + functions: Vec<(String, String, String, String, String)>, + #[pyo3(attribute)] + tests: Vec>, + /// Cursor-rendered model query overrides by test index, replacing the test's own. + #[pyo3(attribute)] + rendered_test_overrides: HashMap>, + #[pyo3(attribute)] + sql_analysis_enabled: bool, + #[pyo3(attribute)] + sql_analysis_dialect: Option, + #[pyo3(attribute)] + set_difference_operator: String, + #[pyo3(attribute)] + requires_derived_table_aliases: bool, + #[pyo3(attribute)] + lexical_syntax: LexicalSyntaxInput, + #[pyo3(attribute)] + render_sql: bool, + #[pyo3(attribute)] + include_plan: bool, +} + +/// The Python chain request: compiled models and the tests whose chains to resolve. +#[derive(FromPyObject)] +struct ChainRequest<'py> { + #[pyo3(attribute)] + models: Vec>, + #[pyo3(attribute)] + tests: Vec>, + #[pyo3(attribute)] + lexical_syntax: LexicalSyntaxInput, +} + +/// Each test's ordered unmocked model chain; direct-logic tests have none. +#[pyfunction] +fn resolve_compiled_sql_test_chains( + py: Python<'_>, + request: ChainRequest<'_>, +) -> PyResult>> { + let batch = SqlTestChainBatch { + models: request + .models + .iter() + .map(|model| { + Ok(SqlTestChainModel { + name: model.getattr("name")?.extract()?, + model_dependencies: model_dependencies(model)?, + }) + }) + .collect::>()?, + tests: request + .tests + .iter() + .map(|test| plan_test(test, None)) + .collect::>()?, + lexical_syntax: request.lexical_syntax.into(), + }; + py.compiler_detach(|| resolve_sql_test_chains(batch)) + .map_err(value_error) +} + +/// Plan SQL tests in one batch and return each plan with its distinct error messages. +#[pyfunction] +fn plan_compiled_sql_tests( + py: Python<'_>, + request: PlanningRequest<'_>, +) -> PyResult<(Vec, u64, u64)> { + let batch: SqlTestPlanBatch = planning_batch(request)?; + let outcome = py + .compiler_detach(|| plan_sql_test_batch(batch)) + .map_err(value_error)?; + Ok(( + outcome.plans.into_iter().map(plan_row).collect(), + outcome.planning_ns, + outcome.rendering_ns, + )) +} + +fn planning_batch(mut request: PlanningRequest<'_>) -> PyResult { + let models: Vec = request + .models + .iter() + .enumerate() + .map(|(index, model)| plan_model(model, request.rendered_model_sql.remove(&index))) + .collect::>()?; + let tests: Vec = request + .tests + .iter() + .enumerate() + .map(|(index, test)| plan_test(test, request.rendered_test_overrides.remove(&index))) + .collect::>()?; + Ok(SqlTestPlanBatch { + models, + functions: request.functions.into_iter().map(plan_function).collect(), + tests, + sql_analysis_enabled: request.sql_analysis_enabled, + sql_analysis_dialect: request.sql_analysis_dialect, + set_difference_operator: request.set_difference_operator, + requires_derived_table_aliases: request.requires_derived_table_aliases, + workers: PLANNING_WORKERS, + render_sql: request.render_sql, + include_plan: request.include_plan, + lexical_syntax: request.lexical_syntax.into(), + }) +} + +fn plan_model( + model: &Bound<'_, PyAny>, + rendered_sql: Option, +) -> PyResult { + Ok(SqlTestPlanModel { + name: model.getattr("name")?.extract()?, + query_sql: match rendered_sql { + Some(sql) => sql, + None => model.getattr("query_sql")?.extract()?, + }, + model_dependencies: model_dependencies(model)?, + }) +} + +/// The names of the models among a compiled model's dependencies, in dependency order. +fn model_dependencies(model: &Bound<'_, PyAny>) -> PyResult> { + let mut names: Vec = Vec::new(); + for dependency in model.getattr("deps")?.try_iter()? { + let dependency = dependency?; + if dependency + .getattr("resource_type")? + .eq(MODEL_RESOURCE_TYPE)? + { + names.push(dependency.getattr("name")?.extract()?); + } + } + Ok(names) +} + +fn plan_function( + (name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix): ( + String, + String, + String, + String, + String, + ), +) -> SqlTestPlanFunction { + SqlTestPlanFunction { + name, + udf_prefix: Some(udf_prefix), + udf_suffix: Some(udf_suffix), + table_function_prefix: Some(table_function_prefix), + table_function_suffix: Some(table_function_suffix), + } +} + +fn plan_test( + test: &Bound<'_, PyAny>, + rendered_overrides: Option>, +) -> PyResult { + let payload = test.getattr("payload")?; + let payload: SqlTestPlanPayload = if payload.hasattr("actual_cte")? { + SqlTestPlanPayload::Direct { + mode: payload.getattr("mode")?.extract()?, + actual_cte: cte(&payload.getattr("actual_cte")?)?, + expected_cte: cte(&payload.getattr("expected_cte")?)?, + helper_ctes: ctes(&payload, "helper_ctes")?, + } + } else { + SqlTestPlanPayload::Model { + authored_ctes: ctes(&payload, "authored_ctes")?, + model_query_overrides: match rendered_overrides { + Some(overrides) => overrides, + None => payload.getattr("model_query_overrides")?.extract()?, + }, + expected_ctes: ctes(&payload, "expected_ctes")?, + expected_model_names: payload.getattr("expected_model_names")?.extract()?, + assertion_ctes: ctes(&payload, "assertion_ctes")?, + read_helper_names: Some(test.getattr("read_helper_names")?.extract()?), + reference_target_model_names: Some( + test.getattr("reference_target_model_names")?.extract()?, + ), + } + }; + Ok(SqlTestPlanTest { + name: test.getattr("name")?.extract()?, + file_label: test + .getattr("test_file")? + .getattr("relative_path")? + .str()? + .extract()?, + payload, + }) +} + +fn ctes(owner: &Bound<'_, PyAny>, attribute: &str) -> PyResult> { + owner + .getattr(attribute)? + .try_iter()? + .map(|value| cte(&value?)) + .collect() +} + +fn cte(value: &Bound<'_, PyAny>) -> PyResult { + Ok(SqlTestCte { + name: value.getattr("name")?.extract()?, + sql_body: value.getattr("sql_body")?.extract()?, + }) +} + +fn plan_row(plan: SqlTestPlan) -> PlanRow { + let error_messages: Vec = sql_test_plan_error_messages(&plan); + ( + plan.sql, + plan.chain.into_iter().map(chain_row).collect(), + plan.assertions.into_iter().map(assertion_row).collect(), + plan.model_names, + plan.warnings + .into_iter() + .map(|warning| (warning.model_name, warning.severity, warning.message)) + .collect(), + error_messages, + ) +} + +fn chain_row(step: SqlTestChainStep) -> ChainRow { + ( + step.model_name, + step.resolved_sql, + step.expected_cte_sql, + step.lifted_ctes, + step.comparison_body_sql, + step.expected_columns, + step.expected_lifted_ctes, + ) +} + +fn assertion_row(step: SqlTestAssertionStep) -> AssertionRow { + ( + step.name, + step.resolved_sql, + step.lifted_ctes, + step.comparison_body_sql, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(plan_compiled_sql_tests, module)?)?; + module.add_function(wrap_pyfunction!(resolve_compiled_sql_test_chains, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs index 641f84817b..d7cf7c6c23 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -2,6 +2,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { - Ok(()) +use crate::bindings::_helpers::sql_test_glue::planning; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + planning::register(module) } diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py b/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py index 456d975490..04d7b4b944 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py @@ -1,9 +1,78 @@ """Failure cases owned by the native SQL-test planning glue lane.""" +from scripts.compiler_differential._helpers.corpus.case_builder import failure_case from scripts.compiler_differential.models import FailureCase +_TEST_PATH: str = "tests/unit/test_customer_totals.sql" +_WINDOWED_PATH: str = "models/marts/windowed_orders.sql" +_WINDOWED_MODEL: str = ( + "MODEL (\n description 'Orders in the cursor window',\n materialized incremental,\n" + " incremental_strategy delete_insert,\n cursor order_id,\n cursor_type integer,\n" + " cursor_inputs (raw_orders order_id,),\n);\n\n" + 'SELECT order_id, amount\nFROM __source("raw_orders")\n' + "WHERE order_id >= __cursor_start() AND order_id < __cursor_end()\n" +) +_ORDERS_FIXTURE: str = "SELECT 1 AS order_id, 10 AS customer_id, 5.0 AS amount, 'placed' AS status" + + +def _test(*, header: str, ctes: str) -> dict[str, str]: + return {_TEST_PATH: f"TEST ({header});\n\nWITH\n{ctes}\nSELECT 1\n"} + def sql_test_glue_failure_cases() -> tuple[FailureCase, ...]: """Return this lane's failure cases; the lane appends here without editing shared lists.""" - return () + return ( + failure_case( + name="sql-test-glue-missing-mock", + expected_code="S000", + expected_message=( + "test 'totals_without_source': model 'stg_orders' references " + '__source("raw_orders") which has no mock' + ), + files=_test( + header='name "totals_without_source"', + ctes=( + "__ref__stg_orders AS (SELECT 10 AS customer_id, 5.0 AS amount),\n" + "__expected__stg_orders AS (SELECT 10 AS customer_id, 5.0 AS amount),\n" + "__expected__customer_totals AS (\n" + " SELECT 10 AS customer_id, 5.0 AS total_amount\n)" + ), + ), + ), + failure_case( + name="sql-test-glue-inverted-window", + expected_code="P001", + expected_message=( + "SQL test 'inverted_window' in 'tests/unit/test_customer_totals.sql': " + "cursor_start '20' must be before the exclusive cursor_end '10' for model " + "'windowed_orders'" + ), + files={ + _WINDOWED_PATH: _WINDOWED_MODEL, + **_test( + header='name "inverted_window", cursor_start 20, cursor_end 10', + ctes=( + f"__source__raw_orders AS ({_ORDERS_FIXTURE}),\n" + "__expected__windowed_orders AS (SELECT 1 AS order_id, 5.0 AS amount)" + ), + ), + }, + ), + failure_case( + name="sql-test-glue-window-without-cursor-model", + expected_code="P001", + expected_message=( + "SQL test 'plain_window' in 'tests/unit/test_customer_totals.sql' declares " + "cursor_start or cursor_end, but no model it evaluates uses __cursor_start() or " + "__cursor_end()" + ), + files=_test( + header='name "plain_window", cursor_start 1', + ctes=( + f"__source__raw_orders AS ({_ORDERS_FIXTURE}),\n" + f"__expected__stg_orders AS ({_ORDERS_FIXTURE})" + ), + ), + ), + ) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 38f4bd7514..7b4ba1fca9 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3,6 +3,12 @@ from collections.abc import Sequence from typing import Any, TypedDict +from sqlbuild.compiler.sql_test_glue.models import ( + NativeSqlTestChainRequest, + NativeSqlTestPlanningRequest, +) +from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow + BUILD_IDENTITY: str class LintPreparationRequest(TypedDict): @@ -354,6 +360,10 @@ def build_fast_column_lineage( ) -> list[tuple[str, list[tuple[str, str, str, list[tuple[str, str, str]]]], bool, str | None]]: ... # Native analysis: SQL test planning glue. +def plan_compiled_sql_tests( + request: NativeSqlTestPlanningRequest, / +) -> tuple[list[NativeSqlTestPlanRow], int, int]: ... +def resolve_compiled_sql_test_chains(request: NativeSqlTestChainRequest, /) -> list[list[str]]: ... # Native analysis: compiled project assembly. diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py index 89a301f85a..fb0b139ef8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py @@ -21,6 +21,8 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( declared_window_cursor_models, declares_cursor_window, @@ -38,6 +40,16 @@ from sqlbuild.compiler.profiling.classes.context import CompileTimingContext from sqlbuild.compiler.profiling.models import CompileTimingCollector from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.compiler.sql_test_glue.constants import CURSOR_INTRINSIC_NAMES +from sqlbuild.compiler.sql_test_glue.models import ( + NativeSqlTestChainRequest, + NativeSqlTestPlanningRequest, +) +from sqlbuild.compiler.sql_test_glue.types import ( + NativeAssertionStepRow, + NativeChainStepRow, + NativeSqlTestPlanRow, +) from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule _UNRESOLVED_REFERENCE_PREFIX: str = "sql_test_reference:" @@ -56,7 +68,7 @@ def plan_and_render_sql_test_artifacts( """Plan and render SQL tests in one deterministic native project call.""" artifacts: list[NativeSqlTestArtifact] = [] - for plan in plan_sql_tests_natively( + for plan in _plan_sql_tests_from_json( project=project, tests=tests, adapter=adapter, @@ -76,6 +88,34 @@ def plan_and_render_sql_test_artifacts( return tuple(artifacts) +def plan_compiled_sql_test_artifacts( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, +) -> tuple[NativeSqlTestArtifact, ...]: + """Plan and render SQL tests from the compiled objects, with natively projected errors.""" + + artifacts: list[NativeSqlTestArtifact] = [] + for plan, error_messages in _plan_compiled_sql_tests( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + render_sql=True, + include_plan=False, + ): + if plan.sql is None: + raise NativeSqlTestPlanningError("native SQL-test planning omitted rendered SQL") + artifacts.append( + NativeSqlTestArtifact( + sql=plan.sql, model_names=plan.model_names, error_messages=error_messages + ) + ) + return tuple(artifacts) + + def sql_test_plan_error_messages(*, warnings: tuple[PlanWarning, ...]) -> tuple[str, ...]: """Return each distinct ERROR-severity planning message for one SQL test, in order.""" @@ -97,6 +137,188 @@ def plan_sql_tests_natively( ) -> tuple[NativeSqlTestPlan, ...]: """Plan SQL-test chains and assertions in one native batch, optionally rendering SQL.""" + if native_stage_enabled(NativeStage.SQL_TEST_GLUE): + return tuple( + plan + for plan, _error_messages in _plan_compiled_sql_tests( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + render_sql=render_sql, + include_plan=include_plan, + ) + ) + return _plan_sql_tests_from_json( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + render_sql=render_sql, + include_plan=include_plan, + ) + + +def _plan_compiled_sql_tests( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, + render_sql: bool, + include_plan: bool, +) -> tuple[tuple[NativeSqlTestPlan, tuple[str, ...]], ...]: + """Plan from the compiled objects natively; Python supplies only the adapter's renderings.""" + + if not tests: + return () + start_ns: int = time.perf_counter_ns() + rendered_model_sql: dict[int, str] = { + index: render_test_cursor_intrinsics( + sql=model.query_sql, model=model, adapter=adapter, test=None + ) + for index, model in enumerate(project.models) + if _mentions_cursor_intrinsic(model.query_sql) + } + functions: tuple[tuple[str, str, str, str, str], ...] = _function_templates( + project=project, adapter=adapter + ) + rendered_test_overrides: dict[int, dict[str, str]] = _rendered_test_overrides( + project=project, tests=tests, adapter=adapter + ) + request: NativeSqlTestPlanningRequest = NativeSqlTestPlanningRequest( + models=project.models, + rendered_model_sql=rendered_model_sql, + functions=functions, + tests=tests, + rendered_test_overrides=rendered_test_overrides, + sql_analysis_enabled=sql_analysis_enabled, + sql_analysis_dialect=adapter.sql_analysis_dialect(), + set_difference_operator=adapter.render_set_difference_operator(), + requires_derived_table_aliases=adapter.requires_derived_table_aliases(), + lexical_syntax=project.sql_lexical_syntax.native_mapping, + render_sql=render_sql, + include_plan=include_plan, + ) + try: + rows, planning_ns, rendering_ns = _native.plan_compiled_sql_tests(request) + except ValueError as error: + raise _planning_error(error=error, tests=tests) from None + _record_planning_timing(start_ns=start_ns, planning_ns=planning_ns, rendering_ns=rendering_ns) + return tuple(_plan_from_row(row=row) for row in rows) + + +def _plan_from_row(*, row: NativeSqlTestPlanRow) -> tuple[NativeSqlTestPlan, tuple[str, ...]]: + sql, chain, assertions, model_names, warnings, error_messages = row + plan: NativeSqlTestPlan = NativeSqlTestPlan( + chain=tuple(_chain_step_from_row(row=step) for step in chain), + assertions=tuple(_assertion_step_from_row(row=step) for step in assertions), + model_names=tuple(model_names), + warnings=tuple( + PlanWarning(model_name=model_name, severity=WarningSeverity(severity), message=message) + for model_name, severity, message in warnings + ), + sql=sql, + ) + return plan, tuple(error_messages) + + +def _assertion_step_from_row(*, row: NativeAssertionStepRow) -> SqlTestAssertionStep: + name, resolved_sql, lifted_ctes, comparison_body_sql = row + return SqlTestAssertionStep( + name=name, + resolved_sql=resolved_sql, + lifted_ctes=tuple(lifted_ctes), + comparison_body_sql=comparison_body_sql, + ) + + +def _chain_step_from_row(*, row: NativeChainStepRow) -> ChainStep: + ( + model_name, + resolved_sql, + expected_cte_sql, + lifted_ctes, + comparison_body_sql, + expected_columns, + expected_lifted_ctes, + ) = row + return ChainStep( + model_name=model_name, + resolved_sql=resolved_sql, + expected_cte_sql=expected_cte_sql, + lifted_ctes=tuple(lifted_ctes), + comparison_body_sql=comparison_body_sql, + expected_columns=None if expected_columns is None else tuple(expected_columns), + expected_lifted_ctes=tuple(expected_lifted_ctes), + ) + + +def _mentions_cursor_intrinsic(sql: str) -> bool: + """Whether SQL names a cursor intrinsic; SQL that does not is never rendered differently.""" + + return any(name in sql for name in CURSOR_INTRINSIC_NAMES) + + +def _rendered_test_overrides( + *, project: CompiledProject, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter +) -> dict[int, dict[str, str]]: + """Cursor-rendered model overrides for the tests whose overrides cursor windows change.""" + + models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + windowed_indexes: tuple[int, ...] = tuple( + index + for index, test in enumerate(tests) + if isinstance(test.payload, CompiledModelSqlTestPayload) + and declares_cursor_window(test=test) + ) + chains: tuple[tuple[str, ...], ...] = ( + resolve_sql_test_model_chains( + project=project, tests=tuple(tests[index] for index in windowed_indexes) + ) + if windowed_indexes + else () + ) + chains_by_index: dict[int, tuple[str, ...]] = dict(zip(windowed_indexes, chains, strict=True)) + rendered: dict[int, dict[str, str]] = {} + for index, test in enumerate(tests): + payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload + if not isinstance(payload, CompiledModelSqlTestPayload): + continue + chain: tuple[str, ...] | None = chains_by_index.get(index) + if chain is None and not any( + _mentions_cursor_intrinsic(sql) for sql in payload.model_query_overrides.values() + ): + continue + overrides: dict[str, str] | None = _windowed_model_query_overrides( + test=test, models_by_name=models_by_name, adapter=adapter, chain=chain + ) + if overrides is not None: + rendered[index] = overrides + return rendered + + +def _record_planning_timing(*, start_ns: int, planning_ns: int, rendering_ns: int) -> None: + elapsed_ns: int = time.perf_counter_ns() - start_ns + boundary_overhead_ns: int = max(0, elapsed_ns - planning_ns - rendering_ns) + collector: CompileTimingCollector | None = CompileTimingContext.active.get() + if collector is not None: + collector.add( + phase="test_planning_ms", + elapsed_ns=planning_ns + boundary_overhead_ns, + ) + collector.add(phase="comparison_render_ms", elapsed_ns=rendering_ns) + + +def _plan_sql_tests_from_json( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, + render_sql: bool, + include_plan: bool, +) -> tuple[NativeSqlTestPlan, ...]: if not tests: return () start_ns: int = time.perf_counter_ns() @@ -143,15 +365,7 @@ def plan_sql_tests_natively( plans: tuple[NativeSqlTestPlan, ...] = tuple( _plan_from_payload(value=value) for value in response ) - elapsed_ns: int = time.perf_counter_ns() - start_ns - boundary_overhead_ns: int = max(0, elapsed_ns - planning_ns - rendering_ns) - collector: CompileTimingCollector | None = CompileTimingContext.active.get() - if collector is not None: - collector.add( - phase="test_planning_ms", - elapsed_ns=planning_ns + boundary_overhead_ns, - ) - collector.add(phase="comparison_render_ms", elapsed_ns=rendering_ns) + _record_planning_timing(start_ns=start_ns, planning_ns=planning_ns, rendering_ns=rendering_ns) return plans @@ -162,6 +376,18 @@ def resolve_sql_test_model_chains( if not tests: return () + if native_stage_enabled(NativeStage.SQL_TEST_GLUE): + try: + native_chains: list[list[str]] = _native.resolve_compiled_sql_test_chains( + NativeSqlTestChainRequest( + models=project.models, + tests=tests, + lexical_syntax=project.sql_lexical_syntax.native_mapping, + ) + ) + except ValueError as error: + raise _planning_error(error=error, tests=tests) from None + return tuple(tuple(chain) for chain in native_chains) request: dict[str, object] = { "models": _chain_model_requests(project=project), "tests": [_test_request(test=test) for test in tests], @@ -330,7 +556,30 @@ def _string_tuple(*, value: object, context: str) -> tuple[str, ...]: def _function_requests( *, project: CompiledProject, adapter: BaseAdapter ) -> list[dict[str, object]]: - functions: list[dict[str, object]] = [] + return [ + { + "name": name, + "udfPrefix": udf_prefix, + "udfSuffix": udf_suffix, + "tableFunctionPrefix": table_function_prefix, + "tableFunctionSuffix": table_function_suffix, + } + for ( + name, + udf_prefix, + udf_suffix, + table_function_prefix, + table_function_suffix, + ) in _function_templates(project=project, adapter=adapter) + ] + + +def _function_templates( + *, project: CompiledProject, adapter: BaseAdapter +) -> tuple[tuple[str, str, str, str, str], ...]: + """Each function's name and the adapter's UDF and table-function call templates.""" + + functions: list[tuple[str, str, str, str, str]] = [] for function in project.functions: target: str | None = function.destination.qualified_name if target is None: @@ -348,15 +597,9 @@ def _function_requests( ) ) functions.append( - { - "name": function.name, - "udfPrefix": udf_prefix, - "udfSuffix": udf_suffix, - "tableFunctionPrefix": table_function_prefix, - "tableFunctionSuffix": table_function_suffix, - } + (function.name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix) ) - return functions + return tuple(functions) def _planning_test_requests( diff --git a/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py new file mode 100644 index 0000000000..b3cfbee3fa --- /dev/null +++ b/src/sqlbuild/compiler/planner/main/execution/_plan_compiled_sql_test_artifacts.py @@ -0,0 +1,25 @@ +"""Planner entrypoint for SQL-test artifacts planned natively from the compiled objects.""" + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( + plan_compiled_sql_test_artifacts as _plan_compiled_sql_test_artifacts, +) +from sqlbuild.compiler.planner.models import NativeSqlTestArtifact + + +def plan_compiled_sql_test_artifacts( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, +) -> tuple[NativeSqlTestArtifact, ...]: + """Plan and render SQL tests natively from the compiled objects, without a JSON request.""" + + return _plan_compiled_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + ) diff --git a/src/sqlbuild/compiler/sql_test_glue/constants.py b/src/sqlbuild/compiler/sql_test_glue/constants.py new file mode 100644 index 0000000000..727ece2d99 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/constants.py @@ -0,0 +1,5 @@ +"""Constants for the native SQL-test planning glue.""" + +from __future__ import annotations + +CURSOR_INTRINSIC_NAMES: tuple[str, ...] = ("__cursor_start", "__cursor_end") diff --git a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py index 7b5861fead..94816d02dc 100644 --- a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py +++ b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py @@ -4,6 +4,9 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.planner.main.execution._plan_compiled_sql_test_artifacts import ( + plan_compiled_sql_test_artifacts, +) from sqlbuild.compiler.planner.models import NativeSqlTestArtifact @@ -14,6 +17,11 @@ def plan_native_sql_test_artifacts( adapter: BaseAdapter, sql_analysis_enabled: bool, ) -> tuple[NativeSqlTestArtifact, ...] | None: - """Return one artifact per test, or None where Python must plan the tests.""" + """Return one artifact per test, planned from the compiled objects without a JSON request.""" - return None + return plan_compiled_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + ) diff --git a/src/sqlbuild/compiler/sql_test_glue/models.py b/src/sqlbuild/compiler/sql_test_glue/models.py new file mode 100644 index 0000000000..0883787198 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/models.py @@ -0,0 +1,34 @@ +"""Requests the native SQL-test planning glue reads from compiled objects.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.models import CompiledModel, CompiledSqlTest + + +@dataclass(frozen=True) +class NativeSqlTestPlanningRequest: + """Compiled models and tests to plan, with the adapter facts native planning cannot derive.""" + + models: tuple[CompiledModel, ...] + rendered_model_sql: dict[int, str] + functions: tuple[tuple[str, str, str, str, str], ...] + tests: tuple[CompiledSqlTest, ...] + rendered_test_overrides: dict[int, dict[str, str]] + sql_analysis_enabled: bool + sql_analysis_dialect: str | None + set_difference_operator: str + requires_derived_table_aliases: bool + lexical_syntax: dict[str, object] + render_sql: bool + include_plan: bool + + +@dataclass(frozen=True) +class NativeSqlTestChainRequest: + """Compiled models and the tests whose unmocked model chains to resolve.""" + + models: tuple[CompiledModel, ...] + tests: tuple[CompiledSqlTest, ...] + lexical_syntax: dict[str, object] diff --git a/src/sqlbuild/compiler/sql_test_glue/types.py b/src/sqlbuild/compiler/sql_test_glue/types.py new file mode 100644 index 0000000000..9a3fdf65c4 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/types.py @@ -0,0 +1,24 @@ +"""Row shapes the native SQL-test planning glue returns.""" + +from __future__ import annotations + +type CtePairRow = tuple[str, str] +type NativeChainStepRow = tuple[ + str, + str, + str | None, + list[CtePairRow], + str | None, + list[str] | None, + list[CtePairRow], +] +type NativeAssertionStepRow = tuple[str, str, list[CtePairRow], str | None] +type NativePlanWarningRow = tuple[str | None, str, str] +type NativeSqlTestPlanRow = tuple[ + str | None, + list[NativeChainStepRow], + list[NativeAssertionStepRow], + list[str], + list[NativePlanWarningRow], + list[str], +] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/_test_types.py new file mode 100644 index 0000000000..23f002f41f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/_test_types.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NativeSqlTestGlueCliTestCase: + """A DuckDB project whose SQL tests every engine must compile, inspect and run alike.""" + + description: str + files: dict[str, str] + expected_compile_returncode: int + expected_test_returncode: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/helpers.py new file mode 100644 index 0000000000..e4e8c564f7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/helpers.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path +from typing import NamedTuple + +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb + +_DURATION: re.Pattern[str] = re.compile(r"\(\d+(?:\.\d+)?m?s\)|\d+\.\d+s") +_VOLATILE_REPORT_KEYS: tuple[str, ...] = ("compiler_engine", "compile_timings") +_VOLATILE_RESULT_KEYS: frozenset[str] = frozenset({"duration_ms"}) + + +class EngineSqlTestRun(NamedTuple): + """One engine's compile report and test SQL, test plan inspection and test results.""" + + compile_returncode: int + compile_report: str + compiled_tests: dict[str, str] + inspect_returncode: int + inspect_output: str + test_returncode: int + test_results: str + + +def engine_sql_test_run(*, root: Path, files: dict[str, str], engine: str) -> EngineSqlTestRun: + """Write the project, then compile, inspect and run its SQL tests under one engine.""" + + project_dir: Path = root / engine + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + environment: dict[str, str] = {REUSE_DISABLE_ENV_VAR: "1"} + engine_args: tuple[str, ...] = ("--compiler-engine", engine) + compiled: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, + args=(*engine_args, "compile", "--json", "--no-cache"), + env=environment, + ) + report: dict[str, object] = json.loads(compiled.stdout.replace(str(project_dir), "")) + for key in _VOLATILE_REPORT_KEYS: + _ = report.pop(key, None) + tests_dir: Path = project_dir / "target" / "compiled" / "tests" + inspected: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=(*engine_args, "test", "--inspect"), env=environment + ) + tested: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=(*engine_args, "test", "--json"), env=environment + ) + return EngineSqlTestRun( + compile_returncode=compiled.returncode, + compile_report=json.dumps(report, indent=2), + compiled_tests={ + path.relative_to(tests_dir).as_posix(): path.read_text("utf-8") + for path in sorted(tests_dir.rglob("*.sql")) + }, + inspect_returncode=inspected.returncode, + inspect_output=_DURATION.sub("", inspected.stdout + inspected.stderr), + test_returncode=tested.returncode, + test_results=_stable_results(tested.stdout.replace(str(project_dir), "")), + ) + + +def _stable_results(stdout: str) -> str: + try: + payload: object = json.loads(stdout, object_hook=_without_durations) + except json.JSONDecodeError: + return _DURATION.sub("", stdout) + return json.dumps(payload, indent=2, sort_keys=True) + + +def _without_durations(value: dict[str, object]) -> dict[str, object]: + return {key: value[key] for key in value.keys() - _VOLATILE_RESULT_KEYS} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/test_native_sql_test_glue_cli.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/test_native_sql_test_glue_cli.py new file mode 100644 index 0000000000..104c4e19f9 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/test_native_sql_test_glue_cli.py @@ -0,0 +1,172 @@ +"""Every engine compiles, inspects and runs SQL tests alike while the preview plans them natively.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_test_glue._test_types import ( + NativeSqlTestGlueCliTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_test_glue.helpers import ( + EngineSqlTestRun, + engine_sql_test_run, +) + +_ENGINES: tuple[str, ...] = ("python", "native", "native-preview") +_ORDER_ROWS: str = ( + "SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount, " + "TIMESTAMP '2026-02-01 10:00:00' AS ordered_at\n" + " UNION ALL SELECT 2, 11, 20, TIMESTAMP '2026-02-05 09:00:00'" +) +_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders_testing"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n description: Orders feed.\n expression: >-\n" + " (SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount,\n" + " TIMESTAMP '2026-02-01 10:00:00' AS ordered_at)\n" + " columns:\n - name: order_id\n type: INTEGER\n" + " - name: customer_id\n type: INTEGER\n" + " - name: amount\n type: INTEGER\n" + " - name: ordered_at\n type: TIMESTAMP\n" + " - name: raw_customers\n description: Customers feed.\n expression: >-\n" + " (SELECT 10 AS customer_id, 'east' AS region)\n" + " columns:\n - name: customer_id\n type: INTEGER\n" + " - name: region\n type: VARCHAR\n" + ), + "macros/amounts.py": ('def cents(column: str) -> str:\n return f"({column}) * 100"\n'), + "models/staging/stg_orders.sql": ( + 'MODEL (description "Staged orders");\n\n' + 'SELECT order_id, customer_id, amount, ordered_at FROM __source("raw_orders")\n' + ), + "models/staging/stg_customers.sql": ( + 'MODEL (description "Staged customers");\n\n' + 'SELECT customer_id, region FROM __source("raw_customers") WHERE region IS NOT NULL\n' + ), + "models/marts/order_regions.sql": ( + 'MODEL (description "Orders with regions");\n\n' + 'WITH big AS (\n SELECT * FROM __ref("stg_orders") WHERE amount > 100\n)\n' + 'SELECT b.order_id, c.region, @cents("b.amount") AS amount_cents\n' + 'FROM big b JOIN __ref("stg_customers") c ON b.customer_id = c.customer_id\n' + ), + "models/marts/daily_orders.sql": ( + "MODEL (description 'Daily orders',\n materialized incremental,\n" + " incremental_strategy delete_insert,\n cursor ordered_at,\n cursor_type timestamp,\n" + " cursor_grain day,\n cursor_inputs (raw_orders ordered_at,),\n);\n\n" + 'SELECT ordered_at, amount\nFROM __source("raw_orders")\n' + "WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end()\n" + ), + "tests/unit/test_order_regions.sql": ( + 'TEST (name "big_orders_get_regions");\n\nWITH\n' + f"__source__raw_orders AS (\n {_ORDER_ROWS}\n),\n" + "__source__raw_customers AS (SELECT 10 AS customer_id, 'east' AS region),\n" + "east AS (SELECT 'east' AS region),\n" + "__expected__order_regions AS (\n" + " SELECT 1 AS order_id, region, 15000 AS amount_cents FROM east\n),\n" + "__assert__amounts_are_positive AS (\n" + ' SELECT * FROM __ref("order_regions") WHERE amount_cents <= 0\n)\n' + "SELECT 1\n\n" + 'TEST (name "mocked_cents");\n\nWITH\n' + "__macro__cents AS (SELECT 'amount'),\n" + "__ref__stg_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount,\n" + " TIMESTAMP '2026-02-01' AS ordered_at\n),\n" + "__ref__stg_customers AS (SELECT 10 AS customer_id, 'east' AS region),\n" + "__expected__order_regions AS (\n" + " SELECT 1 AS order_id, 'east' AS region, 150 AS amount_cents\n)\n" + "SELECT 1\n\n" + 'TEST (name "wrong_region");\n\nWITH\n' + "__ref__stg_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount,\n" + " TIMESTAMP '2026-02-01' AS ordered_at\n),\n" + "__ref__stg_customers AS (SELECT 10 AS customer_id, 'east' AS region),\n" + "__expected__order_regions AS (\n" + " SELECT 1 AS order_id, 'west' AS region, 15000 AS amount_cents\n)\n" + "SELECT 1\n" + ), + "tests/unit/test_daily_orders.sql": ( + 'TEST (name "default_window");\n\nWITH\n' + f"__source__raw_orders AS (\n {_ORDER_ROWS}\n),\n" + "__expected__daily_orders AS (\n" + " SELECT TIMESTAMP '2026-02-01 10:00:00' AS ordered_at, 150 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-02-05 09:00:00' AS ordered_at, 20 AS amount\n)\n" + "SELECT 1\n\n" + 'TEST (name "declared_window", cursor_start "2026-02-01", cursor_end "2026-02-03");\n\n' + f"WITH\n__source__raw_orders AS (\n {_ORDER_ROWS}\n),\n" + "__expected__daily_orders AS (\n" + " SELECT TIMESTAMP '2026-02-01 10:00:00' AS ordered_at, 150 AS amount\n)\n" + "SELECT 1\n" + ), + "tests/unit/test_cents.sql": ( + 'TEST (mode macro, name "doubles_cents");\n\nWITH\n' + "amounts AS (SELECT 2 AS amount),\n" + '__macro_actual__ AS (SELECT @cents("amount") AS cents FROM amounts),\n' + "__macro_expected__ AS (SELECT 200 AS cents)\nSELECT 1\n" + ), +} +_MISSING_MOCK_TEST: str = ( + 'TEST (name "regions_without_customers");\n\nWITH\n' + f"__source__raw_orders AS (\n {_ORDER_ROWS}\n),\n" + "__expected__order_regions AS (\n" + " SELECT 1 AS order_id, 'east' AS region, 15000 AS amount_cents\n)\n" + "SELECT 1\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeSqlTestGlueCliTestCase( + description="mocks, helpers, assertions, macro mocks, cursor windows and macro tests", + files=_PROJECT, + expected_compile_returncode=0, + expected_test_returncode=1, + expected_fragments=( + "real models: stg_customers, stg_orders, order_regions", + "boundary: stg_orders is replaced by __ref__stg_orders", + '"status": "fail"', + '"status": "pass"', + ), + ), + NativeSqlTestGlueCliTestCase( + description="a test missing a source mock fails planning with the planner's error", + files={**_PROJECT, "tests/unit/test_missing_mock.sql": _MISSING_MOCK_TEST}, + expected_compile_returncode=1, + expected_test_returncode=1, + expected_fragments=( + "test 'regions_without_customers': model 'stg_customers' references " + '__source(\\"raw_customers\\") which has no mock', + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_tests_when_compiling_inspecting_and_testing_then_every_engine_agrees( + test_case: NativeSqlTestGlueCliTestCase, tmp_path: Path +) -> None: + runs: dict[str, EngineSqlTestRun] = { + engine: engine_sql_test_run(root=tmp_path, files=test_case.files, engine=engine) + for engine in _ENGINES + } + python_run: EngineSqlTestRun = runs["python"] + combined_output: str = ( + python_run.compile_report + python_run.inspect_output + python_run.test_results + ) + + assert ( + {engine: run == python_run for engine, run in runs.items()}, + python_run.compile_returncode, + python_run.test_returncode, + bool(python_run.compiled_tests) or python_run.compile_returncode != 0, + tuple(fragment in combined_output for fragment in test_case.expected_fragments), + ) == ( + dict.fromkeys(_ENGINES, True), + test_case.expected_compile_returncode, + test_case.expected_test_returncode, + True, + (True,) * len(test_case.expected_fragments), + ), combined_output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 3ec873e7e6..d3a7204a4b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -8,6 +8,7 @@ import pytest from sqlbuild.compiler.lineage.models import ProjectColumnLineage +from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( NativeAnalysisSeamTestCase, ) @@ -35,7 +36,7 @@ "complete_native_semantic_diagnostics": [None], "evaluate_native_model_contracts": [None], "build_native_column_lineage": [ANY], - "plan_native_sql_test_artifacts": [None], + "plan_native_sql_test_artifacts": [ANY], }, ) ], @@ -72,6 +73,10 @@ def test_given_project_when_compiling_with_each_engine_then_seams_defer_and_outp test_case.expected_preview_returns, ) assert isinstance(preview_seams["build_native_column_lineage"][0], ProjectColumnLineage) + planned_artifacts: object = preview_seams["plan_native_sql_test_artifacts"][0] + assert isinstance(planned_artifacts, tuple) + assert planned_artifacts + assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) assert (native_run.returncode, preview_run.returncode) == (0, 0) assert report_without_engine(native_run) == report_without_engine(python_run) assert report_without_engine(preview_run) == report_without_engine(python_run) diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/__init__.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py new file mode 100644 index 0000000000..3f83db0a35 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GeneratedSqlTestPlanningParityTestCase: + """Seeded SQL-test projects the native planning glue and the JSON request must agree on.""" + + description: str + seed: int + count: int + test_count: int + adapter_names: tuple[str, ...] + expected_minimum_planned: int + expected_minimum_with_errors: int + expected_minimum_raised: int diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py new file mode 100644 index 0000000000..27739171d5 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -0,0 +1,337 @@ +"""Seeded SQL-test projects and both planning paths for the native planning glue tests.""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from dataclasses import dataclass +from itertools import chain, compress +from operator import attrgetter +from pathlib import Path +from typing import cast + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.pipeline.main.graph import build_project_graph +from sqlbuild.compiler.planner._helpers.sql_tests import native_planning +from sqlbuild.compiler.planner.models import ( + NativeSqlTestArtifact, + NativeSqlTestPlan, + PlanWarning, +) +from sqlbuild.compiler.planner.types import WarningSeverity + + +@dataclass(frozen=True) +class PlanningCallOutcome: + """What one planning call returned, or the type and text of what it raised.""" + + value: object + raised: tuple[str, str] | None + + +_PROJECT_TOML: str = ( + 'name = "orders_testing"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +_SOURCES: str = """sources: + - name: raw_orders + description: Orders feed. + schema: main + table: raw_orders + columns: + - name: order_id + type: INTEGER + - name: customer_id + type: INTEGER + - name: amount + type: INTEGER + - name: status + type: VARCHAR + - name: ordered_at + type: TIMESTAMP + - name: raw_customers + description: Customers feed. + schema: main + table: raw_customers + columns: + - name: customer_id + type: INTEGER + - name: region + type: VARCHAR +""" +_SEEDS: str = """seeds: + - name: regions + description: Region labels. + columns: + - name: region + type: VARCHAR + - name: label + type: VARCHAR +""" +_MACROS: str = ( + "def cents(column: str) -> str:\n" + ' return f"({column}) * 100"\n\n' + "def known_region() -> str:\n" + ' return "region IS NOT NULL"\n' +) +_UDF: str = ( + 'FUNCTION (\n description "Whether an amount is large",\n' + " arguments (amount INTEGER),\n returns BOOLEAN,\n);\n\namount > 100\n" +) +_TABLE_FUNCTION: str = ( + 'FUNCTION (\n description "Orders of one customer",\n' + " arguments (p_customer_id INTEGER),\n" + " returns table (order_id INTEGER, amount INTEGER)\n);\n\n" + 'SELECT order_id, amount FROM __ref("stg_orders") WHERE customer_id = p_customer_id\n' +) +_BASE_MODELS: dict[str, str] = { + "stg_orders": ( + 'SELECT order_id, customer_id, amount, status, ordered_at\nFROM __source("raw_orders")' + ), + "stg_customers": ( + 'SELECT customer_id, region\nFROM __source("raw_customers")\nWHERE @known_region()' + ), + "int_orders": ( + 'SELECT o.order_id, o.customer_id, c.region, @cents("o.amount") AS amount_cents,\n' + ' __udf("udf__is_big")(o.amount) AS is_big\n' + 'FROM __ref("stg_orders") o\nJOIN __ref("stg_customers") c ON o.customer_id = c.customer_id' + ), + "fct_regions": ( + "WITH totals AS (\n" + ' SELECT region, SUM(amount_cents) AS total_cents FROM __ref("int_orders") GROUP BY 1\n' + ")\nSELECT t.region, r.label, t.total_cents\n" + 'FROM totals t\nLEFT JOIN __seed("regions") r ON t.region = r.region' + ), + "customer_orders": ('SELECT order_id, amount\nFROM __table_fn("table_fn__customer_orders")(1)'), +} +_DAILY_ORDERS: str = ( + "MODEL (description 'Daily orders.',\n materialized incremental,\n" + " incremental_strategy delete_insert,\n cursor ordered_at,\n cursor_type timestamp,\n" + " cursor_grain day,\n cursor_inputs (raw_orders ordered_at,),\n);\n\n" + "SELECT ordered_at, amount\n" + 'FROM __source("raw_orders")\n' + "WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end()\n" +) +_FIXTURES: dict[str, str] = { + "__source__raw_orders": ( + "SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount, 'placed' AS status, " + "TIMESTAMP '2026-02-01 10:00:00' AS ordered_at" + ), + "__source__raw_customers": "SELECT 10 AS customer_id, 'east' AS region", + "__seed__regions": "SELECT 'east' AS region, 'East' AS label", + "__ref__stg_orders": ( + "SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount, 'placed' AS status, " + "TIMESTAMP '2026-02-01 10:00:00' AS ordered_at" + ), + "__ref__stg_customers": "SELECT 10 AS customer_id, 'east' AS region", + "__ref__int_orders": ( + "SELECT 1 AS order_id, 10 AS customer_id, 'east' AS region, 15000 AS amount_cents, " + "TRUE AS is_big" + ), + "__table_fn__table_fn__customer_orders": "SELECT 1 AS order_id, 150 AS amount", + "__macro__cents": "SELECT 'amount'", +} +_UPSTREAM: dict[str, tuple[str, ...]] = { + "stg_orders": ("__source__raw_orders",), + "stg_customers": ("__source__raw_customers",), + "int_orders": ("__ref__stg_orders", "__ref__stg_customers", "__source__raw_orders"), + "fct_regions": ("__ref__int_orders", "__seed__regions", "__source__raw_customers"), + "customer_orders": ("__table_fn__table_fn__customer_orders",), + "daily_orders": ("__source__raw_orders",), +} +_DIRECT_TESTS: tuple[str, ...] = ( + 'TEST (mode macro, name "doubles_cents");\n\nWITH\n' + "amounts AS (SELECT 2 AS amount),\n" + '__macro_actual__ AS (SELECT @cents("amount") AS cents FROM amounts),\n' + "__macro_expected__ AS (SELECT 200 AS cents)\nSELECT 1\n", + 'TEST (mode udf, name "flags_big_amounts");\n\nWITH\n' + '__udf_actual__ AS (SELECT __udf("udf__is_big")(150) AS is_big),\n' + "__udf_expected__ AS (SELECT TRUE AS is_big)\nSELECT 1\n", + 'TEST (mode table_fn, name "lists_customer_orders");\n\nWITH\n' + "__table_fn_actual__ AS (\n" + ' SELECT order_id, amount FROM __table_fn("table_fn__customer_orders")(10)\n),\n' + "__table_fn_expected__ AS (SELECT 1 AS order_id, 150 AS amount)\nSELECT 1\n", +) +_DIRECT_TEST_SHARE: float = 0.8 +_STRAY_WINDOW_SHARE: float = 0.1 +_MOCK_SHARE: float = 0.85 +_MACRO_MOCK_SHARE: float = 0.5 +_HELPER_SHARE: float = 0.4 +_ASSERTION_SHARE: float = 0.2 +_WINDOWS: tuple[str, ...] = ( + "", + ', cursor_start "2026-02-01", cursor_end "2026-02-03"', + ', cursor_start "2026-02-03", cursor_end "2026-02-01"', +) + + +def generated_sql_test_files(*, rng: random.Random, test_count: int) -> dict[str, str]: + """A project of mocked, expected, asserted, helper, direct and failing SQL tests.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": _SOURCES, + "seeds/regions.yml": _SEEDS, + "seeds/regions.csv": "region,label\neast,East\n", + "macros/amounts.py": _MACROS, + "functions/sql/udf__is_big.sql": _UDF, + "functions/sql/table_fn__customer_orders.sql": _TABLE_FUNCTION, + "models/daily_orders.sql": _DAILY_ORDERS, + } + files.update( + (f"models/{name}.sql", f'MODEL (description "Model {name}.");\n\n{sql}\n') + for name, sql in _BASE_MODELS.items() + ) + files.update( + (f"tests/unit/test_generated_{index}.sql", _model_test(rng=rng, index=index)) + for index in range(test_count) + ) + files.update( + compress( + ( + (f"tests/unit/test_direct_{index}.sql", sql) + for index, sql in enumerate(_DIRECT_TESTS) + ), + [rng.random() < _DIRECT_TEST_SHARE for _ in _DIRECT_TESTS], + ) + ) + return files + + +def _model_test(*, rng: random.Random, index: int) -> str: + """One model test: the first target's first mock, then random mocks, helper and checks.""" + + targets: list[str] = rng.sample(sorted(_UPSTREAM), k=rng.choice((1, 1, 2))) + windowed: bool = "daily_orders" in targets or rng.random() < _STRAY_WINDOW_SHARE + window: str = rng.choice(((_WINDOWS[0],), _WINDOWS)[windowed]) + fixtures: list[str] = list(chain.from_iterable(_UPSTREAM[target] for target in targets)) + mocks: list[str] = [ + fixtures[0], + *compress(fixtures, [rng.random() < _MOCK_SHARE for _ in fixtures]), + *rng.sample(sorted(_FIXTURES), k=rng.choice((0, 0, 0, 1))), + *(("__macro__cents",) * ("int_orders" in targets and rng.random() < _MACRO_MOCK_SHARE)), + ] + has_helper: bool = rng.random() < _HELPER_SHARE + helper: str = f"wanted_{index}" + helper_ctes: list[tuple[str, str]] = [ + ( + helper, + rng.choice( + ("SELECT 1 AS order_id", f'SELECT order_id FROM __ref("{rng.choice(targets)}")') + ), + ) + ][: int(has_helper)] + expected_sql: str = ("SELECT 1 AS order_id", f"SELECT order_id FROM {helper}")[has_helper] + checks: list[tuple[str, str]] = [ + ( + (f"__expected__{target}", expected_sql), + (f"__assert__{target}_has_rows", f'SELECT * FROM __ref("{target}") WHERE 1 = 0'), + )[rng.random() < _ASSERTION_SHARE] + for target in targets + ] + ctes: dict[str, str] = { + **{mock: _FIXTURES[mock] for mock in mocks}, + **dict(helper_ctes), + **dict(checks), + } + body: str = ",\n".join(f"{name} AS (\n {sql}\n)" for name, sql in ctes.items()) + return f'TEST (name "generated_{index}"{window});\n\nWITH\n{body}\nSELECT 1\n' + + +def compiled_project(*, project_dir: Path, files: dict[str, str]) -> CompiledProject: + """Write and compile a project, keeping any compile diagnostics on it.""" + + for relative_path, contents in files.items(): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + return build_project_graph( + discovered_inputs=discover_project_inputs(project_dir=project_dir), + adapter=DuckDbAdapter(), + ).project + + +def use_sql_test_glue(*, monkeypatch: pytest.MonkeyPatch, enabled: bool) -> None: + """Route SQL-test planning through the native glue, or through the JSON request.""" + + monkeypatch.setattr( + native_planning, + "native_stage_enabled", + lambda stage: enabled and stage is NativeStage.SQL_TEST_GLUE, + ) + + +def planning_outcome( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + render_sql: bool, +) -> PlanningCallOutcome: + """The plans `plan_sql_tests_natively` returns, or what it raises.""" + + return _outcome( + lambda: native_planning.plan_sql_tests_natively( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + render_sql=render_sql, + include_plan=not render_sql, + ) + ) + + +def artifact_outcome( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + glue: bool, +) -> PlanningCallOutcome: + """Rendered artifacts from the glue or from the JSON request, or what planning raises.""" + + plan: Callable[..., tuple[NativeSqlTestArtifact, ...]] = ( + native_planning.plan_and_render_sql_test_artifacts, + native_planning.plan_compiled_sql_test_artifacts, + )[glue] + return _outcome( + lambda: plan( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + ) + ) + + +def chain_outcome( + *, project: CompiledProject, tests: tuple[CompiledSqlTest, ...] +) -> PlanningCallOutcome: + """Each test's model chain, or what chain resolution raises.""" + + return _outcome( + lambda: native_planning.resolve_sql_test_model_chains(project=project, tests=tests) + ) + + +def outcome_kind(outcome: object) -> str: + """`raised`, `errors` when a plan reports an error, or `planned`.""" + + called: PlanningCallOutcome = cast(PlanningCallOutcome, outcome) + plans: tuple[NativeSqlTestPlan, ...] = cast(tuple[NativeSqlTestPlan, ...], called.value or ()) + warnings: chain[PlanWarning] = chain.from_iterable(map(attrgetter("warnings"), plans)) + has_errors: bool = any(warning.severity is WarningSeverity.ERROR for warning in warnings) + return ("planned", "errors", "raised", "raised")[has_errors + 2 * (called.raised is not None)] + + +def _outcome(call: Callable[[], object]) -> PlanningCallOutcome: + try: + return PlanningCallOutcome(value=call(), raised=None) + except Exception as error: # noqa: BLE001 - both paths must raise the same error + return PlanningCallOutcome(value=None, raised=(type(error).__name__, str(error))) diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py new file mode 100644 index 0000000000..0ddccb4bfd --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py @@ -0,0 +1,123 @@ +"""SQL tests planned natively from compiled objects equal the JSON-request plans, test by test.""" + +from __future__ import annotations + +import random +from collections import Counter +from pathlib import Path + +import pytest + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from tests.integration.src.sqlbuild.compiler.helpers import mismatches +from tests.integration.src.sqlbuild.compiler.sql_test_glue._test_types import ( + GeneratedSqlTestPlanningParityTestCase, +) +from tests.integration.src.sqlbuild.compiler.sql_test_glue.helpers import ( + PlanningCallOutcome, + artifact_outcome, + chain_outcome, + compiled_project, + generated_sql_test_files, + outcome_kind, + planning_outcome, + use_sql_test_glue, +) + +_ADAPTERS: dict[str, BaseAdapter] = { + "duckdb": DuckDbAdapter(), + "postgres": PostgresAdapter(), + "snowflake": SnowflakeAdapter(), + "bigquery": BigQueryAdapter(), + "sqlserver": SqlServerAdapter(), +} + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedSqlTestPlanningParityTestCase( + description="mocks, expected outputs, assertions, helpers, windows and direct tests", + seed=20261009, + count=4, + test_count=12, + adapter_names=("duckdb", "postgres", "snowflake", "bigquery", "sqlserver"), + expected_minimum_planned=400, + expected_minimum_with_errors=80, + expected_minimum_raised=40, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_sql_tests_when_planning_natively_then_plans_match_the_json_request( + test_case: GeneratedSqlTestPlanningParityTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + rng: random.Random = random.Random(test_case.seed) + outcomes: Counter[str] = Counter() + differences: list[tuple[object, object, object]] = [] + for index in range(test_case.count): + project: CompiledProject = compiled_project( + project_dir=tmp_path / f"project_{index}", + files=generated_sql_test_files(rng=rng, test_count=test_case.test_count), + ) + batches: tuple[tuple[CompiledSqlTest, ...], ...] = ( + project.sql_tests, + *((test,) for test in project.sql_tests), + ) + for adapter_name in test_case.adapter_names: + adapter: BaseAdapter = _ADAPTERS[adapter_name] + for render_sql in (False, True): + inputs: list[object] = [] + expected: list[object] = [] + actual: list[object] = [] + for tests in batches: + use_sql_test_glue(monkeypatch=monkeypatch, enabled=False) + expected.append( + planning_outcome( + project=project, tests=tests, adapter=adapter, render_sql=render_sql + ) + ) + use_sql_test_glue(monkeypatch=monkeypatch, enabled=True) + actual.append( + planning_outcome( + project=project, tests=tests, adapter=adapter, render_sql=render_sql + ) + ) + inputs.append((adapter_name, render_sql, tuple(test.name for test in tests))) + differences.extend(mismatches(inputs=inputs, expected=expected, actual=actual)) + outcomes.update(map(outcome_kind, actual[1:])) + differences.extend( + mismatches( + inputs=[(adapter_name, "artifacts", tests) for tests in batches], + expected=[ + artifact_outcome(project=project, tests=tests, adapter=adapter, glue=False) + for tests in batches + ], + actual=[ + artifact_outcome(project=project, tests=tests, adapter=adapter, glue=True) + for tests in batches + ], + ) + ) + use_sql_test_glue(monkeypatch=monkeypatch, enabled=False) + python_chains: PlanningCallOutcome = chain_outcome(project=project, tests=project.sql_tests) + use_sql_test_glue(monkeypatch=monkeypatch, enabled=True) + native_chains: PlanningCallOutcome = chain_outcome(project=project, tests=project.sql_tests) + differences.extend( + mismatches(inputs=["chains"], expected=[python_chains], actual=[native_chains]) + ) + + assert ( + differences, + outcomes["planned"] >= test_case.expected_minimum_planned, + outcomes["errors"] >= test_case.expected_minimum_with_errors, + outcomes["raised"] >= test_case.expected_minimum_raised, + ) == ([], True, True, True), outcomes From 78b77be9ef871e1b0bc7dfcd22ab962867ba3f02 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:37:12 +0100 Subject: [PATCH 2/5] test(native): fold SQL-test glue checks into the existing planning, cursor-window and inspection tests --- .../sqlbuild-analysis/src/compiler/tests.rs | 2 - .../src/compiler/tests/test_sql_test_glue.rs | 217 ----------------- .../compiler/tests/test_sql_test_planning.rs | 219 +++++++++++++++++- .../main/compile/sql_test_glue/__init__.py | 0 .../main/compile/sql_test_glue/_test_types.py | 14 -- .../main/compile/sql_test_glue/helpers.py | 77 ------ .../test_native_sql_test_glue_cli.py | 172 -------------- .../cli/commands/main/test/_test_types.py | 2 + .../commands/main/test/test_cursor_window.py | 40 +++- .../cli/commands/main/test/test_test.py | 13 ++ 10 files changed, 270 insertions(+), 486 deletions(-) delete mode 100644 crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_glue.rs delete mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/__init__.py delete mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/_test_types.py delete mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/helpers.py delete mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/test_native_sql_test_glue_cli.py diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index ea9f705748..cc4183b024 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -12,8 +12,6 @@ mod sql_test_expected_columns; mod sql_test_extraction; #[path = "tests/test_sql_test_extraction_rules.rs"] mod sql_test_extraction_rules; -#[path = "tests/test_sql_test_glue.rs"] -mod sql_test_glue; #[path = "tests/test_sql_test_helper_references.rs"] mod sql_test_helper_references; #[path = "tests/test_sql_test_markers.rs"] diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_glue.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_glue.rs deleted file mode 100644 index 3976f6d8e0..0000000000 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_glue.rs +++ /dev/null @@ -1,217 +0,0 @@ -use serde_json::{Value, json}; - -use crate::compiler::main::sql_test_chain_resolution::resolve_chains_json; -use crate::compiler::main::sql_test_chains::resolve_sql_test_chains; -use crate::compiler::main::sql_test_glue::plan_sql_test_batch; -use crate::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; -use crate::compiler::main::sql_test_planning::plan_and_render_json; -use crate::compiler::models::{SqlTestPlan, SqlTestPlanWarning}; -use crate::compiler::tests::test_types::{ - GlueErrorProjectionTestCase, GluePlanParityTestCase, GluePlannerErrorTestCase, -}; - -/// A mocked model test, a test missing its source mock and a UDF test over two models. -const GLUE_REQUEST: &str = r#"{ - "lexicalSyntax": { - "backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, - "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"] - }, - "models": [ - {"name": "stg_orders", "querySql": "SELECT order_id, amount FROM __source(\"raw_orders\")"}, - { - "name": "orders", - "querySql": "WITH big AS (SELECT * FROM __ref(\"stg_orders\") WHERE __udf(\"udf__is_big\")(amount)) SELECT order_id FROM big", - "modelDependencies": ["stg_orders"] - } - ], - "functions": [{ - "name": "udf__is_big", "udfPrefix": "main.udf__is_big(", "udfSuffix": ")", - "tableFunctionPrefix": "main.udf__is_big(", "tableFunctionSuffix": ")" - }], - "tests": [ - { - "name": "mocked", "fileLabel": "tests/mocked.sql", - "payload": { - "kind": "model", - "authoredCtes": [ - {"name": "__source__raw_orders", "sqlBody": "SELECT 1 AS order_id, 5 AS amount"}, - {"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"} - ], - "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], - "expectedModelNames": ["orders"], - "assertionCtes": [{ - "name": "__assert__orders_exist", - "sqlBody": "SELECT * FROM __ref(\"orders\") WHERE 1 = 0" - }], - "readHelperNames": [], "referenceTargetModelNames": [] - } - }, - { - "name": "missing_mock", "fileLabel": "tests/missing_mock.sql", - "payload": { - "kind": "model", - "authoredCtes": [{"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"}], - "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], - "expectedModelNames": ["orders"], - "readHelperNames": [], "referenceTargetModelNames": [] - } - }, - { - "name": "udf_case", "fileLabel": "tests/udf_case.sql", - "payload": { - "kind": "direct", "mode": "udf", - "actualCte": { - "name": "__udf_actual__", "sqlBody": "SELECT __udf(\"udf__is_big\")(150) AS is_big" - }, - "expectedCte": {"name": "__udf_expected__", "sqlBody": "SELECT TRUE AS is_big"} - } - } - ] -}"#; - -#[test] -fn given_typed_requests_when_planning_then_plans_and_chains_equal_the_json_entry_points() { - let test_cases = [ - GluePlanParityTestCase { - description: "generic SQL, planned without rendering", - dialect: None, - render_sql: false, - expected_plan_count: 3, - expected_missing_mock_severity: "error", - }, - GluePlanParityTestCase { - description: "DuckDB, rendered", - dialect: Some("duckdb"), - render_sql: true, - expected_plan_count: 3, - expected_missing_mock_severity: "error", - }, - GluePlanParityTestCase { - description: "Snowflake, rendered", - dialect: Some("snowflake"), - render_sql: true, - expected_plan_count: 3, - expected_missing_mock_severity: "error", - }, - GluePlanParityTestCase { - description: "SQL Server rejects nested WITH and renders anyway", - dialect: Some("tsql"), - render_sql: true, - expected_plan_count: 3, - expected_missing_mock_severity: "error", - }, - ]; - let without_timings = |mut value: Value| { - for key in ["planningNs", "renderingNs"] { - let _ = value.as_object_mut().map(|object| object.remove(key)); - } - value - }; - for test_case in test_cases { - let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); - request["sqlAnalysisDialect"] = json!(test_case.dialect); - request["renderSql"] = json!(test_case.render_sql); - request["includePlan"] = json!(!test_case.render_sql); - let typed = plan_sql_test_batch(serde_json::from_value(request.clone()).expect("typed")) - .expect("typed plans"); - let typed: Value = without_timings(serde_json::to_value(&typed).expect("serializable")); - let json: Value = without_timings( - serde_json::from_str(&plan_and_render_json(&request.to_string()).expect("plans")) - .expect("JSON plans"), - ); - let typed_chains = resolve_sql_test_chains( - serde_json::from_value(request.clone()).expect("typed chains request"), - ) - .expect("typed chains"); - let json_chains: Value = - serde_json::from_str(&resolve_chains_json(&request.to_string()).expect("chains")) - .expect("JSON chains"); - assert_eq!( - ( - typed["artifacts"].as_array().map(Vec::len), - typed["artifacts"][1]["warnings"][0]["severity"].clone(), - typed == json, - json!({"chains": typed_chains}) == json_chains, - ), - ( - Some(test_case.expected_plan_count), - json!(test_case.expected_missing_mock_severity), - true, - true, - ), - "{}", - test_case.description - ); - } -} - -#[test] -fn given_a_test_without_compiler_reads_when_planning_then_the_typed_batch_returns_its_error() { - let test_cases = [ - GluePlannerErrorTestCase { - description: "helper reads are required for model tests", - payload_field: "readHelperNames", - expected_error_prefix: "planner_input:", - }, - GluePlannerErrorTestCase { - description: "reference targets are required for model tests", - payload_field: "referenceTargetModelNames", - expected_error_prefix: "planner_input:", - }, - ]; - for test_case in test_cases { - let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); - request["tests"][0]["payload"][test_case.payload_field] = json!(null); - let error = plan_sql_test_batch(serde_json::from_value(request).expect("typed")).err(); - assert_eq!( - error.map(|message| message.starts_with(test_case.expected_error_prefix)), - Some(true), - "{}", - test_case.description - ); - } -} - -#[test] -fn given_plan_warnings_when_projecting_errors_then_each_distinct_error_message_is_kept_in_order() { - let test_cases = [ - GlueErrorProjectionTestCase { - description: "warnings are not errors", - warnings: &[("warning", "mock is unreachable"), ("info", "note")], - expected_messages: &[], - }, - GlueErrorProjectionTestCase { - description: "repeated errors are reported once, in first-seen order", - warnings: &[ - ("error", "b has no mock"), - ("warning", "a is unreachable"), - ("error", "a has no mock"), - ("error", "b has no mock"), - ], - expected_messages: &["b has no mock", "a has no mock"], - }, - ]; - for test_case in test_cases { - let plan = SqlTestPlan { - sql: None, - chain: Vec::new(), - assertions: Vec::new(), - model_names: Vec::new(), - warnings: test_case - .warnings - .iter() - .map(|(severity, message)| SqlTestPlanWarning { - model_name: None, - severity, - message: (*message).to_owned(), - }) - .collect(), - }; - assert_eq!( - sql_test_plan_error_messages(&plan), - test_case.expected_messages.to_vec(), - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs index eb6f3a9359..79a1fe15d9 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs @@ -1,3 +1,11 @@ +use serde_json::{Value, json}; + +use crate::compiler::main::sql_test_chain_resolution::resolve_chains_json; +use crate::compiler::main::sql_test_chains::resolve_sql_test_chains; +use crate::compiler::main::sql_test_glue::plan_sql_test_batch; +use crate::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; +use crate::compiler::main::sql_test_planning::plan_and_render_json; +use crate::compiler::models::{SqlTestPlan, SqlTestPlanWarning}; use crate::compiler::tests::helpers::{ chain_resolution_orders_unmocked_models, concurrent_requests_initialize_shared_template_once, deep_shared_graph_reports_missing_mock_once, @@ -9,7 +17,10 @@ use crate::compiler::tests::helpers::{ unicode_cte_after_leading_with_preserves_identifier, unresolved_reference_fast_rejection_preserves_warning, upstream_fallback_resolves, }; -use crate::compiler::tests::test_types::SqlTestPlanningTestCase; +use crate::compiler::tests::test_types::{ + GlueErrorProjectionTestCase, GluePlanParityTestCase, GluePlannerErrorTestCase, + SqlTestPlanningTestCase, +}; #[test] fn given_sql_test_upstream_fallback_when_planning_then_descendants_and_assertions_resolve() { @@ -107,3 +118,209 @@ fn given_sql_test_planning_cases_when_exercising_native_planner_then_expected_be ); } } + +/// A mocked model test, a test missing its source mock and a UDF test over two models. +const GLUE_REQUEST: &str = r#"{ + "lexicalSyntax": { + "backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, + "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"] + }, + "models": [ + {"name": "stg_orders", "querySql": "SELECT order_id, amount FROM __source(\"raw_orders\")"}, + { + "name": "orders", + "querySql": "WITH big AS (SELECT * FROM __ref(\"stg_orders\") WHERE __udf(\"udf__is_big\")(amount)) SELECT order_id FROM big", + "modelDependencies": ["stg_orders"] + } + ], + "functions": [{ + "name": "udf__is_big", "udfPrefix": "main.udf__is_big(", "udfSuffix": ")", + "tableFunctionPrefix": "main.udf__is_big(", "tableFunctionSuffix": ")" + }], + "tests": [ + { + "name": "mocked", "fileLabel": "tests/mocked.sql", + "payload": { + "kind": "model", + "authoredCtes": [ + {"name": "__source__raw_orders", "sqlBody": "SELECT 1 AS order_id, 5 AS amount"}, + {"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"} + ], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], + "expectedModelNames": ["orders"], + "assertionCtes": [{ + "name": "__assert__orders_exist", + "sqlBody": "SELECT * FROM __ref(\"orders\") WHERE 1 = 0" + }], + "readHelperNames": [], "referenceTargetModelNames": [] + } + }, + { + "name": "missing_mock", "fileLabel": "tests/missing_mock.sql", + "payload": { + "kind": "model", + "authoredCtes": [{"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"}], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], + "expectedModelNames": ["orders"], + "readHelperNames": [], "referenceTargetModelNames": [] + } + }, + { + "name": "udf_case", "fileLabel": "tests/udf_case.sql", + "payload": { + "kind": "direct", "mode": "udf", + "actualCte": { + "name": "__udf_actual__", "sqlBody": "SELECT __udf(\"udf__is_big\")(150) AS is_big" + }, + "expectedCte": {"name": "__udf_expected__", "sqlBody": "SELECT TRUE AS is_big"} + } + } + ] +}"#; + +#[test] +fn given_typed_requests_when_planning_then_plans_and_chains_equal_the_json_entry_points() { + let test_cases = [ + GluePlanParityTestCase { + description: "generic SQL, planned without rendering", + dialect: None, + render_sql: false, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "DuckDB, rendered", + dialect: Some("duckdb"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "Snowflake, rendered", + dialect: Some("snowflake"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + GluePlanParityTestCase { + description: "SQL Server rejects nested WITH and renders anyway", + dialect: Some("tsql"), + render_sql: true, + expected_plan_count: 3, + expected_missing_mock_severity: "error", + }, + ]; + let without_timings = |mut value: Value| { + for key in ["planningNs", "renderingNs"] { + let _ = value.as_object_mut().map(|object| object.remove(key)); + } + value + }; + for test_case in test_cases { + let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); + request["sqlAnalysisDialect"] = json!(test_case.dialect); + request["renderSql"] = json!(test_case.render_sql); + request["includePlan"] = json!(!test_case.render_sql); + let typed = plan_sql_test_batch(serde_json::from_value(request.clone()).expect("typed")) + .expect("typed plans"); + let typed: Value = without_timings(serde_json::to_value(&typed).expect("serializable")); + let json: Value = without_timings( + serde_json::from_str(&plan_and_render_json(&request.to_string()).expect("plans")) + .expect("JSON plans"), + ); + let typed_chains = resolve_sql_test_chains( + serde_json::from_value(request.clone()).expect("typed chains request"), + ) + .expect("typed chains"); + let json_chains: Value = + serde_json::from_str(&resolve_chains_json(&request.to_string()).expect("chains")) + .expect("JSON chains"); + assert_eq!( + ( + typed["artifacts"].as_array().map(Vec::len), + typed["artifacts"][1]["warnings"][0]["severity"].clone(), + typed == json, + json!({"chains": typed_chains}) == json_chains, + ), + ( + Some(test_case.expected_plan_count), + json!(test_case.expected_missing_mock_severity), + true, + true, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_test_without_compiler_reads_when_planning_then_the_typed_batch_returns_its_error() { + let test_cases = [ + GluePlannerErrorTestCase { + description: "helper reads are required for model tests", + payload_field: "readHelperNames", + expected_error_prefix: "planner_input:", + }, + GluePlannerErrorTestCase { + description: "reference targets are required for model tests", + payload_field: "referenceTargetModelNames", + expected_error_prefix: "planner_input:", + }, + ]; + for test_case in test_cases { + let mut request: Value = serde_json::from_str(GLUE_REQUEST).expect("valid request"); + request["tests"][0]["payload"][test_case.payload_field] = json!(null); + let error = plan_sql_test_batch(serde_json::from_value(request).expect("typed")).err(); + assert_eq!( + error.map(|message| message.starts_with(test_case.expected_error_prefix)), + Some(true), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_plan_warnings_when_projecting_errors_then_each_distinct_error_message_is_kept_in_order() { + let test_cases = [ + GlueErrorProjectionTestCase { + description: "warnings are not errors", + warnings: &[("warning", "mock is unreachable"), ("info", "note")], + expected_messages: &[], + }, + GlueErrorProjectionTestCase { + description: "repeated errors are reported once, in first-seen order", + warnings: &[ + ("error", "b has no mock"), + ("warning", "a is unreachable"), + ("error", "a has no mock"), + ("error", "b has no mock"), + ], + expected_messages: &["b has no mock", "a has no mock"], + }, + ]; + for test_case in test_cases { + let plan = SqlTestPlan { + sql: None, + chain: Vec::new(), + assertions: Vec::new(), + model_names: Vec::new(), + warnings: test_case + .warnings + .iter() + .map(|(severity, message)| SqlTestPlanWarning { + model_name: None, + severity, + message: (*message).to_owned(), + }) + .collect(), + }; + assert_eq!( + sql_test_plan_error_messages(&plan), + test_case.expected_messages.to_vec(), + "{}", + test_case.description + ); + } +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/_test_types.py deleted file mode 100644 index 23f002f41f..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/_test_types.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class NativeSqlTestGlueCliTestCase: - """A DuckDB project whose SQL tests every engine must compile, inspect and run alike.""" - - description: str - files: dict[str, str] - expected_compile_returncode: int - expected_test_returncode: int - expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/helpers.py deleted file mode 100644 index e4e8c564f7..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/helpers.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import json -import re -import subprocess -from pathlib import Path -from typing import NamedTuple - -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR -from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import run_installed_sqb - -_DURATION: re.Pattern[str] = re.compile(r"\(\d+(?:\.\d+)?m?s\)|\d+\.\d+s") -_VOLATILE_REPORT_KEYS: tuple[str, ...] = ("compiler_engine", "compile_timings") -_VOLATILE_RESULT_KEYS: frozenset[str] = frozenset({"duration_ms"}) - - -class EngineSqlTestRun(NamedTuple): - """One engine's compile report and test SQL, test plan inspection and test results.""" - - compile_returncode: int - compile_report: str - compiled_tests: dict[str, str] - inspect_returncode: int - inspect_output: str - test_returncode: int - test_results: str - - -def engine_sql_test_run(*, root: Path, files: dict[str, str], engine: str) -> EngineSqlTestRun: - """Write the project, then compile, inspect and run its SQL tests under one engine.""" - - project_dir: Path = root / engine - for relative_path, contents in files.items(): - path: Path = project_dir / relative_path - path.parent.mkdir(parents=True, exist_ok=True) - _ = path.write_text(contents, encoding="utf-8") - environment: dict[str, str] = {REUSE_DISABLE_ENV_VAR: "1"} - engine_args: tuple[str, ...] = ("--compiler-engine", engine) - compiled: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir, - args=(*engine_args, "compile", "--json", "--no-cache"), - env=environment, - ) - report: dict[str, object] = json.loads(compiled.stdout.replace(str(project_dir), "")) - for key in _VOLATILE_REPORT_KEYS: - _ = report.pop(key, None) - tests_dir: Path = project_dir / "target" / "compiled" / "tests" - inspected: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir, args=(*engine_args, "test", "--inspect"), env=environment - ) - tested: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir, args=(*engine_args, "test", "--json"), env=environment - ) - return EngineSqlTestRun( - compile_returncode=compiled.returncode, - compile_report=json.dumps(report, indent=2), - compiled_tests={ - path.relative_to(tests_dir).as_posix(): path.read_text("utf-8") - for path in sorted(tests_dir.rglob("*.sql")) - }, - inspect_returncode=inspected.returncode, - inspect_output=_DURATION.sub("", inspected.stdout + inspected.stderr), - test_returncode=tested.returncode, - test_results=_stable_results(tested.stdout.replace(str(project_dir), "")), - ) - - -def _stable_results(stdout: str) -> str: - try: - payload: object = json.loads(stdout, object_hook=_without_durations) - except json.JSONDecodeError: - return _DURATION.sub("", stdout) - return json.dumps(payload, indent=2, sort_keys=True) - - -def _without_durations(value: dict[str, object]) -> dict[str, object]: - return {key: value[key] for key in value.keys() - _VOLATILE_RESULT_KEYS} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/test_native_sql_test_glue_cli.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/test_native_sql_test_glue_cli.py deleted file mode 100644 index 104c4e19f9..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_test_glue/test_native_sql_test_glue_cli.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Every engine compiles, inspects and runs SQL tests alike while the preview plans them natively.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_test_glue._test_types import ( - NativeSqlTestGlueCliTestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.compile.sql_test_glue.helpers import ( - EngineSqlTestRun, - engine_sql_test_run, -) - -_ENGINES: tuple[str, ...] = ("python", "native", "native-preview") -_ORDER_ROWS: str = ( - "SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount, " - "TIMESTAMP '2026-02-01 10:00:00' AS ordered_at\n" - " UNION ALL SELECT 2, 11, 20, TIMESTAMP '2026-02-05 09:00:00'" -) -_PROJECT: dict[str, str] = { - "sqlbuild_project.toml": ( - 'name = "orders_testing"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n description: Orders feed.\n expression: >-\n" - " (SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount,\n" - " TIMESTAMP '2026-02-01 10:00:00' AS ordered_at)\n" - " columns:\n - name: order_id\n type: INTEGER\n" - " - name: customer_id\n type: INTEGER\n" - " - name: amount\n type: INTEGER\n" - " - name: ordered_at\n type: TIMESTAMP\n" - " - name: raw_customers\n description: Customers feed.\n expression: >-\n" - " (SELECT 10 AS customer_id, 'east' AS region)\n" - " columns:\n - name: customer_id\n type: INTEGER\n" - " - name: region\n type: VARCHAR\n" - ), - "macros/amounts.py": ('def cents(column: str) -> str:\n return f"({column}) * 100"\n'), - "models/staging/stg_orders.sql": ( - 'MODEL (description "Staged orders");\n\n' - 'SELECT order_id, customer_id, amount, ordered_at FROM __source("raw_orders")\n' - ), - "models/staging/stg_customers.sql": ( - 'MODEL (description "Staged customers");\n\n' - 'SELECT customer_id, region FROM __source("raw_customers") WHERE region IS NOT NULL\n' - ), - "models/marts/order_regions.sql": ( - 'MODEL (description "Orders with regions");\n\n' - 'WITH big AS (\n SELECT * FROM __ref("stg_orders") WHERE amount > 100\n)\n' - 'SELECT b.order_id, c.region, @cents("b.amount") AS amount_cents\n' - 'FROM big b JOIN __ref("stg_customers") c ON b.customer_id = c.customer_id\n' - ), - "models/marts/daily_orders.sql": ( - "MODEL (description 'Daily orders',\n materialized incremental,\n" - " incremental_strategy delete_insert,\n cursor ordered_at,\n cursor_type timestamp,\n" - " cursor_grain day,\n cursor_inputs (raw_orders ordered_at,),\n);\n\n" - 'SELECT ordered_at, amount\nFROM __source("raw_orders")\n' - "WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end()\n" - ), - "tests/unit/test_order_regions.sql": ( - 'TEST (name "big_orders_get_regions");\n\nWITH\n' - f"__source__raw_orders AS (\n {_ORDER_ROWS}\n),\n" - "__source__raw_customers AS (SELECT 10 AS customer_id, 'east' AS region),\n" - "east AS (SELECT 'east' AS region),\n" - "__expected__order_regions AS (\n" - " SELECT 1 AS order_id, region, 15000 AS amount_cents FROM east\n),\n" - "__assert__amounts_are_positive AS (\n" - ' SELECT * FROM __ref("order_regions") WHERE amount_cents <= 0\n)\n' - "SELECT 1\n\n" - 'TEST (name "mocked_cents");\n\nWITH\n' - "__macro__cents AS (SELECT 'amount'),\n" - "__ref__stg_orders AS (\n" - " SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount,\n" - " TIMESTAMP '2026-02-01' AS ordered_at\n),\n" - "__ref__stg_customers AS (SELECT 10 AS customer_id, 'east' AS region),\n" - "__expected__order_regions AS (\n" - " SELECT 1 AS order_id, 'east' AS region, 150 AS amount_cents\n)\n" - "SELECT 1\n\n" - 'TEST (name "wrong_region");\n\nWITH\n' - "__ref__stg_orders AS (\n" - " SELECT 1 AS order_id, 10 AS customer_id, 150 AS amount,\n" - " TIMESTAMP '2026-02-01' AS ordered_at\n),\n" - "__ref__stg_customers AS (SELECT 10 AS customer_id, 'east' AS region),\n" - "__expected__order_regions AS (\n" - " SELECT 1 AS order_id, 'west' AS region, 15000 AS amount_cents\n)\n" - "SELECT 1\n" - ), - "tests/unit/test_daily_orders.sql": ( - 'TEST (name "default_window");\n\nWITH\n' - f"__source__raw_orders AS (\n {_ORDER_ROWS}\n),\n" - "__expected__daily_orders AS (\n" - " SELECT TIMESTAMP '2026-02-01 10:00:00' AS ordered_at, 150 AS amount\n" - " UNION ALL SELECT TIMESTAMP '2026-02-05 09:00:00' AS ordered_at, 20 AS amount\n)\n" - "SELECT 1\n\n" - 'TEST (name "declared_window", cursor_start "2026-02-01", cursor_end "2026-02-03");\n\n' - f"WITH\n__source__raw_orders AS (\n {_ORDER_ROWS}\n),\n" - "__expected__daily_orders AS (\n" - " SELECT TIMESTAMP '2026-02-01 10:00:00' AS ordered_at, 150 AS amount\n)\n" - "SELECT 1\n" - ), - "tests/unit/test_cents.sql": ( - 'TEST (mode macro, name "doubles_cents");\n\nWITH\n' - "amounts AS (SELECT 2 AS amount),\n" - '__macro_actual__ AS (SELECT @cents("amount") AS cents FROM amounts),\n' - "__macro_expected__ AS (SELECT 200 AS cents)\nSELECT 1\n" - ), -} -_MISSING_MOCK_TEST: str = ( - 'TEST (name "regions_without_customers");\n\nWITH\n' - f"__source__raw_orders AS (\n {_ORDER_ROWS}\n),\n" - "__expected__order_regions AS (\n" - " SELECT 1 AS order_id, 'east' AS region, 15000 AS amount_cents\n)\n" - "SELECT 1\n" -) - - -@pytest.mark.parametrize( - "test_case", - [ - NativeSqlTestGlueCliTestCase( - description="mocks, helpers, assertions, macro mocks, cursor windows and macro tests", - files=_PROJECT, - expected_compile_returncode=0, - expected_test_returncode=1, - expected_fragments=( - "real models: stg_customers, stg_orders, order_regions", - "boundary: stg_orders is replaced by __ref__stg_orders", - '"status": "fail"', - '"status": "pass"', - ), - ), - NativeSqlTestGlueCliTestCase( - description="a test missing a source mock fails planning with the planner's error", - files={**_PROJECT, "tests/unit/test_missing_mock.sql": _MISSING_MOCK_TEST}, - expected_compile_returncode=1, - expected_test_returncode=1, - expected_fragments=( - "test 'regions_without_customers': model 'stg_customers' references " - '__source(\\"raw_customers\\") which has no mock', - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_sql_tests_when_compiling_inspecting_and_testing_then_every_engine_agrees( - test_case: NativeSqlTestGlueCliTestCase, tmp_path: Path -) -> None: - runs: dict[str, EngineSqlTestRun] = { - engine: engine_sql_test_run(root=tmp_path, files=test_case.files, engine=engine) - for engine in _ENGINES - } - python_run: EngineSqlTestRun = runs["python"] - combined_output: str = ( - python_run.compile_report + python_run.inspect_output + python_run.test_results - ) - - assert ( - {engine: run == python_run for engine, run in runs.items()}, - python_run.compile_returncode, - python_run.test_returncode, - bool(python_run.compiled_tests) or python_run.compile_returncode != 0, - tuple(fragment in combined_output for fragment in test_case.expected_fragments), - ) == ( - dict.fromkeys(_ENGINES, True), - test_case.expected_compile_returncode, - test_case.expected_test_returncode, - True, - (True,) * len(test_case.expected_fragments), - ), combined_output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 403c5373fb..24d8b32133 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -48,6 +48,7 @@ class SqlTestPlanInspectionE2ETestCase: expected_stdout_fragments: tuple[str, ...] expected_exit_code: int = 0 unexpected_stdout_fragments: tuple[str, ...] = () + engine_env: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) @@ -178,6 +179,7 @@ class CursorWindowE2ETestCase: expected_exit_code: int expected_output_fragments: tuple[str, ...] expected_compiled_fragments: tuple[str, ...] = () + engine_env: tuple[tuple[str, str], ...] = () @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py index cac1a1ad82..ed17a867e3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py @@ -19,6 +19,8 @@ "SQL test 'declared_window_case' in 'tests/unit/declared_window_case.sql'" ) +_PREVIEW_ENGINE: tuple[tuple[str, str], ...] = (("SQLBUILD_COMPILER_ENGINE", "native-preview"),) + @pytest.mark.parametrize( "test_case", @@ -33,6 +35,17 @@ "TIMESTAMP '2999-12-31 00:00:00'", ), ), + CursorWindowE2ETestCase( + description="windows planned natively on the native-preview engine", + tests=CURSOR_WINDOW_PASSING_TESTS, + expected_exit_code=0, + expected_output_fragments=("PASS=6 FAIL=0 TOTAL=6",), + expected_compiled_fragments=( + "TIMESTAMP '1900-01-01 00:00:00'", + "TIMESTAMP '2999-12-31 00:00:00'", + ), + engine_env=_PREVIEW_ENGINE, + ), ), ids=lambda case: case.description, ) @@ -45,11 +58,12 @@ def test_given_models_using_cursor_intrinsics_when_testing_then_window_is_render project_name="cursor_window_project", repo_files=build_cursor_window_project_files(tests=test_case.tests), ) + engine_env: dict[str, str] = dict(test_case.engine_env) test_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "test"), project_dir=project_dir + command=("--no-color", "test"), project_dir=project_dir, env=engine_env ) compile_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "compile"), project_dir=project_dir + command=("--no-color", "compile"), project_dir=project_dir, env=engine_env ) output: str = test_result.stdout + test_result.stderr @@ -94,6 +108,22 @@ def test_given_models_using_cursor_intrinsics_when_testing_then_window_is_render ), ), ), + CursorWindowE2ETestCase( + description="inverted declared window on the native-preview engine", + tests=build_declared_window_test( + window='cursor_start "2026-02-03", cursor_end "2026-02-01"', + model_name="daily_orders", + ), + expected_exit_code=1, + expected_output_fragments=( + ( + _DECLARED_ERROR_PREFIX + + ": cursor_start '2026-02-03' must be before the exclusive cursor_end " + "'2026-02-01' for model 'daily_orders'" + ), + ), + engine_env=_PREVIEW_ENGINE, + ), CursorWindowE2ETestCase( description="invalid timestamp bound", tests=build_declared_window_test( @@ -167,7 +197,11 @@ def test_given_invalid_declared_cursor_window_when_testing_and_compiling_then_er ) results: tuple[subprocess.CompletedProcess[str], ...] = tuple( - run_sqb(command=command, project_dir=project_dir) + run_sqb( + command=command, + project_dir=project_dir, + env=dict(test_case.engine_env), + ) for command in (("--no-color", "test"), ("--no-color", "compile")) ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py index e03de5d349..ce59820a8f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py @@ -116,6 +116,8 @@ "expected output 'orders' lists columns that model 'orders' does not output: region" ) +_PREVIEW_ENGINE: tuple[tuple[str, str], ...] = (("SQLBUILD_COMPILER_ENGINE", "native-preview"),) + @pytest.mark.parametrize( "test_case", @@ -536,6 +538,16 @@ def test_given_invalid_fixture_when_testing_then_static_diagnostics_prevent_conn "boundary: stg_orders is replaced by __ref__stg_orders", ), ), + SqlTestPlanInspectionE2ETestCase( + description="ref mock boundary planned natively on the native-preview engine", + repo_files=build_mock_boundary_test_project_files(), + expected_stdout_fragments=( + "mocked refs: stg_orders", + "real models: int_orders, fact_orders", + "boundary: stg_orders is replaced by __ref__stg_orders", + ), + engine_env=_PREVIEW_ENGINE, + ), SqlTestPlanInspectionE2ETestCase( description="recursive derived columns are not required from the mocked input", repo_files=build_recursive_ref_fixture_project_files(), @@ -570,6 +582,7 @@ def test_given_sql_test_when_inspecting_then_fixture_boundaries_and_chain_are_sh result: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "test", "--inspect"), project_dir=project_dir, + env=dict(test_case.engine_env), ) assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr From bcb2bba2a316dc319a3ed59a00cf0495500d1f72 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:37:55 +0100 Subject: [PATCH 3/5] test(native): declare the planning request before the planning tests --- .../compiler/tests/test_sql_test_planning.rs | 118 +++++++++--------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs index 79a1fe15d9..a54b22b9de 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs @@ -22,6 +22,65 @@ use crate::compiler::tests::test_types::{ SqlTestPlanningTestCase, }; +/// A mocked model test, a test missing its source mock and a UDF test over two models. +const GLUE_REQUEST: &str = r#"{ + "lexicalSyntax": { + "backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, + "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"] + }, + "models": [ + {"name": "stg_orders", "querySql": "SELECT order_id, amount FROM __source(\"raw_orders\")"}, + { + "name": "orders", + "querySql": "WITH big AS (SELECT * FROM __ref(\"stg_orders\") WHERE __udf(\"udf__is_big\")(amount)) SELECT order_id FROM big", + "modelDependencies": ["stg_orders"] + } + ], + "functions": [{ + "name": "udf__is_big", "udfPrefix": "main.udf__is_big(", "udfSuffix": ")", + "tableFunctionPrefix": "main.udf__is_big(", "tableFunctionSuffix": ")" + }], + "tests": [ + { + "name": "mocked", "fileLabel": "tests/mocked.sql", + "payload": { + "kind": "model", + "authoredCtes": [ + {"name": "__source__raw_orders", "sqlBody": "SELECT 1 AS order_id, 5 AS amount"}, + {"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"} + ], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], + "expectedModelNames": ["orders"], + "assertionCtes": [{ + "name": "__assert__orders_exist", + "sqlBody": "SELECT * FROM __ref(\"orders\") WHERE 1 = 0" + }], + "readHelperNames": [], "referenceTargetModelNames": [] + } + }, + { + "name": "missing_mock", "fileLabel": "tests/missing_mock.sql", + "payload": { + "kind": "model", + "authoredCtes": [{"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"}], + "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], + "expectedModelNames": ["orders"], + "readHelperNames": [], "referenceTargetModelNames": [] + } + }, + { + "name": "udf_case", "fileLabel": "tests/udf_case.sql", + "payload": { + "kind": "direct", "mode": "udf", + "actualCte": { + "name": "__udf_actual__", "sqlBody": "SELECT __udf(\"udf__is_big\")(150) AS is_big" + }, + "expectedCte": {"name": "__udf_expected__", "sqlBody": "SELECT TRUE AS is_big"} + } + } + ] +}"#; + #[test] fn given_sql_test_upstream_fallback_when_planning_then_descendants_and_assertions_resolve() { let test_cases = [SqlTestPlanningTestCase { @@ -119,65 +178,6 @@ fn given_sql_test_planning_cases_when_exercising_native_planner_then_expected_be } } -/// A mocked model test, a test missing its source mock and a UDF test over two models. -const GLUE_REQUEST: &str = r#"{ - "lexicalSyntax": { - "backslashEscapeQuotes": [], "escapeStringPrefix": false, "rawStringPrefix": false, - "tripleQuotedStrings": false, "nestedBlockComments": false, "lineCommentPrefixes": ["--"] - }, - "models": [ - {"name": "stg_orders", "querySql": "SELECT order_id, amount FROM __source(\"raw_orders\")"}, - { - "name": "orders", - "querySql": "WITH big AS (SELECT * FROM __ref(\"stg_orders\") WHERE __udf(\"udf__is_big\")(amount)) SELECT order_id FROM big", - "modelDependencies": ["stg_orders"] - } - ], - "functions": [{ - "name": "udf__is_big", "udfPrefix": "main.udf__is_big(", "udfSuffix": ")", - "tableFunctionPrefix": "main.udf__is_big(", "tableFunctionSuffix": ")" - }], - "tests": [ - { - "name": "mocked", "fileLabel": "tests/mocked.sql", - "payload": { - "kind": "model", - "authoredCtes": [ - {"name": "__source__raw_orders", "sqlBody": "SELECT 1 AS order_id, 5 AS amount"}, - {"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"} - ], - "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], - "expectedModelNames": ["orders"], - "assertionCtes": [{ - "name": "__assert__orders_exist", - "sqlBody": "SELECT * FROM __ref(\"orders\") WHERE 1 = 0" - }], - "readHelperNames": [], "referenceTargetModelNames": [] - } - }, - { - "name": "missing_mock", "fileLabel": "tests/missing_mock.sql", - "payload": { - "kind": "model", - "authoredCtes": [{"name": "__seed__unused", "sqlBody": "SELECT 1 AS id"}], - "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id"}], - "expectedModelNames": ["orders"], - "readHelperNames": [], "referenceTargetModelNames": [] - } - }, - { - "name": "udf_case", "fileLabel": "tests/udf_case.sql", - "payload": { - "kind": "direct", "mode": "udf", - "actualCte": { - "name": "__udf_actual__", "sqlBody": "SELECT __udf(\"udf__is_big\")(150) AS is_big" - }, - "expectedCte": {"name": "__udf_expected__", "sqlBody": "SELECT TRUE AS is_big"} - } - } - ] -}"#; - #[test] fn given_typed_requests_when_planning_then_plans_and_chains_equal_the_json_entry_points() { let test_cases = [ From 7f366e365b76ac6f38042d0c38e404da3f16794d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:32:57 +0100 Subject: [PATCH 4/5] test(native): count only native plans and chains as native work --- .../compile/test_native_analysis_seams.py | 8 ++- .../compiler/sql_test_glue/_test_types.py | 5 +- .../compiler/sql_test_glue/helpers.py | 54 +++++++++++++++---- .../test_native_sql_test_planning_glue.py | 14 +++-- 4 files changed, 62 insertions(+), 19 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 3beafe63c7..3c7ab24016 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -3,10 +3,11 @@ from __future__ import annotations from pathlib import Path -from unittest.mock import ANY +from unittest.mock import ANY, Mock import pytest +import sqlbuild._native as native_module from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.planner.models import NativeSqlTestArtifact from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( @@ -50,6 +51,8 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ ) -> None: prepared_project: Path = tmp_path / "orders" prepare_compile_reuse_project(project_dir=prepared_project) + native_sql_test_planning: Mock = Mock(wraps=native_module.plan_compiled_sql_tests) + monkeypatch.setattr(native_module, "plan_compiled_sql_tests", native_sql_test_planning) outcomes: dict[str, tuple[CompileReuseRun, dict[str, list[object]]]] = { engine: engine_in_process_compile( project_dir=copy_compile_project( @@ -79,6 +82,9 @@ def test_given_project_when_compiling_with_each_engine_then_native_seams_answer_ assert isinstance(planned_artifacts, tuple) assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) + assert [len(call.args[0].tests) for call in native_sql_test_planning.call_args_list] == [ + len(planned_artifacts) + ] assert (native_run.returncode, preview_run.returncode) == (0, 0) assert report_without_engine(native_run) == report_without_engine(python_run) assert report_without_engine(preview_run) == report_without_engine(python_run) diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py index 3f83db0a35..de07d6ceba 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py @@ -12,6 +12,7 @@ class GeneratedSqlTestPlanningParityTestCase: count: int test_count: int adapter_names: tuple[str, ...] - expected_minimum_planned: int - expected_minimum_with_errors: int + expected_minimum_native_planned: int + expected_minimum_native_with_errors: int + expected_minimum_native_chains: int expected_minimum_raised: int diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index 27739171d5..9d284a45ca 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -3,15 +3,16 @@ from __future__ import annotations import random +from collections import Counter from collections.abc import Callable from dataclasses import dataclass from itertools import chain, compress -from operator import attrgetter from pathlib import Path from typing import cast import pytest +import sqlbuild._native as native_module from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest @@ -19,12 +20,12 @@ from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.pipeline.main.graph import build_project_graph from sqlbuild.compiler.planner._helpers.sql_tests import native_planning -from sqlbuild.compiler.planner.models import ( - NativeSqlTestArtifact, - NativeSqlTestPlan, - PlanWarning, +from sqlbuild.compiler.planner.models import NativeSqlTestArtifact +from sqlbuild.compiler.sql_test_glue.models import ( + NativeSqlTestChainRequest, + NativeSqlTestPlanningRequest, ) -from sqlbuild.compiler.planner.types import WarningSeverity +from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestPlanRow @dataclass(frozen=True) @@ -321,13 +322,44 @@ def chain_outcome( def outcome_kind(outcome: object) -> str: - """`raised`, `errors` when a plan reports an error, or `planned`.""" + """`raised` when the call raised, else `answered`; only native rows count as native work.""" called: PlanningCallOutcome = cast(PlanningCallOutcome, outcome) - plans: tuple[NativeSqlTestPlan, ...] = cast(tuple[NativeSqlTestPlan, ...], called.value or ()) - warnings: chain[PlanWarning] = chain.from_iterable(map(attrgetter("warnings"), plans)) - has_errors: bool = any(warning.severity is WarningSeverity.ERROR for warning in warnings) - return ("planned", "errors", "raised", "raised")[has_errors + 2 * (called.raised is not None)] + return ("answered", "raised")[called.raised is not None] + + +def record_native_answers(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: + """Count the plans and chains the native glue itself returns for the rest of the test. + + A plan with native error messages counts as `native_with_errors`, any other as + `native_planned`; JSON-path plans never reach these bindings, so they are never counted. + """ + + answers: Counter[str] = Counter() + plan: Callable[[NativeSqlTestPlanningRequest], tuple[list[NativeSqlTestPlanRow], int, int]] = ( + native_module.plan_compiled_sql_tests + ) + resolve: Callable[[NativeSqlTestChainRequest], list[list[str]]] = ( + native_module.resolve_compiled_sql_test_chains + ) + + def counted_plans( + request: NativeSqlTestPlanningRequest, + ) -> tuple[list[NativeSqlTestPlanRow], int, int]: + response: tuple[list[NativeSqlTestPlanRow], int, int] = plan(request) + answers.update( + ("native_planned", "native_with_errors")[bool(row[5])] for row in response[0] + ) + return response + + def counted_chains(request: NativeSqlTestChainRequest) -> list[list[str]]: + chains: list[list[str]] = resolve(request) + answers["native_chains"] += len(chains) + return chains + + monkeypatch.setattr(native_module, "plan_compiled_sql_tests", counted_plans) + monkeypatch.setattr(native_module, "resolve_compiled_sql_test_chains", counted_chains) + return answers def _outcome(call: Callable[[], object]) -> PlanningCallOutcome: diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py index 0ddccb4bfd..cda6d62d5c 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py @@ -27,6 +27,7 @@ generated_sql_test_files, outcome_kind, planning_outcome, + record_native_answers, use_sql_test_glue, ) @@ -48,8 +49,9 @@ count=4, test_count=12, adapter_names=("duckdb", "postgres", "snowflake", "bigquery", "sqlserver"), - expected_minimum_planned=400, - expected_minimum_with_errors=80, + expected_minimum_native_planned=850, + expected_minimum_native_with_errors=150, + expected_minimum_native_chains=350, expected_minimum_raised=40, ) ], @@ -61,6 +63,7 @@ def test_given_generated_sql_tests_when_planning_natively_then_plans_match_the_j monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) + native_answers: Counter[str] = record_native_answers(monkeypatch=monkeypatch) outcomes: Counter[str] = Counter() differences: list[tuple[object, object, object]] = [] for index in range(test_case.count): @@ -117,7 +120,8 @@ def test_given_generated_sql_tests_when_planning_natively_then_plans_match_the_j assert ( differences, - outcomes["planned"] >= test_case.expected_minimum_planned, - outcomes["errors"] >= test_case.expected_minimum_with_errors, + native_answers["native_planned"] >= test_case.expected_minimum_native_planned, + native_answers["native_with_errors"] >= test_case.expected_minimum_native_with_errors, + native_answers["native_chains"] >= test_case.expected_minimum_native_chains, outcomes["raised"] >= test_case.expected_minimum_raised, - ) == ([], True, True, True), outcomes + ) == ([], True, True, True, True), (native_answers, outcomes) From 35b90e53b72e5bbc6967bcca096773a1ce89e0dd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 9 Oct 2026 08:50:32 +0100 Subject: [PATCH 5/5] test(native): gate SQL-test planning on native-raised errors and batch ordering The parity test counts only errors the glue bindings raise, reaches compile_input, planner_input and unresolved-reference errors, and adds a clean seeded variant whose multi-test batches plan. sqb test planning takes the native error messages under preview; the dead None fallback is removed. --- .../_helpers/compile/target_writer.py | 24 ++-- .../planner/_helpers/sql_tests/assembly.py | 3 +- .../_helpers/sql_tests/native_planning.py | 35 +++-- src/sqlbuild/compiler/planner/models.py | 1 + .../main/plan_native_sql_test_artifacts.py | 2 +- .../compiler/sql_test_glue/_test_types.py | 8 +- .../compiler/sql_test_glue/helpers.py | 133 ++++++++++++++---- .../test_native_sql_test_planning_glue.py | 64 +++++++-- 8 files changed, 200 insertions(+), 70 deletions(-) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index 8fda15011b..6f04c17d01 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -574,23 +574,17 @@ def _plan_static_test_artifacts( with OperationLifecycle( operation_kind="project", operation_name="sql_test_planning" ) as lifecycle: - native_artifacts: tuple[NativeSqlTestArtifact, ...] | None = ( - plan_native_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - ) + plan_artifacts: Callable[..., tuple[NativeSqlTestArtifact, ...]] = ( + plan_native_sql_test_artifacts if native_stage_enabled(NativeStage.SQL_TEST_GLUE) - else None + else plan_and_render_sql_test_artifacts + ) + native_artifacts: tuple[NativeSqlTestArtifact, ...] = plan_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, ) - if native_artifacts is None: - native_artifacts = plan_and_render_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - ) lifecycle.completed(metadata={"item_count": len(native_artifacts)}) return native_artifacts diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py index 4f01af717f..92b6bae348 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py @@ -31,7 +31,6 @@ from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( plan_sql_tests_natively, resolve_sql_test_model_chains, - sql_test_plan_error_messages, ) from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError from sqlbuild.compiler.planner.models import ( @@ -94,7 +93,7 @@ def plan_sql_tests( ) continue plan_diagnostics: tuple[str, ...] = ( - *sql_test_plan_error_messages(warnings=plan.warnings), + *plan.error_messages, *_expected_column_diagnostics( test=test, plan=plan, fixture_planning_context=validation_context ), diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py index fb0b139ef8..54c533f253 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py @@ -82,7 +82,7 @@ def plan_and_render_sql_test_artifacts( NativeSqlTestArtifact( sql=plan.sql, model_names=plan.model_names, - error_messages=sql_test_plan_error_messages(warnings=plan.warnings), + error_messages=plan.error_messages, ) ) return tuple(artifacts) @@ -98,7 +98,7 @@ def plan_compiled_sql_test_artifacts( """Plan and render SQL tests from the compiled objects, with natively projected errors.""" artifacts: list[NativeSqlTestArtifact] = [] - for plan, error_messages in _plan_compiled_sql_tests( + for plan in _plan_compiled_sql_tests( project=project, tests=tests, adapter=adapter, @@ -110,7 +110,7 @@ def plan_compiled_sql_test_artifacts( raise NativeSqlTestPlanningError("native SQL-test planning omitted rendered SQL") artifacts.append( NativeSqlTestArtifact( - sql=plan.sql, model_names=plan.model_names, error_messages=error_messages + sql=plan.sql, model_names=plan.model_names, error_messages=plan.error_messages ) ) return tuple(artifacts) @@ -138,16 +138,13 @@ def plan_sql_tests_natively( """Plan SQL-test chains and assertions in one native batch, optionally rendering SQL.""" if native_stage_enabled(NativeStage.SQL_TEST_GLUE): - return tuple( - plan - for plan, _error_messages in _plan_compiled_sql_tests( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=sql_analysis_enabled, - render_sql=render_sql, - include_plan=include_plan, - ) + return _plan_compiled_sql_tests( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=sql_analysis_enabled, + render_sql=render_sql, + include_plan=include_plan, ) return _plan_sql_tests_from_json( project=project, @@ -167,7 +164,7 @@ def _plan_compiled_sql_tests( sql_analysis_enabled: bool, render_sql: bool, include_plan: bool, -) -> tuple[tuple[NativeSqlTestPlan, tuple[str, ...]], ...]: +) -> tuple[NativeSqlTestPlan, ...]: """Plan from the compiled objects natively; Python supplies only the adapter's renderings.""" if not tests: @@ -208,9 +205,9 @@ def _plan_compiled_sql_tests( return tuple(_plan_from_row(row=row) for row in rows) -def _plan_from_row(*, row: NativeSqlTestPlanRow) -> tuple[NativeSqlTestPlan, tuple[str, ...]]: +def _plan_from_row(*, row: NativeSqlTestPlanRow) -> NativeSqlTestPlan: sql, chain, assertions, model_names, warnings, error_messages = row - plan: NativeSqlTestPlan = NativeSqlTestPlan( + return NativeSqlTestPlan( chain=tuple(_chain_step_from_row(row=step) for step in chain), assertions=tuple(_assertion_step_from_row(row=step) for step in assertions), model_names=tuple(model_names), @@ -218,9 +215,9 @@ def _plan_from_row(*, row: NativeSqlTestPlanRow) -> tuple[NativeSqlTestPlan, tup PlanWarning(model_name=model_name, severity=WarningSeverity(severity), message=message) for model_name, severity, message in warnings ), + error_messages=tuple(error_messages), sql=sql, ) - return plan, tuple(error_messages) def _assertion_step_from_row(*, row: NativeAssertionStepRow) -> SqlTestAssertionStep: @@ -466,11 +463,13 @@ def _plan_from_payload(*, value: object) -> NativeSqlTestPlan: or not isinstance(warnings, list) ): raise NativeSqlTestPlanningError("native SQL-test planning returned an invalid result") + plan_warnings: tuple[PlanWarning, ...] = tuple(_warning(value=warning) for warning in warnings) return NativeSqlTestPlan( chain=tuple(_chain_step(value=step) for step in chain), assertions=tuple(_assertion_step(value=step) for step in assertions), model_names=_string_tuple(value=payload.get("modelNames"), context="model names"), - warnings=tuple(_warning(value=warning) for warning in warnings), + warnings=plan_warnings, + error_messages=sql_test_plan_error_messages(warnings=plan_warnings), sql=sql if isinstance(sql, str) else None, ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index b726b8ff07..a3959b72e0 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -1426,6 +1426,7 @@ class NativeSqlTestPlan: assertions: tuple[SqlTestAssertionStep, ...] model_names: tuple[str, ...] warnings: tuple[PlanWarning, ...] + error_messages: tuple[str, ...] sql: str | None = None diff --git a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py index 94816d02dc..0304327f4f 100644 --- a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py +++ b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py @@ -16,7 +16,7 @@ def plan_native_sql_test_artifacts( tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter, sql_analysis_enabled: bool, -) -> tuple[NativeSqlTestArtifact, ...] | None: +) -> tuple[NativeSqlTestArtifact, ...]: """Return one artifact per test, planned from the compiled objects without a JSON request.""" return plan_compiled_sql_test_artifacts( diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py index de07d6ceba..1ead44a814 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/_test_types.py @@ -2,6 +2,8 @@ from dataclasses import dataclass +from tests.integration.src.sqlbuild.compiler.sql_test_glue.helpers import SqlTestCorpusShape + @dataclass(frozen=True) class GeneratedSqlTestPlanningParityTestCase: @@ -11,8 +13,12 @@ class GeneratedSqlTestPlanningParityTestCase: seed: int count: int test_count: int + shape: SqlTestCorpusShape adapter_names: tuple[str, ...] expected_minimum_native_planned: int expected_minimum_native_with_errors: int expected_minimum_native_chains: int - expected_minimum_raised: int + expected_minimum_native_raised: int + expected_native_raised_kinds: frozenset[str] + expected_minimum_answered_batches: int + expected_minimum_raised_outcomes: int diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py index 9d284a45ca..cfbadc548b 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/helpers.py @@ -5,7 +5,7 @@ import random from collections import Counter from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, replace from itertools import chain, compress from pathlib import Path from typing import cast @@ -15,7 +15,12 @@ import sqlbuild._native as native_module from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.compile.models import ( + CompiledModelSqlTestPayload, + CompiledProject, + CompiledSqlTest, + CompileSqlTestCte, +) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.pipeline.main.graph import build_project_graph @@ -36,6 +41,17 @@ class PlanningCallOutcome: raised: tuple[str, str] | None +@dataclass(frozen=True) +class SqlTestCorpusShape: + """The cursor windows and failing shapes a generated SQL-test project draws from.""" + + windows: tuple[str, ...] + stray_window_share: float + helper_redefinition_share: float + assertion_share: float + unflattenable_assertion_share: float + + _PROJECT_TOML: str = ( 'name = "orders_testing"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' ) @@ -159,19 +175,23 @@ class PlanningCallOutcome: "__table_fn_expected__ AS (SELECT 1 AS order_id, 150 AS amount)\nSELECT 1\n", ) _DIRECT_TEST_SHARE: float = 0.8 -_STRAY_WINDOW_SHARE: float = 0.1 _MOCK_SHARE: float = 0.85 _MACRO_MOCK_SHARE: float = 0.5 _HELPER_SHARE: float = 0.4 -_ASSERTION_SHARE: float = 0.2 -_WINDOWS: tuple[str, ...] = ( - "", - ', cursor_start "2026-02-01", cursor_end "2026-02-03"', - ', cursor_start "2026-02-03", cursor_end "2026-02-01"', +_UNMOCKED_ASSERTION: CompileSqlTestCte = CompileSqlTestCte( + name="__assert__no_returns", sql_body='SELECT order_id FROM __source("raw_returns") WHERE 1 = 0' ) +_REGIONS_ASSERTION: str = 'SELECT * FROM __ref("fct_regions") WHERE 1 = 0' +_NATIVE_ERROR_KIND_SEPARATOR: str = ":" +_NATIVE_RAISED_PREFIX: str = "native_raised:" +NO_WINDOW: str = "" +VALID_WINDOW: str = ', cursor_start "2026-02-01", cursor_end "2026-02-03"' +INVERTED_WINDOW: str = ', cursor_start "2026-02-03", cursor_end "2026-02-01"' -def generated_sql_test_files(*, rng: random.Random, test_count: int) -> dict[str, str]: +def generated_sql_test_files( + *, rng: random.Random, test_count: int, shape: SqlTestCorpusShape +) -> dict[str, str]: """A project of mocked, expected, asserted, helper, direct and failing SQL tests.""" files: dict[str, str] = { @@ -189,7 +209,7 @@ def generated_sql_test_files(*, rng: random.Random, test_count: int) -> dict[str for name, sql in _BASE_MODELS.items() ) files.update( - (f"tests/unit/test_generated_{index}.sql", _model_test(rng=rng, index=index)) + (f"tests/unit/test_generated_{index}.sql", _model_test(rng=rng, index=index, shape=shape)) for index in range(test_count) ) files.update( @@ -204,12 +224,16 @@ def generated_sql_test_files(*, rng: random.Random, test_count: int) -> dict[str return files -def _model_test(*, rng: random.Random, index: int) -> str: - """One model test: the first target's first mock, then random mocks, helper and checks.""" +def _model_test(*, rng: random.Random, index: int, shape: SqlTestCorpusShape) -> str: + """One model test: the first target's first mock, then random mocks, helper and checks. + + A helper may be read through a CTE whose nested WITH redefines it, which native planning + rejects; an assertion on a model beginning with WITH cannot be flattened for SQL Server. + """ targets: list[str] = rng.sample(sorted(_UPSTREAM), k=rng.choice((1, 1, 2))) - windowed: bool = "daily_orders" in targets or rng.random() < _STRAY_WINDOW_SHARE - window: str = rng.choice(((_WINDOWS[0],), _WINDOWS)[windowed]) + windowed: bool = "daily_orders" in targets or rng.random() < shape.stray_window_share + window: str = rng.choice(((NO_WINDOW,), shape.windows)[windowed]) fixtures: list[str] = list(chain.from_iterable(_UPSTREAM[target] for target in targets)) mocks: list[str] = [ fixtures[0], @@ -219,26 +243,42 @@ def _model_test(*, rng: random.Random, index: int) -> str: ] has_helper: bool = rng.random() < _HELPER_SHARE helper: str = f"wanted_{index}" + shadow: str = f"shadowing_{index}" + redefined: bool = rng.random() < shape.helper_redefinition_share helper_ctes: list[tuple[str, str]] = [ + ( + shadow, + f"WITH {helper} AS (SELECT 1 AS order_id) SELECT order_id FROM {helper}", + ), ( helper, - rng.choice( - ("SELECT 1 AS order_id", f'SELECT order_id FROM __ref("{rng.choice(targets)}")') - ), - ) - ][: int(has_helper)] + ( + rng.choice( + ( + "SELECT 1 AS order_id", + f'SELECT order_id FROM __ref("{rng.choice(targets)}")', + ) + ), + f"SELECT order_id FROM {shadow}", + )[redefined], + ), + ][int(not redefined) : 2 * int(has_helper)] expected_sql: str = ("SELECT 1 AS order_id", f"SELECT order_id FROM {helper}")[has_helper] checks: list[tuple[str, str]] = [ ( (f"__expected__{target}", expected_sql), (f"__assert__{target}_has_rows", f'SELECT * FROM __ref("{target}") WHERE 1 = 0'), - )[rng.random() < _ASSERTION_SHARE] + )[rng.random() < shape.assertion_share] for target in targets ] + unflattenable: list[tuple[str, str]] = [(f"__assert__regions_{index}", _REGIONS_ASSERTION)][ + : int(rng.random() < shape.unflattenable_assertion_share) + ] ctes: dict[str, str] = { **{mock: _FIXTURES[mock] for mock in mocks}, **dict(helper_ctes), **dict(checks), + **dict(unflattenable), } body: str = ",\n".join(f"{name} AS (\n {sql}\n)" for name, sql in ctes.items()) return f'TEST (name "generated_{index}"{window});\n\nWITH\n{body}\nSELECT 1\n' @@ -257,6 +297,29 @@ def compiled_project(*, project_dir: Path, files: dict[str, str]) -> CompiledPro ).project +def with_unmocked_assertion(*, project: CompiledProject) -> CompiledSqlTest: + """The first model test with an assertion calling a source nothing mocks. + + The compiler reports such a call itself, so this stands in for a reference it missed and + reaches the planner's own unresolved-reference error. + """ + + test: CompiledSqlTest = next( + filter( + lambda test: isinstance(test.payload, CompiledModelSqlTestPayload), project.sql_tests + ) + ) + payload: CompiledModelSqlTestPayload = cast(CompiledModelSqlTestPayload, test.payload) + return replace( + test, + payload=replace( + payload, + assertion_ctes=(*payload.assertion_ctes, _UNMOCKED_ASSERTION), + authored_ctes=(*payload.authored_ctes, _UNMOCKED_ASSERTION), + ), + ) + + def use_sql_test_glue(*, monkeypatch: pytest.MonkeyPatch, enabled: bool) -> None: """Route SQL-test planning through the native glue, or through the JSON request.""" @@ -329,10 +392,12 @@ def outcome_kind(outcome: object) -> str: def record_native_answers(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count the plans and chains the native glue itself returns for the rest of the test. + """Count the plans, chains and errors the native glue itself returns for the rest of the test. A plan with native error messages counts as `native_with_errors`, any other as - `native_planned`; JSON-path plans never reach these bindings, so they are never counted. + `native_planned`; an error a glue binding raises counts as `native_raised` and under + `native_raised:`. JSON-path planning never reaches these bindings, and errors Python + raises before calling them are never counted. """ answers: Counter[str] = Counter() @@ -346,14 +411,16 @@ def record_native_answers(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: def counted_plans( request: NativeSqlTestPlanningRequest, ) -> tuple[list[NativeSqlTestPlanRow], int, int]: - response: tuple[list[NativeSqlTestPlanRow], int, int] = plan(request) + response: tuple[list[NativeSqlTestPlanRow], int, int] = _counting_raises( + call=lambda: plan(request), answers=answers + ) answers.update( ("native_planned", "native_with_errors")[bool(row[5])] for row in response[0] ) return response def counted_chains(request: NativeSqlTestChainRequest) -> list[list[str]]: - chains: list[list[str]] = resolve(request) + chains: list[list[str]] = _counting_raises(call=lambda: resolve(request), answers=answers) answers["native_chains"] += len(chains) return chains @@ -362,6 +429,24 @@ def counted_chains(request: NativeSqlTestChainRequest) -> list[list[str]]: return answers +def native_raised_kinds(*, answers: Counter[str]) -> frozenset[str]: + """The kinds (message prefixes) of the errors the native glue bindings raised.""" + + return frozenset( + kind.removeprefix(_NATIVE_RAISED_PREFIX) + for kind in filter(lambda kind: kind.startswith(_NATIVE_RAISED_PREFIX), answers) + ) + + +def _counting_raises[T](*, call: Callable[[], T], answers: Counter[str]) -> T: + try: + return call() + except ValueError as error: + answers["native_raised"] += 1 + answers[_NATIVE_RAISED_PREFIX + str(error).partition(_NATIVE_ERROR_KIND_SEPARATOR)[0]] += 1 + raise + + def _outcome(call: Callable[[], object]) -> PlanningCallOutcome: try: return PlanningCallOutcome(value=call(), raised=None) diff --git a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py index cda6d62d5c..c24cf34efd 100644 --- a/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py +++ b/tests/integration/src/sqlbuild/compiler/sql_test_glue/test_native_sql_test_planning_glue.py @@ -20,15 +20,21 @@ GeneratedSqlTestPlanningParityTestCase, ) from tests.integration.src.sqlbuild.compiler.sql_test_glue.helpers import ( + INVERTED_WINDOW, + NO_WINDOW, + VALID_WINDOW, PlanningCallOutcome, + SqlTestCorpusShape, artifact_outcome, chain_outcome, compiled_project, generated_sql_test_files, + native_raised_kinds, outcome_kind, planning_outcome, record_native_answers, use_sql_test_glue, + with_unmocked_assertion, ) _ADAPTERS: dict[str, BaseAdapter] = { @@ -44,16 +50,49 @@ "test_case", [ GeneratedSqlTestPlanningParityTestCase( - description="mocks, expected outputs, assertions, helpers, windows and direct tests", + description="mocks, expected outputs, assertions, helpers, windows and failing tests", seed=20261009, count=4, test_count=12, + shape=SqlTestCorpusShape( + windows=(NO_WINDOW, VALID_WINDOW, INVERTED_WINDOW), + stray_window_share=0.1, + helper_redefinition_share=0.15, + assertion_share=0.2, + unflattenable_assertion_share=0.1, + ), adapter_names=("duckdb", "postgres", "snowflake", "bigquery", "sqlserver"), - expected_minimum_native_planned=850, - expected_minimum_native_with_errors=150, - expected_minimum_native_chains=350, - expected_minimum_raised=40, - ) + expected_minimum_native_planned=624, + expected_minimum_native_with_errors=144, + expected_minimum_native_chains=398, + expected_minimum_native_raised=102, + expected_native_raised_kinds=frozenset( + {"compile_input", "planner_input", "sql_test_reference"} + ), + expected_minimum_answered_batches=0, + expected_minimum_raised_outcomes=108, + ), + GeneratedSqlTestPlanningParityTestCase( + description="multi-test batches that plan without window or helper errors", + seed=20261010, + count=3, + test_count=12, + shape=SqlTestCorpusShape( + windows=(NO_WINDOW, VALID_WINDOW), + stray_window_share=0.0, + helper_redefinition_share=0.0, + assertion_share=0.3, + unflattenable_assertion_share=0.0, + ), + adapter_names=("duckdb", "postgres", "snowflake", "bigquery", "sqlserver"), + expected_minimum_native_planned=1068, + expected_minimum_native_with_errors=192, + expected_minimum_native_chains=122, + expected_minimum_native_raised=45, + expected_native_raised_kinds=frozenset({"sql_test_reference"}), + expected_minimum_answered_batches=30, + expected_minimum_raised_outcomes=30, + ), ], ids=lambda case: case.description, ) @@ -69,11 +108,14 @@ def test_given_generated_sql_tests_when_planning_natively_then_plans_match_the_j for index in range(test_case.count): project: CompiledProject = compiled_project( project_dir=tmp_path / f"project_{index}", - files=generated_sql_test_files(rng=rng, test_count=test_case.test_count), + files=generated_sql_test_files( + rng=rng, test_count=test_case.test_count, shape=test_case.shape + ), ) batches: tuple[tuple[CompiledSqlTest, ...], ...] = ( project.sql_tests, *((test,) for test in project.sql_tests), + (with_unmocked_assertion(project=project),), ) for adapter_name in test_case.adapter_names: adapter: BaseAdapter = _ADAPTERS[adapter_name] @@ -97,6 +139,7 @@ def test_given_generated_sql_tests_when_planning_natively_then_plans_match_the_j inputs.append((adapter_name, render_sql, tuple(test.name for test in tests))) differences.extend(mismatches(inputs=inputs, expected=expected, actual=actual)) outcomes.update(map(outcome_kind, actual[1:])) + outcomes[f"batch_{outcome_kind(actual[0])}"] += 1 differences.extend( mismatches( inputs=[(adapter_name, "artifacts", tests) for tests in batches], @@ -123,5 +166,8 @@ def test_given_generated_sql_tests_when_planning_natively_then_plans_match_the_j native_answers["native_planned"] >= test_case.expected_minimum_native_planned, native_answers["native_with_errors"] >= test_case.expected_minimum_native_with_errors, native_answers["native_chains"] >= test_case.expected_minimum_native_chains, - outcomes["raised"] >= test_case.expected_minimum_raised, - ) == ([], True, True, True, True), (native_answers, outcomes) + native_answers["native_raised"] >= test_case.expected_minimum_native_raised, + native_raised_kinds(answers=native_answers) >= test_case.expected_native_raised_kinds, + outcomes["batch_answered"] >= test_case.expected_minimum_answered_batches, + outcomes["raised"] >= test_case.expected_minimum_raised_outcomes, + ) == ([], True, True, True, True, True, True, True), (native_answers, outcomes)