diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs index b5501d357a..3dbcec432b 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs @@ -6,7 +6,7 @@ use polyglot_sql::Dialect; use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::SliceDialect; -use crate::compiler::_helpers::sql_tests::cte_sql::with_leading_ctes; +use crate::compiler::_helpers::sql_tests::cte_sql::{with_leading_ctes, with_unique_ctes}; use crate::compiler::_helpers::sql_tests::helper_names::{ helper_cte_name, rename_helper_references, }; @@ -24,6 +24,8 @@ use crate::constants::{ pub(crate) struct HelperScope { pub(crate) ctes: Vec<(String, String)>, pub(crate) reached_mocks: HashSet, + /// Mock and model CTEs resolved helpers read, which defer to the reader's own copy. + pub(crate) generated: HashSet, } /// One mock CTE other test CTEs can read by its generated name. @@ -41,6 +43,16 @@ struct ScopeHelper { cte_name: String, sql: String, tokens: Vec, + resolved: Option, +} + +/// A helper whose relation references were resolved like an assertion's, after the model chain. +pub(crate) struct ResolvedHelper { + /// The helper body with every reference replaced by the CTE that stands in for it. + pub(crate) sql: String, + /// The mock and model CTEs the resolved body reads, dependencies first. + pub(crate) lifted_ctes: Vec<(String, String)>, + pub(crate) reached_mocks: HashSet, } /// One test CTE another test CTE reads: a mock, or a helper when `mock_name` is `None`. @@ -161,6 +173,7 @@ impl ScopeGraph { cte_name: cte.name.clone(), sql: cte.sql_body.clone(), tokens: identifier_tokens(&cte.sql_body, patterns), + resolved: None, }); } let readers = test_readers(fixtures); @@ -281,6 +294,76 @@ impl ScopeGraph { order.ordered } + /// Helpers the test's expected rows and assertions read, except those a mock it uses reads. + pub(crate) fn read_helpers( + &self, + readers: &[&str], + used_mocks: &HashSet, + patterns: &SqlTestPatterns, + ) -> HashSet { + let helpers = |nodes: Vec| -> HashSet { + nodes + .into_iter() + .filter_map(|node| match node { + ScopeNode::Helper(index) => Some(index), + ScopeNode::Mock(_) => None, + }) + .collect() + }; + let mut read: HashSet = HashSet::new(); + for sql in readers { + read.extend(helpers( + self.closure(&identifier_tokens(sql, patterns), true), + )); + } + for mock in &self.mocks { + if used_mocks.contains(&mock.mock_name) { + for index in helpers(self.closure(&mock.tokens, true)) { + read.remove(&index); + } + } + } + read + } + + /// Read helpers whose SQL, as placed in the test query, calls a reference such as `__ref()`. + pub(crate) fn referencing_helpers( + &self, + read: &HashSet, + patterns: &SqlTestPatterns, + ) -> Vec<(usize, String)> { + self.helpers + .iter() + .enumerate() + .filter(|(index, helper)| helper.resolved.is_none() && read.contains(index)) + .filter_map(|(index, _)| { + let sql = self.placed_helper_sql(index); + (patterns.test_reference.is_match(sql) || patterns.udf.is_match(sql)) + .then(|| (index, sql.to_string())) + }) + .collect() + } + + /// Replace a helper's references with the CTEs a reader must define ahead of it. + pub(crate) fn resolve_helper(&mut self, index: usize, resolved: ResolvedHelper) { + let placement = self.placement; + let Some(helper) = self.helpers.get_mut(index) else { + return; + }; + match placement { + HelperPlacement::TopLevel => helper.sql.clone_from(&resolved.sql), + HelperPlacement::Inline => helper.original.clone_from(&resolved.sql), + } + helper.resolved = Some(resolved); + } + + fn placed_helper_sql(&self, index: usize) -> &str { + match self.placement { + HelperPlacement::TopLevel => &self.helpers[index].sql, + HelperPlacement::Inline => &self.helpers[index].original, + } + } + /// Whether a CTE name is a test helper placed at the top level of the test query. pub(crate) fn is_top_level_helper(&self, name: &str) -> bool { self.placement == HelperPlacement::TopLevel @@ -343,6 +426,7 @@ impl ScopeGraph { let mut scope = HelperScope { ctes: Vec::new(), reached_mocks: HashSet::new(), + generated: HashSet::new(), }; for node in nodes { if let ScopeNode::Mock(index) = node { @@ -350,11 +434,31 @@ impl ScopeGraph { .reached_mocks .insert(self.mocks[index].mock_name.clone()); } + if let ScopeNode::Helper(index) = node + && let Some(resolved) = &self.helpers[index].resolved + { + scope + .reached_mocks + .extend(resolved.reached_mocks.iter().cloned()); + scope.generated.extend( + resolved + .lifted_ctes + .iter() + .map(|(name, _)| name.to_ascii_lowercase()), + ); + scope.ctes = with_unique_ctes( + std::mem::take(&mut scope.ctes), + resolved.lifted_ctes.iter().cloned(), + ); + } if let Some(dependency) = self.dependency(node, None) { - scope.ctes.push(( - dependency.generated_name.to_string(), - dependency.sql.to_string(), - )); + scope.ctes = with_unique_ctes( + std::mem::take(&mut scope.ctes), + [( + dependency.generated_name.to_string(), + dependency.sql.to_string(), + )], + ); } } scope @@ -428,6 +532,7 @@ pub(crate) fn helper_scope_ctes( return Ok(HelperScope { ctes: Vec::new(), reached_mocks: HashSet::new(), + generated: HashSet::new(), }); } let nodes = graph.closure(&identifier_tokens(sql, patterns), true); @@ -444,14 +549,31 @@ pub(crate) fn helper_scope_ctes( Ok(graph.collect_scope(nodes)) } -/// Append scoped CTEs after a step's generated CTEs, keeping one definition per name. +/// Merge scoped CTEs into a step's CTEs; relations resolved helpers read come first and win. pub(crate) fn merged_scoped_ctes( - mut lifted: Vec<(String, String)>, - scoped: Vec<(String, String)>, + lifted: Vec<(String, String)>, + scoped: HelperScope, file_label: &str, ) -> Result, String> { - for (name, sql) in scoped { - match lifted + type Ctes = Vec<(String, String)>; + let (generated, authored): (Ctes, Ctes) = scoped + .ctes + .into_iter() + .partition(|(name, _)| scoped.generated.contains(&name.to_ascii_lowercase())); + let mut merged: Vec<(String, String)> = lifted; + if !generated.is_empty() { + let own = std::mem::replace(&mut merged, generated); + for (name, sql) in own { + if !merged + .iter() + .any(|(existing, _)| existing.eq_ignore_ascii_case(&name)) + { + merged.push((name, sql)); + } + } + } + for (name, sql) in authored { + match merged .iter() .find(|(existing, _)| existing.eq_ignore_ascii_case(&name)) { @@ -461,10 +583,10 @@ pub(crate) fn merged_scoped_ctes( "SQL test '{file_label}' defines CTE '{name}', which conflicts with the generated CTE" ))); } - None => lifted.push((name, sql)), + None => merged.push((name, sql)), } } - Ok(lifted) + Ok(merged) } /// Names the comparison renderer generates, which a scoped helper CTE must not shadow. diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index 801b526570..a163db7201 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -17,7 +17,7 @@ use crate::compiler::_helpers::sql_tests::cte_sql::{ use crate::compiler::_helpers::sql_tests::expected_columns::expected_columns; use crate::compiler::_helpers::sql_tests::helper_names::HELPER_PREFIX; use crate::compiler::_helpers::sql_tests::helper_scope::{ - ScopeGraph, ScopeRequest, helper_scope_ctes, merged_scoped_ctes, + ResolvedHelper, ScopeGraph, ScopeRequest, helper_scope_ctes, merged_scoped_ctes, }; use crate::compiler::_helpers::sql_tests::markers::{ ProtectedRanges, marker_names, marker_names_in, replace_callable_markers, @@ -154,6 +154,8 @@ enum TestPayload { expected_model_names: Vec, #[serde(default)] assertion_ctes: Vec, + #[serde(default)] + helper_target_model_names: Vec, }, Direct { mode: String, @@ -223,10 +225,10 @@ pub(crate) struct SqlTestPatterns { reference: Regex, source: Regex, seed: Regex, - udf: Regex, + pub(crate) udf: Regex, table_function: Regex, dbt_reference: Regex, - test_reference: Regex, + pub(crate) test_reference: Regex, pub(crate) lexical: LexicalSyntax, pub(crate) identifier: Regex, } @@ -300,6 +302,7 @@ struct ModelTestPlan { expected_ctes: Vec, expected_model_names: Vec, assertion_ctes: Vec, + helper_target_model_names: Vec, } struct TextualChainRequest<'a> { @@ -579,10 +582,16 @@ pub(crate) fn resolve_chains_json(request_json: &str) -> Result model_query_overrides, expected_model_names, assertion_ctes, + helper_target_model_names, .. } => { let fixtures = classify_fixtures(authored_ctes, Vec::new(), assertion_ctes); - let expected_names = chain_root_names(expected_model_names, &fixtures, &patterns); + let expected_names = chain_root_names( + expected_model_names, + helper_target_model_names, + &fixtures, + &patterns, + ); topo_sort_model_chain(TopoSortRequest { expected_names: &expected_names, models: &models, @@ -596,12 +605,15 @@ pub(crate) fn resolve_chains_json(request_json: &str) -> Result serde_json::to_string(&ChainBatchResponse { chains }).map_err(|error| error.to_string()) } +/// Models the test runs: expected models, assertion targets, and the compiler's helper targets. fn chain_root_names( mut expected_names: Vec, + helper_target_names: Vec, fixtures: &TestFixtures, patterns: &SqlTestPatterns, ) -> Vec { expected_names.extend(assertion_ref_targets(&fixtures.assertions, patterns)); + expected_names.extend(helper_target_names); dedupe(expected_names) } @@ -735,6 +747,7 @@ fn plan_test(test: TestInput, context: &ProjectContext) -> Result plan_model_test( ModelTestPlan { test_name: test.name, @@ -744,6 +757,7 @@ fn plan_test(test: TestInput, context: &ProjectContext) -> Result = Vec::new(); - let mut textual_assertion_chain: Option = None; for (assertion_name, assertion_sql) in &fixtures.assertions { let placed_sql = fixtures.scope.reader_sql(assertion_sql, &context.patterns); - let table_functions = - resolve_table_function_fixtures(&placed_sql, &fixtures, &context.patterns)?; - reachable_mocks.extend(table_functions.reached.iter().cloned()); - let fixture_resolved_sql = &table_functions.sql; - let analyzed = - if context.sql_analysis_enabled && analysis_resolved.len() == ordered_names.len() { - analyze_and_resolve_sql(AnalysisResolutionRequest { - query_sql: fixture_resolved_sql, - fixture_ctes: &table_functions.ctes, - fixtures: &fixtures, - resolved_chain: &analysis_resolved, - functions: &context.functions, - file_label: &plan.file_label, - dialect_name: &context.dialect, - templates: &context.analysis_templates, - patterns: &context.patterns, - })? - } else { - None - }; - let (mut resolved_sql, mut lifted_ctes, comparison_body_sql) = match analyzed { - Some(value) - if !has_unresolved_test_reference(&value.resolved_sql, &context.patterns) => - { - reachable_mocks.extend(value.reachable_mock_names.iter().cloned()); - ( - value.resolved_sql, - value.generated_ctes, - Some(value.cte_body_sql), - ) - } - _ => { - if textual_assertion_chain.is_none() { - let (chain, reached) = build_textual_chain( - &ordered_names, - &plan.model_query_overrides, - &fixtures, - context, - )?; - reachable_mocks.extend(reached); - textual_assertion_chain = Some(chain); - } - let chain = textual_assertion_chain - .as_ref() - .ok_or_else(|| planner_error("textual assertion chain is unavailable"))?; - let (resolved, reached) = - resolve_assertion_textual_sql(AssertionResolutionRequest { - assertion_sql: fixture_resolved_sql, - fixtures: &fixtures, - chain, - functions: &context.functions, - requires_flat_ctes: context.requires_derived_table_aliases, - patterns: &context.patterns, - })?; - reachable_mocks.extend(reached); - let lifted = with_unique_ctes(table_functions.ctes.clone(), resolved.lifted_ctes); - ( - with_leading_ctes(&lifted, &resolved.body_sql), - lifted, - Some(resolved.body_sql), - ) - } - }; + let resolved = readers.resolve(&placed_sql, &fixtures)?; + reachable_mocks.extend(resolved.chain_reached); + reachable_mocks.extend(resolved.reached); + let mut resolved_sql = resolved.resolved_sql; + let mut lifted_ctes = resolved.lifted_ctes; + let comparison_body_sql = Some(resolved.body_sql); let scope = helper_scope_ctes( assertion_sql, &fixtures, @@ -1004,8 +980,8 @@ fn plan_model_test( &plan.file_label, )?; if !scope.ctes.is_empty() { - reachable_mocks.extend(scope.reached_mocks); - lifted_ctes = merged_scoped_ctes(lifted_ctes, scope.ctes, &plan.file_label)?; + reachable_mocks.extend(scope.reached_mocks.iter().cloned()); + lifted_ctes = merged_scoped_ctes(lifted_ctes, scope, &plan.file_label)?; resolved_sql = with_leading_ctes( &lifted_ctes, comparison_body_sql.as_deref().unwrap_or(&resolved_sql), @@ -1040,6 +1016,150 @@ fn plan_model_test( }) } +/// The planned chain and fixtures whose helpers are resolved once every chain model is known. +struct HelperResolution<'r, 'a> { + readers: &'r mut ReaderContext<'a>, + fixtures: &'r mut TestFixtures, + chain: &'r mut [ChainStep], + reachable_mocks: &'r mut HashSet, +} + +/// Resolve the references helpers call, then give expected rows the CTEs their helpers now read. +fn resolve_referencing_helpers(request: HelperResolution<'_, '_>) -> Result<(), String> { + let HelperResolution { + readers, + fixtures, + chain, + reachable_mocks, + } = request; + let patterns = &readers.context.patterns; + let read_sql: Vec<&str> = fixtures + .expected + .values() + .chain(fixtures.assertions.iter().map(|(_, sql)| sql)) + .map(String::as_str) + .collect(); + let read_helpers = fixtures + .scope + .read_helpers(&read_sql, reachable_mocks, patterns); + let mut resolved_helpers: Vec<(usize, ResolvedHelper)> = Vec::new(); + for (index, helper_sql) in fixtures.scope.referencing_helpers(&read_helpers, patterns) { + let resolved = readers.resolve(&helper_sql, fixtures)?; + reachable_mocks.extend(resolved.chain_reached); + resolved_helpers.push(( + index, + ResolvedHelper { + sql: resolved.body_sql, + lifted_ctes: resolved.lifted_ctes, + reached_mocks: resolved.reached, + }, + )); + } + if resolved_helpers.is_empty() { + return Ok(()); + } + for (index, resolved) in resolved_helpers { + fixtures.scope.resolve_helper(index, resolved); + } + for step in chain { + if let Some(sql) = fixtures.expected.get(&step.model_name) { + let scope = helper_scope_ctes(sql, fixtures, patterns, readers.file_label)?; + reachable_mocks.extend(scope.reached_mocks); + step.expected_lifted_ctes = scope.ctes; + step.expected_cte_sql = Some(fixtures.scope.reader_sql(sql, patterns)); + } + } + Ok(()) +} + +/// What resolving one test-authored query's relation references needs once the chain is planned. +struct ReaderContext<'a> { + context: &'a ProjectContext, + ordered_names: &'a [String], + overrides: &'a BTreeMap, + file_label: &'a str, + analysis_resolved: &'a HashMap, + textual_chain: Option, +} + +/// One assertion or helper query with its references resolved to the CTEs standing in for them. +struct ResolvedReader { + resolved_sql: String, + lifted_ctes: Vec<(String, String)>, + body_sql: String, + reached: HashSet, + /// Mocks the shared textual chain reached when this query first needed it. + chain_reached: HashSet, +} + +impl ReaderContext<'_> { + /// Resolve an assertion or helper query against the planned chain, mocks and functions. + fn resolve(&mut self, sql: &str, fixtures: &TestFixtures) -> Result { + let context = self.context; + let table_functions = resolve_table_function_fixtures(sql, fixtures, &context.patterns)?; + let mut reached: HashSet = table_functions.reached.clone(); + let fixture_resolved_sql = &table_functions.sql; + let analyzed = if context.sql_analysis_enabled + && self.analysis_resolved.len() == self.ordered_names.len() + { + analyze_and_resolve_sql(AnalysisResolutionRequest { + query_sql: fixture_resolved_sql, + fixture_ctes: &table_functions.ctes, + fixtures, + resolved_chain: self.analysis_resolved, + functions: &context.functions, + file_label: self.file_label, + dialect_name: &context.dialect, + templates: &context.analysis_templates, + patterns: &context.patterns, + })? + } else { + None + }; + if let Some(value) = analyzed + && !has_unresolved_test_reference(&value.resolved_sql, &context.patterns) + { + reached.extend(value.reachable_mock_names); + return Ok(ResolvedReader { + resolved_sql: value.resolved_sql, + lifted_ctes: value.generated_ctes, + body_sql: value.cte_body_sql, + reached, + chain_reached: HashSet::new(), + }); + } + let mut chain_reached: HashSet = HashSet::new(); + if self.textual_chain.is_none() { + let (chain, chain_mocks) = + build_textual_chain(self.ordered_names, self.overrides, fixtures, context)?; + chain_reached = chain_mocks; + self.textual_chain = Some(chain); + } + let chain = self + .textual_chain + .as_ref() + .ok_or_else(|| planner_error("textual assertion chain is unavailable"))?; + let (resolved, textual_reached) = + resolve_assertion_textual_sql(AssertionResolutionRequest { + assertion_sql: fixture_resolved_sql, + fixtures, + chain, + functions: &context.functions, + requires_flat_ctes: context.requires_derived_table_aliases, + patterns: &context.patterns, + })?; + reached.extend(textual_reached); + let lifted = with_unique_ctes(table_functions.ctes, resolved.lifted_ctes); + Ok(ResolvedReader { + resolved_sql: with_leading_ctes(&lifted, &resolved.body_sql), + lifted_ctes: lifted, + body_sql: resolved.body_sql, + reached, + chain_reached, + }) + } +} + /// Drop SQL from chain steps the renderer never emits, keeping plan output linear in chain length. fn omit_unrendered_step_sql(chain: Vec, assertions: &[AssertionStep]) -> Vec { let rendered_steps = rendered_chain_steps(&chain, assertions); diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index 9b28b3cb13..b3c3a19419 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -8,6 +8,8 @@ mod sql_test_cte_slices; mod sql_test_expected_columns; #[path = "tests/test_sql_test_extraction.rs"] mod sql_test_extraction; +#[path = "tests/test_sql_test_helper_references.rs"] +mod sql_test_helper_references; #[path = "tests/test_sql_test_markers.rs"] mod sql_test_markers; #[path = "tests/test_sql_test_name_isolation.rs"] diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index a006e32514..88b54d4943 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -9,7 +9,9 @@ use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::{SliceDialect, split_top_level_with}; use crate::compiler::_helpers::sql_tests::relation_markers::relation_marker_calls; use crate::compiler::main::sql_test_extraction::extract_batch_json; -use crate::compiler::tests::test_types::{PlanShape, RenderedShapeTestCase, SharedNameTestCase}; +use crate::compiler::tests::test_types::{ + HelperReferenceTestCase, PlanShape, RenderedShapeTestCase, SharedNameTestCase, +}; pub(crate) fn mixed_expanded_tests_preserve_order_and_payloads() -> bool { let response = extract_batch_json(r#"{"tests":[{"sql":"WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT id FROM helper), __expected__orders AS (SELECT id FROM helper), __assert__positive AS (SELECT id FROM helper WHERE id < 0) SELECT 1","fileLabel":"tests/orders.sql","mode":"model"},{"sql":"WITH input AS (SELECT 1 AS value), __udf_actual__ AS (SELECT __udf(\"increment\")(value) AS value FROM input), __udf_expected__ AS (SELECT 2 AS value) SELECT 1","fileLabel":"tests/increment.sql","mode":"udf"}]}"#).expect("batch succeeds"); @@ -1801,3 +1803,95 @@ fn assert_names_isolated(sql: &str, dialect: &str, helper_name: &str, label: &st ); } } + +fn helper_reference_test(case: &HelperReferenceTestCase) -> Value { + let named = |ctes: &[(&str, &str)]| -> Vec { + ctes.iter() + .map(|(name, sql)| json!({"name": name, "sqlBody": sql})) + .collect() + }; + let mut authored = named(&[ + ( + "__source__raw_orders", + "SELECT 1 AS order_id, 10 AS amount, 7 AS region_id", + ), + ( + "__seed__regions", + "SELECT 7 AS region_id, 'north' AS region_name", + ), + ]); + authored.extend(named(case.helpers)); + json!({ + "name": "orders_case", + "fileLabel": "tests/orders.sql", + "payload": { + "kind": "model", + "authoredCtes": authored, + "expectedCtes": named(case.expected), + "expectedModelNames": case + .expected + .iter() + .map(|(name, _)| name.trim_start_matches("__expected__")) + .collect::>(), + "assertionCtes": named(case.assertions), + "helperTargetModelNames": case.helper_targets + } + }) +} + +fn helper_reference_models() -> Value { + json!([ + { + "name": "stg_orders", + "querySql": "SELECT order_id, amount, region_id FROM __source(\"raw_orders\")", + "modelDependencies": [] + }, + { + "name": "orders", + "querySql": "SELECT order_id, amount * 2 AS amount_doubled FROM __ref(\"stg_orders\")", + "modelDependencies": ["stg_orders"] + } + ]) +} + +/// Plan and render one helper-reference case, returning its SQL, chain and warnings. +pub(crate) fn plan_helper_reference_case(case: &HelperReferenceTestCase) -> Value { + let response: Value = serde_json::from_str( + &crate::compiler::main::sql_test_planning::plan_and_render_json( + &json!({ + "lexicalSyntax": generic_lexical_syntax(), + "models": helper_reference_models(), + "tests": [helper_reference_test(case)], + "sqlAnalysisEnabled": case.sql_analysis_enabled, + "sqlAnalysisDialect": "duckdb", + "setDifferenceOperator": "EXCEPT" + }) + .to_string(), + ) + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + response["artifacts"][0].clone() +} + +/// Resolve one helper-reference case's model chain without planning SQL. +pub(crate) fn chain_helper_reference_case(case: &HelperReferenceTestCase) -> Value { + let response: Value = serde_json::from_str( + &crate::compiler::main::sql_test_chain_resolution::resolve_chains_json( + &json!({ + "lexicalSyntax": generic_lexical_syntax(), + "models": helper_reference_models(), + "tests": [helper_reference_test(case)] + }) + .to_string(), + ) + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + response["chains"][0].clone() +} + +/// Whether `first` is defined before `second` at the top level of a rendered test query. +pub(crate) fn defined_before(sql: &str, first: &str, second: &str) -> bool { + top_level_cte_position(sql, first) < top_level_cte_position(sql, second) +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs new file mode 100644 index 0000000000..f05fd17ccf --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs @@ -0,0 +1,222 @@ +use serde_json::{Value, json}; + +use crate::compiler::tests::helpers::{ + chain_helper_reference_case, defined_before, plan_helper_reference_case, +}; +use crate::compiler::tests::test_types::HelperReferenceTestCase; + +const HELPER_READS_MODEL: (&str, &str) = ( + "doubled", + "SELECT order_id, amount_doubled FROM __ref(\"orders\")", +); +const ASSERT_DOUBLED: (&str, &str) = ( + "__assert__doubles_amount", + "SELECT order_id FROM doubled WHERE amount_doubled <> 20", +); + +#[test] +fn given_helper_cte_references_when_planning_then_references_resolve_in_dependency_order() { + let test_cases = [ + HelperReferenceTestCase { + description: "helper reading a model is emitted after the model and its mocks", + sql_analysis_enabled: true, + helpers: &[HELPER_READS_MODEL], + expected: &[], + assertions: &[ASSERT_DOUBLED], + helper_targets: &["orders"], + expected_chain: &["stg_orders", "orders"], + expected_order: &[ + ("__source__raw_orders", "__ref__stg_orders"), + ("__ref__stg_orders", "__ref__orders"), + ("__ref__orders", "__helper__doubled"), + ("__helper__doubled", "__assert__doubles_amount"), + ], + expected_fragments: &[ + "__helper__doubled AS (SELECT order_id, amount_doubled FROM __ref__orders)", + ], + expected_absent_fragments: &[], + }, + HelperReferenceTestCase { + description: "helper inlined without sql analysis reads the model CTE", + sql_analysis_enabled: false, + helpers: &[HELPER_READS_MODEL], + expected: &[], + assertions: &[ASSERT_DOUBLED], + helper_targets: &["orders"], + expected_chain: &["stg_orders", "orders"], + expected_order: &[ + ("__source__raw_orders", "__ref__orders"), + ("__ref__orders", "__assert__doubles_amount"), + ], + expected_fragments: &[ + "WITH doubled AS (SELECT order_id, amount_doubled FROM __ref__orders)", + ], + expected_absent_fragments: &[], + }, + HelperReferenceTestCase { + description: "helper reading another helper, a model and a seed is ordered after all three", + sql_analysis_enabled: true, + helpers: &[ + ( + "with_region", + "SELECT b.order_id, r.region_name FROM base AS b \ + JOIN __ref(\"stg_orders\") AS s USING (order_id) \ + JOIN __seed(\"regions\") AS r USING (region_id)", + ), + ( + "base", + "SELECT order_id, amount_doubled FROM __ref(\"orders\")", + ), + ], + expected: &[], + assertions: &[( + "__assert__has_region", + "SELECT order_id FROM with_region WHERE region_name IS NULL", + )], + helper_targets: &["stg_orders", "orders"], + expected_chain: &["stg_orders", "orders"], + expected_order: &[ + ("__ref__orders", "__helper__base"), + ("__helper__base", "__helper__with_region"), + ("__ref__stg_orders", "__helper__with_region"), + ("__seed__regions", "__helper__with_region"), + ("__helper__with_region", "__assert__has_region"), + ], + expected_fragments: &[ + "FROM __helper__base AS b JOIN __ref__stg_orders AS s USING (order_id) \ + JOIN __seed__regions AS r USING (region_id)", + ], + expected_absent_fragments: &[], + }, + HelperReferenceTestCase { + description: "helper read by expected rows resolves sources and seeds to their mocks", + sql_analysis_enabled: true, + helpers: &[( + "expected_rows", + "SELECT s.order_id, s.amount * 2 AS amount_doubled FROM __source(\"raw_orders\") AS s \ + JOIN __seed(\"regions\") AS r USING (region_id)", + )], + expected: &[( + "__expected__orders", + "SELECT order_id, amount_doubled FROM expected_rows", + )], + assertions: &[], + helper_targets: &[], + expected_chain: &["stg_orders", "orders"], + expected_order: &[ + ("__source__raw_orders", "__helper__expected_rows"), + ("__seed__regions", "__helper__expected_rows"), + ("__helper__expected_rows", "__expected__orders"), + ], + expected_fragments: &[ + "FROM __source__raw_orders AS s JOIN __seed__regions AS r USING (region_id)", + ], + expected_absent_fragments: &[], + }, + HelperReferenceTestCase { + description: "inlined helper read by expected rows reads the model CTE", + sql_analysis_enabled: false, + helpers: &[HELPER_READS_MODEL], + expected: &[( + "__expected__orders", + "SELECT order_id, amount_doubled FROM doubled", + )], + assertions: &[], + helper_targets: &["orders"], + expected_chain: &["stg_orders", "orders"], + expected_order: &[("__ref__orders", "__expected__orders")], + expected_fragments: &[ + "__expected__orders AS (WITH doubled AS (SELECT order_id, amount_doubled FROM __ref__orders)", + ], + expected_absent_fragments: &["__helper__"], + }, + HelperReferenceTestCase { + description: "helper and assertion reading a mocked model through different paths share one mock CTE", + sql_analysis_enabled: true, + helpers: &[ + ( + "__ref__stg_orders", + "SELECT order_id, 10 AS amount FROM base_ids", + ), + ("base_ids", "SELECT 1 AS order_id"), + ( + "doubled", + "SELECT o.order_id, o.amount_doubled FROM __ref(\"orders\") AS o", + ), + ], + expected: &[], + assertions: &[( + "__assert__doubles_amount", + "SELECT order_id FROM doubled JOIN __ref(\"stg_orders\") USING (order_id) \ + WHERE amount_doubled <> 20", + )], + helper_targets: &["orders"], + expected_chain: &["orders", "stg_orders"], + expected_order: &[ + ("__helper__base_ids", "__ref__orders"), + ("__ref__orders", "__helper__doubled"), + ("__helper__doubled", "__assert__doubles_amount"), + ], + expected_fragments: &[], + expected_absent_fragments: &[], + }, + HelperReferenceTestCase { + description: "helper nothing reads is neither resolved nor emitted", + sql_analysis_enabled: true, + helpers: &[("unused_rows", "SELECT order_id FROM __ref(\"orders\")")], + expected: &[( + "__expected__stg_orders", + "SELECT 1 AS order_id, 10 AS amount, 7 AS region_id", + )], + assertions: &[], + helper_targets: &[], + expected_chain: &["stg_orders"], + expected_order: &[("__source__raw_orders", "__expected__stg_orders")], + expected_fragments: &[], + expected_absent_fragments: &["unused_rows", "__ref__orders"], + }, + ]; + for test_case in test_cases { + let artifact: Value = plan_helper_reference_case(&test_case); + let chain: Value = chain_helper_reference_case(&test_case); + let sql = artifact["sql"].as_str().expect("test assumption must hold"); + let errors: Vec<&Value> = artifact["warnings"] + .as_array() + .expect("test assumption must hold") + .iter() + .filter(|warning| warning["severity"] == json!("error")) + .collect(); + + assert_eq!( + chain, + json!(test_case.expected_chain), + "{}", + test_case.description + ); + for (first, second) in test_case.expected_order { + assert!( + defined_before(sql, first, second), + "{}: {first} before {second}: {sql}", + test_case.description + ); + } + for fragment in test_case.expected_absent_fragments { + assert!( + !sql.contains(fragment), + "{}: {fragment}: {sql}", + test_case.description + ); + } + for fragment in test_case.expected_fragments { + assert!( + sql.contains(fragment), + "{}: {fragment}: {sql}", + test_case.description + ); + } + for marker in ["__ref(", "__source(", "__seed("] { + assert!(!sql.contains(marker), "{}: {sql}", test_case.description); + } + assert!(errors.is_empty(), "{}: {errors:?}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 0dbecaf8ec..39a9489144 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -118,3 +118,16 @@ pub(crate) struct PlanShape<'a> { pub(crate) helper_name: &'a str, pub(crate) expected_sql: &'a str, } + +pub(crate) struct HelperReferenceTestCase { + pub(crate) description: &'static str, + pub(crate) sql_analysis_enabled: bool, + pub(crate) helpers: &'static [(&'static str, &'static str)], + pub(crate) expected: &'static [(&'static str, &'static str)], + pub(crate) assertions: &'static [(&'static str, &'static str)], + pub(crate) helper_targets: &'static [&'static str], + pub(crate) expected_chain: &'static [&'static str], + pub(crate) expected_order: &'static [(&'static str, &'static str)], + pub(crate) expected_fragments: &'static [&'static str], + pub(crate) expected_absent_fragments: &'static [&'static str], +} diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index cfe2c9ebcd..257a265ffa 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -245,6 +245,23 @@ def _reference_cases() -> tuple[FailureCase, ...]: ) }, ), + failure_case( + name="test-helper-reads-unknown-model", + expected_code="P013", + expected_message="SQL test helper CTE 'archived_orders' references unknown model", + files={ + "tests/unit/test_stg_orders.sql": ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status\n),\n" + 'archived_orders AS (\n SELECT order_id FROM __ref("archived_orders")\n),\n' + "__assert__no_archived_orders AS (\n" + ' SELECT order_id FROM __ref("stg_orders")\n' + " JOIN archived_orders USING (order_id)\n" + ")\nSELECT 1\n" + ) + }, + ), failure_case( name="dbt-reference-without-manifest", expected_code="C214", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 0e912fe427..5eadc2239a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -165,6 +165,36 @@ __assert__no_unexpected_order_ids AS ( SQLBuild emits each helper and mock a query needs once, in dependency order, including mocks that a helper reaches through another mock. +Helpers can also call the references an `__assert__` CTE can, such as `__ref()`, `__source()` and +`__seed()`, and they resolve exactly as there: a mocked relation reads its mock, and an unmocked +`__ref("")` reads the model's real SQL, so the test runs that model and +`sqb test --select ` selects the test even when only a helper reads it. SQLBuild emits a +helper after the mocks and models it reads: + +```sql +TEST(); + +WITH +__ref__stg_orders AS ( + SELECT 1 AS order_id, 3 AS quantity +), +order_totals AS ( + SELECT order_id, quantity FROM __ref("orders") +), +__assert__quantities_are_positive AS ( + SELECT order_id FROM order_totals WHERE quantity <= 0 +) +``` + +Only helpers the test reads count: those its `__expected__` and `__assert__` CTEs reach, directly +or through other helpers and mocks. A helper nothing reads is left out of the test query, and its +references are neither checked nor run. Compile rejects references the test query cannot resolve +with `P013`, at the helper or call that causes it: a `__ref()` to a model that does not exist, +helpers or mocks that read each other in a cycle, and a mock or `__table_fn__` fixture the test +uses that reads a helper calling a reference. Mocks are defined before the models a test runs, so +a mock reads other mocks by CTE name, such as `FROM __ref__stg_orders`, rather than through +`__ref("stg_orders")`. + ## Multi-model tests Tests can span multiple models in a single file. Mock your sources, define an expected output for the model you care about, and SQLBuild automatically resolves every intermediate model using its real SQL. diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 36529c2ea9..0d8e1b8c9d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -93,6 +93,9 @@ ) from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile._helpers.sharing.binding import shareable_binding_prekeys +from sqlbuild.compiler.compile._helpers.sql_tests.helper_ctes import ( + report_mocks_reading_referencing_helpers, +) from sqlbuild.compiler.compile._helpers.sql_tests.identity import build_sql_test_case_fingerprint from sqlbuild.compiler.compile.classes.stored_model_analyses import StoredModelAnalyses from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME @@ -1746,9 +1749,23 @@ def _assemble_compiled_sql_test( model_payload: CompileModelSqlTestInputPayload = test_input.payload target_model_names = tuple( dict.fromkeys( - (*model_payload.expected_model_names, *model_payload.assertion_target_model_names) + ( + *model_payload.expected_model_names, + *model_payload.assertion_target_model_names, + *model_payload.helper_target_model_names, + ) ) ) + report_mocks_reading_referencing_helpers( + authored_ctes=model_payload.authored_ctes, + reader_ctes=(*model_payload.expected_ctes, *model_payload.assertion_ctes), + model_inputs=model_inputs, + target_model_names=target_model_names, + mock_model_names=model_payload.mock_model_names, + test_file=test_input.test_file, + test_block=test_input.test_block, + syntax=inputs.sql_lexical_syntax, + ) scope_deps = sql_test_scope_deps(expected_model_names=target_model_names) compiled_payload = CompiledModelSqlTestPayload( authored_ctes=model_payload.authored_ctes, @@ -1824,6 +1841,11 @@ def _assemble_compiled_sql_test( if isinstance(test_input.payload, CompileModelSqlTestInputPayload) else () ), + helper_target_model_names=( + test_input.payload.helper_target_model_names + if isinstance(test_input.payload, CompileModelSqlTestInputPayload) + else () + ), target_model_names=target_model_names, tested_resources=tested_resources, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index 8c62622748..e471b8d787 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -33,6 +33,10 @@ extract_assertion_target_model_names, extract_sql_test_ctes, ) +from sqlbuild.compiler.compile._helpers.sql_tests.helper_ctes import ( + helper_target_model_names, + report_unresolvable_helper_ctes, +) from sqlbuild.compiler.compile._helpers.sql_tests.native import ( extract_expanded_sql_tests_cached, ) @@ -48,6 +52,7 @@ CompileSqlScenarioCte, CompileSqlScenarioCtes, CompileSqlScenarioInput, + CompileSqlTestCte, CompileSqlTestCtes, CompileSqlTestInput, DeclarationExpansionContext, @@ -317,6 +322,28 @@ def build_test_inputs( loaded_macros=loaded_macros, assertion_target_model_names=assertion_target_model_names, ) + helper_targets: tuple[str, ...] = () + if isinstance(test_ctes.payload, CompileModelSqlTestCtes): + reader_ctes: tuple[CompileSqlTestCte, ...] = ( + *test_ctes.payload.expected_ctes, + *test_ctes.payload.assertion_ctes, + ) + if report_unresolvable_helper_ctes( + authored_ctes=test_ctes.payload.authored_ctes, + reader_ctes=reader_ctes, + mock_model_names=test_ctes.payload.mock_model_names, + test_file=test.test_file, + test_block=test.test_block, + known_model_names=known_model_names, + syntax=sql_lexical_syntax, + ): + continue + helper_targets = helper_target_model_names( + authored_ctes=test_ctes.payload.authored_ctes, + reader_ctes=reader_ctes, + mock_model_names=test_ctes.payload.mock_model_names, + syntax=sql_lexical_syntax, + ) test_inputs.append( CompileSqlTestInput( test_file=test.test_file, @@ -327,6 +354,7 @@ def build_test_inputs( test_ctes=test_ctes, tested_resource_names=test.tested_resource_names, assertion_target_model_names=assertion_target_model_names, + helper_target_model_names=helper_targets, ), declaration_usages=test.declaration_usages, parent_name=test.parent_name, @@ -388,6 +416,7 @@ def _build_test_input_payload( test_ctes: CompileSqlTestCtes, tested_resource_names: tuple[str, ...], assertion_target_model_names: tuple[str, ...], + helper_target_model_names: tuple[str, ...], ) -> CompileModelSqlTestInputPayload | CompileDirectLogicSqlTestInputPayload: match test_ctes.payload: case CompileModelSqlTestCtes() as model_payload: @@ -404,6 +433,7 @@ def _build_test_input_payload( assertion_ctes=model_payload.assertion_ctes, assertion_names=model_payload.assertion_names, assertion_target_model_names=assertion_target_model_names, + helper_target_model_names=helper_target_model_names, ) case CompileDirectLogicSqlTestCtes() as direct_logic_payload: return CompileDirectLogicSqlTestInputPayload( diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py new file mode 100644 index 0000000000..1563f60c57 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py @@ -0,0 +1,458 @@ +"""Relation references in the SQL-test helper CTEs a test reads, and what cannot resolve.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from typing import Any + +from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic +from sqlbuild.compiler.compile._helpers.refs.references import ( + reference_call_location, + scan_sql_reference_calls, +) +from sqlbuild.compiler.compile.constants import ( + DBT_REF_TEST_CTE_PREFIX, + REF_TEST_CTE_PREFIX, + SEED_TEST_CTE_PREFIX, + SOURCE_TEST_CTE_PREFIX, + SQL_TEST_HELPER_REFERENCE_CODE, + TABLE_FN_TEST_CTE_PREFIX, +) +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + CompilerDiagnostic, + CompileSqlReference, + CompileSqlTestCte, + SqlTestCteGraph, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.spec.contracts.models import SourceLocation + +_MOCK_CTE_PREFIXES: tuple[str, ...] = ( + REF_TEST_CTE_PREFIX, + SOURCE_TEST_CTE_PREFIX, + SEED_TEST_CTE_PREFIX, + DBT_REF_TEST_CTE_PREFIX, + TABLE_FN_TEST_CTE_PREFIX, +) +_RELATION_REFERENCE_KINDS: frozenset[SqlReferenceKind] = frozenset( + { + SqlReferenceKind.REF, + SqlReferenceKind.SOURCE, + SqlReferenceKind.SEED, + SqlReferenceKind.DBT_REF, + SqlReferenceKind.TABLE_FUNCTION, + } +) +_MOCK_PREFIX_BY_KIND: dict[SqlReferenceKind, str] = { + SqlReferenceKind.REF: REF_TEST_CTE_PREFIX, + SqlReferenceKind.SOURCE: SOURCE_TEST_CTE_PREFIX, + SqlReferenceKind.SEED: SEED_TEST_CTE_PREFIX, + SqlReferenceKind.DBT_REF: DBT_REF_TEST_CTE_PREFIX, + SqlReferenceKind.TABLE_FUNCTION: TABLE_FN_TEST_CTE_PREFIX, +} +_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"[A-Za-z_][\w$]*") + + +def sql_test_cte_graph( + *, + authored_ctes: tuple[CompileSqlTestCte, ...], + reader_ctes: tuple[CompileSqlTestCte, ...], +) -> SqlTestCteGraph: + """Return the authored CTEs each test CTE reads, ignoring names its own nested CTEs define.""" + + ctes: dict[str, CompileSqlTestCte] = {cte.name.casefold(): cte for cte in authored_ctes} + keys: frozenset[str] = frozenset(ctes) + reads: dict[str, tuple[str, ...]] = {} + parsed_reads: dict[str, tuple[str, ...]] = {} + for key, cte in ctes.items(): + parsed: tuple[str, ...] | None = _parsed_reads(sql=cte.sql_body, keys=keys - {key}) + parsed_reads[key] = parsed or () + reads[key] = ( + parsed if parsed is not None else _token_reads(sql=cte.sql_body, keys=keys - {key}) + ) + reader_reads: list[str] = [] + for reader in reader_ctes: + parsed_reader: tuple[str, ...] | None = _parsed_reads(sql=reader.sql_body, keys=keys) + reader_reads.extend( + parsed_reader + if parsed_reader is not None + else _token_reads(sql=reader.sql_body, keys=keys) + ) + return SqlTestCteGraph( + ctes=ctes, + reads=reads, + parsed_reads=parsed_reads, + reader_reads=tuple(dict.fromkeys(reader_reads)), + ) + + +def reachable_cte_keys(*, graph: SqlTestCteGraph, roots: Iterable[str]) -> tuple[str, ...]: + """Return the authored CTEs reachable from ``roots``, including the roots, in visit order.""" + + pending: list[str] = [root for root in roots if root in graph.ctes] + visited: list[str] = list(dict.fromkeys(pending)) + while pending: + key: str = pending.pop(0) + for dependency in graph.reads.get(key, ()): + if dependency not in visited: + visited.append(dependency) + pending.append(dependency) + return tuple(visited) + + +def helper_target_model_names( + *, + authored_ctes: tuple[CompileSqlTestCte, ...], + reader_ctes: tuple[CompileSqlTestCte, ...], + mock_model_names: tuple[str, ...], + syntax: SqlLexicalSyntax, +) -> tuple[str, ...]: + """Return unmocked models that helpers the test reads call ``__ref()`` on.""" + + graph: SqlTestCteGraph = sql_test_cte_graph( + authored_ctes=authored_ctes, reader_ctes=reader_ctes + ) + mocked: frozenset[str] = frozenset(mock_model_names) + targets: list[str] = [] + for key in _read_helper_keys(graph): + targets.extend( + reference.ref_name + for reference in _references(sql=graph.ctes[key].sql_body, syntax=syntax) + if reference.ref_kind == SqlReferenceKind.REF and reference.ref_name not in mocked + ) + return tuple(dict.fromkeys(targets)) + + +def report_unresolvable_helper_ctes( + *, + authored_ctes: tuple[CompileSqlTestCte, ...], + reader_ctes: tuple[CompileSqlTestCte, ...], + mock_model_names: tuple[str, ...], + test_file: DiscoveredSqlTestFile, + test_block: DiscoveredSqlTestBlock, + known_model_names: set[str], + syntax: SqlLexicalSyntax, +) -> bool: + """Report read helpers with unknown models or read cycles; return whether any was found.""" + + graph: SqlTestCteGraph = sql_test_cte_graph( + authored_ctes=authored_ctes, reader_ctes=reader_ctes + ) + read_helpers: tuple[str, ...] = _read_helper_keys(graph) + if not read_helpers: + return False + reporter: _HelperDiagnostics = _HelperDiagnostics(test_file=test_file, test_block=test_block) + mocked: frozenset[str] = frozenset(mock_model_names) + for key in read_helpers: + cte: CompileSqlTestCte = graph.ctes[key] + for reference in _references(sql=cte.sql_body, syntax=syntax): + if ( + reference.ref_kind == SqlReferenceKind.REF + and reference.ref_name not in mocked + and reference.ref_name not in known_model_names + ): + reporter.report( + cte_name=cte.name, + call=SqlReferenceKind.REF.example_call(reference.ref_name, quote='"'), + message=( + f"SQL test helper CTE '{cte.name}' references unknown model " + f"'{reference.ref_name}'" + ), + help=( + 'Read a model defined in the project, for example __ref(""), ' + f"or remove the reference from helper CTE '{cte.name}'." + ), + ) + cycle: tuple[str, ...] | None = _first_cycle( + graph=graph, keys=reachable_cte_keys(graph=graph, roots=graph.reader_reads) + ) + if cycle is not None: + names: tuple[str, ...] = tuple(graph.ctes[key].name for key in cycle) + path: str = " -> ".join(f"'{name}'" for name in names) + reporter.report( + cte_name=names[0], + call=None, + message=( + f"SQL test CTE '{names[0]}' reads itself through {path}, so the test query " + "cannot define its CTEs in dependency order" + ), + help=( + "Break the cycle by moving the shared rows into a helper CTE that reads neither, " + "for example shared_rows AS (SELECT ...), and read shared_rows from both CTEs." + ), + ) + return reporter.reported + + +def report_mocks_reading_referencing_helpers( + *, + authored_ctes: tuple[CompileSqlTestCte, ...], + reader_ctes: tuple[CompileSqlTestCte, ...], + model_inputs: tuple[CompileModelInput, ...], + target_model_names: tuple[str, ...], + mock_model_names: tuple[str, ...], + test_file: DiscoveredSqlTestFile, + test_block: DiscoveredSqlTestBlock, + syntax: SqlLexicalSyntax, +) -> None: + """Report mocks the test reads that read a helper calling a reference, which cannot resolve.""" + + referencing: dict[str, CompileSqlReference] = {} + for cte in authored_ctes: + if cte.name.startswith(_MOCK_CTE_PREFIXES): + continue + reference: CompileSqlReference | None = next( + ( + reference + for reference in _references(sql=cte.sql_body, syntax=syntax) + if reference.ref_kind in _RELATION_REFERENCE_KINDS + ), + None, + ) + if reference is not None: + referencing[cte.name.casefold()] = reference + if not referencing: + return + graph: SqlTestCteGraph = sql_test_cte_graph( + authored_ctes=authored_ctes, reader_ctes=reader_ctes + ) + used_mocks: tuple[str, ...] = _used_mock_keys( + graph=graph, + model_references={ + model_input.model_file.file_path.stem: model_input.references + for model_input in model_inputs + }, + target_model_names=target_model_names, + mocked=frozenset(mock_model_names), + ) + reporter: _HelperDiagnostics = _HelperDiagnostics(test_file=test_file, test_block=test_block) + for mock_key in used_mocks: + helper_key: str | None = next( + ( + key + for key in reachable_cte_keys(graph=graph, roots=graph.reads.get(mock_key, ())) + if key in referencing + ), + None, + ) + if helper_key is None: + continue + mock_name: str = graph.ctes[mock_key].name + helper_name: str = graph.ctes[helper_key].name + call: str = _reference_call(referencing[helper_key]) + reporter.report( + cte_name=helper_name, + call=call, + message=( + f"SQL test mock '{mock_name}' reads helper CTE '{helper_name}', which calls " + f"{call}; mocks and fixtures are defined before the models the test runs, so " + "the helper cannot be resolved for them" + ), + help=_mock_reference_help( + mock_name=mock_name, call=call, reference=referencing[helper_key] + ), + ) + + +def _read_helper_keys(graph: SqlTestCteGraph) -> tuple[str, ...]: + return tuple( + key + for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads) + if not graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES) + ) + + +def _used_mock_keys( + *, + graph: SqlTestCteGraph, + model_references: dict[str, tuple[CompileSqlReference, ...]], + target_model_names: tuple[str, ...], + mocked: frozenset[str], +) -> tuple[str, ...]: + """Return mocks the test's readers or the models it runs read, in a stable order.""" + + used: list[str] = [ + key + for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads) + if graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES) + ] + pending: list[str] = [name for name in target_model_names if name not in mocked] + visited: set[str] = set(pending) + while pending: + model_name: str = pending.pop(0) + for reference in model_references.get(model_name, ()): + kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) + for mock_key in _mock_keys(kind=kind, reference=reference): + if mock_key in graph.ctes and mock_key not in used: + used.append(mock_key) + if ( + kind is SqlReferenceKind.REF + and reference.ref_name not in mocked + and reference.ref_name not in visited + ): + visited.add(reference.ref_name) + pending.append(reference.ref_name) + return tuple(used) + + +def _mock_keys(*, kind: SqlReferenceKind, reference: CompileSqlReference) -> tuple[str, ...]: + prefix: str | None = _MOCK_PREFIX_BY_KIND.get(kind) + if prefix is None: + return () + names: list[str] = [reference.ref_name] + if reference.ref_package is not None: + names.append(f"{reference.ref_package}__{reference.ref_name}") + return tuple(f"{prefix}{name}".casefold() for name in names) + + +def _mock_reference_help(*, mock_name: str, call: str, reference: CompileSqlReference) -> str: + kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) + if kind is not SqlReferenceKind.TABLE_FUNCTION and reference.ref_package is None: + mock_cte: str = f"{kind.fixture_cte_prefix}{reference.ref_name}" + return ( + f"Read a mock by its CTE name instead, for example FROM {mock_cte} rather than " + f"FROM {call}, defining {mock_cte} AS (SELECT ...) if the test does not mock it, " + f"or write the rows of '{mock_name}' directly." + ) + return f"Write the rows of '{mock_name}' directly, for example {mock_name} AS (SELECT ...)." + + +def _reference_call(reference: CompileSqlReference) -> str: + kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) + if kind is SqlReferenceKind.DBT_REF and reference.ref_package is not None: + return kind.example_call(reference.ref_package, reference.ref_name, quote='"') + return kind.example_call(reference.ref_name, quote='"') + + +def _references(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[CompileSqlReference, ...]: + return scan_sql_reference_calls(sql=sql, syntax=syntax).references + + +def _parsed_reads(*, sql: str, keys: frozenset[str]) -> tuple[str, ...] | None: + """Return unqualified table reads of ``keys`` not shadowed by a nested CTE, or None.""" + + folded: str = sql.casefold() + if not any(key in folded for key in keys): + return () + polyglot_module: Any = import_polyglot_sql() + try: + parsed: Any = polyglot_module.parse_one(sql, dialect="generic") + except polyglot_module.PolyglotError: + return None + nested: frozenset[str] = _defined_cte_keys(parsed.to_dict()) + reads: list[str] = [] + for table in parsed.find_all("table"): + if table.arg("schema") is not None or table.arg("catalog") is not None: + continue + key: str = str(getattr(table, "name", "") or "").casefold() + if key in keys and key not in nested and key not in reads: + reads.append(key) + return tuple(reads) + + +def _defined_cte_keys(tree: Any) -> frozenset[str]: + """Return the case-folded names of every CTE a parsed query defines, at any depth.""" + + keys: set[str] = set() + pending: list[Any] = [tree] + while pending: + value: Any = pending.pop() + if isinstance(value, list): + pending.extend(value) + continue + if not isinstance(value, dict): + continue + ctes: Any = value.get("ctes") + for cte in ctes if isinstance(ctes, list) else (): + alias: Any = cte.get("alias") if isinstance(cte, dict) else None + name: Any = alias.get("name") if isinstance(alias, dict) else None + if isinstance(name, str): + keys.add(name.casefold()) + pending.extend(value.values()) + return frozenset(keys) + + +def _token_reads(*, sql: str, keys: frozenset[str]) -> tuple[str, ...]: + tokens: tuple[str, ...] = tuple( + dict.fromkeys(match.casefold() for match in _IDENTIFIER_PATTERN.findall(sql)) + ) + return tuple(token for token in tokens if token in keys) + + +def _first_cycle(*, graph: SqlTestCteGraph, keys: tuple[str, ...]) -> tuple[str, ...] | None: + """Return the first parsed read cycle among ``keys``, closed on its first CTE.""" + + allowed: frozenset[str] = frozenset(keys) + for origin in keys: + pending: list[tuple[str, ...]] = [(origin,)] + visited: set[str] = set() + while pending: + path: tuple[str, ...] = pending.pop(0) + for dependency in graph.parsed_reads.get(path[-1], ()): + if dependency == origin: + return (*path, origin) + if dependency in allowed and dependency not in visited: + visited.add(dependency) + pending.append((*path, dependency)) + return None + + +class _HelperDiagnostics: + """Located P013 diagnostics for one SQL test block.""" + + def __init__( + self, *, test_file: DiscoveredSqlTestFile, test_block: DiscoveredSqlTestBlock + ) -> None: + self._test_file: DiscoveredSqlTestFile = test_file + self._block_offset: int = max(test_file.contents.find(test_block.sql_body), 0) + self._test_name: str = test_block.name or test_file.relative_path.stem + self.reported: bool = False + + def report(self, *, cte_name: str, call: str | None, message: str, help: str) -> None: + location: SourceLocation = self._location(cte_name=cte_name, call=call) + self.reported = True + report_compile_diagnostic( + key=( + SQL_TEST_HELPER_REFERENCE_CODE, + self._test_file.relative_path.as_posix(), + f"{location.line:09d}:{location.column:09d}", + message, + ), + diagnostic=CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=SQL_TEST_HELPER_REFERENCE_CODE, + message=message, + resource_type=CompiledResourceType.SQL_TEST, + resource_name=self._test_name, + location=location, + help=help, + ), + ) + + def _location(self, *, cte_name: str, call: str | None) -> SourceLocation: + contents: str = self._test_file.contents + header: re.Match[str] | None = re.compile( + rf"(? 20\n" + "),\n" + "__assert__has_one_row AS (\n" + ' SELECT COUNT(*) AS row_count FROM __ref("item_totals") HAVING COUNT(*) <> 1\n' + ")\n" + ), + "only_helper_reads_model": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "doubled AS (\n" + ' SELECT item_id, amount_doubled FROM __ref("item_totals")\n' + "),\n" + "__assert__doubles_amount AS (\n" + " SELECT item_id FROM doubled WHERE amount_doubled <> 20\n" + ")\n" + ), + "helper_reads_helper_and_mock": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "joined AS (\n" + " SELECT d.item_id, d.amount_doubled, i.amount FROM doubled AS d\n" + ' JOIN __ref("items") AS i USING (item_id)\n' + "),\n" + "doubled AS (\n" + ' SELECT item_id, amount_doubled FROM __ref("item_totals")\n' + "),\n" + "__expected__item_totals AS (\n" + " SELECT item_id, amount_doubled FROM joined\n" + "),\n" + "__assert__doubles_mocked_amount AS (\n" + " SELECT item_id FROM joined WHERE amount_doubled <> amount * 2\n" + ")\n" + ), + "mocked_model_read_by_helper_and_assertion": ( + "__ref__items AS (\n SELECT item_id, 10 AS amount FROM base_ids\n),\n" + "base_ids AS (SELECT 1 AS item_id),\n" + 'doubled AS (SELECT t.item_id, t.amount_doubled FROM __ref("item_totals") AS t),\n' + "__assert__doubles_amount AS (\n" + ' SELECT item_id FROM doubled JOIN __ref("items") USING (item_id)\n' + " WHERE amount_doubled <> 20\n" + ")\n" + ), + "unread_helpers_and_mocks": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + 'archived_rows AS (SELECT item_id FROM __ref("item_archive")),\n' + 'other_rows AS (SELECT item_id, amount FROM __ref("item_extras")),\n' + "__ref__item_extras AS (SELECT item_id, amount FROM other_rows),\n" + "__expected__item_totals AS (\n SELECT 1 AS item_id, 20 AS amount_doubled\n)\n" + ), + "nested_cte_shadows_helper": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "expected_rows AS (\n" + " WITH doubled AS (SELECT 1 AS item_id, 20 AS amount_doubled)\n" + " SELECT item_id, amount_doubled FROM doubled\n" + "),\n" + "doubled AS (SELECT item_id, amount_doubled FROM expected_rows),\n" + "__expected__item_totals AS (\n SELECT item_id, amount_doubled FROM doubled\n)\n" + ), + "helper_cycle": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "first_rows AS (SELECT item_id FROM second_rows),\n" + "second_rows AS (SELECT item_id FROM first_rows),\n" + "__assert__no_rows AS (\n" + ' SELECT item_id FROM __ref("item_totals") JOIN first_rows USING (item_id)\n' + ")\n" + ), + "helper_reads_unknown_model": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + 'missing_rows AS (SELECT item_id FROM __ref("item_archive")),\n' + "__assert__no_rows AS (\n" + ' SELECT item_id FROM __ref("item_totals") JOIN missing_rows USING (item_id)\n' + ")\n" + ), + "mock_reads_referencing_helper": ( + "__ref__items AS (\n SELECT item_id, 10 AS amount FROM base_rows\n),\n" + 'base_rows AS (SELECT item_id FROM __ref("item_totals")),\n' + "__assert__no_rows AS (\n" + ' SELECT item_id FROM __ref("item_totals") WHERE item_id IS NULL\n' + ")\n" + ), +} + + +def build_helper_reference_project_files( + *, tests: tuple[str, ...], sql_analysis: bool = True +) -> dict[str, str]: + """Build an items project whose SQL tests read models through helper CTEs.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "helper_reference_demo"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "helper_reference_demo.duckdb"\n\n' + "[settings]\n" + f"sql_analysis = {str(sql_analysis).lower()}\n" + ), + "models/item_extras.sql": ( + "MODEL (description 'Extra item amounts', materialized table);\n\n" + 'SELECT item_id, amount FROM __ref("items")\n' + ), + "models/items.sql": ( + "MODEL (description 'Base items', materialized table);\n\n" + "SELECT 1 AS item_id, 10 AS amount\n" + ), + "models/item_totals.sql": ( + "MODEL (description 'Doubled item amounts', materialized table);\n\n" + 'SELECT item_id, amount * 2 AS amount_doubled FROM __ref("items")\n' + ), + } + for test_name in tests: + files[f"tests/unit/test_{test_name}.sql"] = ( + f'TEST (name "{test_name}");\n\nWITH\n{_HELPER_REFERENCE_TESTS[test_name]}\nSELECT 1\n' + ) + return files diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py new file mode 100644 index 0000000000..badcfe9370 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py @@ -0,0 +1,375 @@ +"""E2E tests for `__ref()` and mock references inside SQL-test helper CTEs.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + HelperRedefinitionE2ETestCase, + HelperReferenceE2ETestCase, + HelperReferenceErrorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import ( + build_helper_reference_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" + + +@pytest.mark.parametrize( + "test_case", + ( + HelperReferenceE2ETestCase( + description="helper and assertion read the model on the python engine", + tests=("helper_and_assertion_read_model",), + command=("--no-color", "test"), + engine="python", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + expected_compiled_fragments=("\n__helper__doubled AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="only a helper reads the model on the python engine", + tests=("only_helper_reads_model",), + command=("--no-color", "test", "--select", "item_totals"), + engine="python", + sql_analysis=True, + expected_output_fragments=("1 selected", "PASS=1 FAIL=0 TOTAL=1"), + expected_compiled_fragments=("\n__ref__item_totals AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="helper reads a helper and a mock on the python engine", + tests=("helper_reads_helper_and_mock",), + command=("--no-color", "test"), + engine="python", + sql_analysis=True, + expected_output_fragments=( + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_mocked_amount", + ), + expected_compiled_fragments=("\n__helper__joined AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="inlined helpers read by expected rows and assertions on the python engine", + tests=("helper_reads_helper_and_mock",), + command=("--no-color", "test"), + engine="python", + sql_analysis=False, + expected_output_fragments=( + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_mocked_amount", + ), + expected_compiled_fragments=( + "(WITH doubled AS (SELECT item_id, amount_doubled FROM __ref__item_totals)", + ), + expected_absent_compiled_fragments=("__ref(", "__helper__"), + ), + HelperReferenceE2ETestCase( + description="helper and assertion read a mocked model on the python engine", + tests=("mocked_model_read_by_helper_and_assertion",), + command=("--no-color", "test"), + engine="python", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + expected_compiled_fragments=("\n__helper__doubled AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="unread helpers and mocks are ignored on the python engine", + tests=("unread_helpers_and_mocks",), + command=("--no-color", "test"), + engine="python", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1",), + expected_compiled_fragments=(), + expected_absent_compiled_fragments=("archived_rows", "other_rows", "__ref("), + ), + HelperReferenceE2ETestCase( + description="unread helpers add no tested models on the python engine", + tests=("unread_helpers_and_mocks",), + command=("--no-color", "test", "--select", "item_extras"), + engine="python", + sql_analysis=True, + expected_output_fragments=("0 selected",), + expected_compiled_fragments=(), + expected_absent_compiled_fragments=("archived_rows", "other_rows"), + ), + HelperReferenceE2ETestCase( + description="helper and assertion read the model on the native engine", + tests=("helper_and_assertion_read_model",), + command=("--no-color", "test"), + engine="native", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + expected_compiled_fragments=("\n__helper__doubled AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="only a helper reads the model on the native engine", + tests=("only_helper_reads_model",), + command=("--no-color", "test", "--select", "item_totals"), + engine="native", + sql_analysis=True, + expected_output_fragments=("1 selected", "PASS=1 FAIL=0 TOTAL=1"), + expected_compiled_fragments=("\n__ref__item_totals AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="helper reads a helper and a mock on the native engine", + tests=("helper_reads_helper_and_mock",), + command=("--no-color", "test"), + engine="native", + sql_analysis=True, + expected_output_fragments=( + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_mocked_amount", + ), + expected_compiled_fragments=("\n__helper__joined AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="inlined helpers read by expected rows and assertions on the native engine", + tests=("helper_reads_helper_and_mock",), + command=("--no-color", "test"), + engine="native", + sql_analysis=False, + expected_output_fragments=( + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_mocked_amount", + ), + expected_compiled_fragments=( + "(WITH doubled AS (SELECT item_id, amount_doubled FROM __ref__item_totals)", + ), + expected_absent_compiled_fragments=("__ref(", "__helper__"), + ), + HelperReferenceE2ETestCase( + description="helper and assertion read a mocked model on the native engine", + tests=("mocked_model_read_by_helper_and_assertion",), + command=("--no-color", "test"), + engine="native", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + expected_compiled_fragments=("\n__helper__doubled AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="unread helpers and mocks are ignored on the native engine", + tests=("unread_helpers_and_mocks",), + command=("--no-color", "test"), + engine="native", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1",), + expected_compiled_fragments=(), + expected_absent_compiled_fragments=("archived_rows", "other_rows", "__ref("), + ), + HelperReferenceE2ETestCase( + description="unread helpers add no tested models on the native engine", + tests=("unread_helpers_and_mocks",), + command=("--no-color", "test", "--select", "item_extras"), + engine="native", + sql_analysis=True, + expected_output_fragments=("0 selected",), + expected_compiled_fragments=(), + expected_absent_compiled_fragments=("archived_rows", "other_rows"), + ), + ), + ids=lambda case: case.description, +) +def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_models( + test_case: HelperReferenceE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="helper_reference_project", + repo_files=build_helper_reference_project_files( + tests=test_case.tests, sql_analysis=test_case.sql_analysis + ), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine}, + ) + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 0, output + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + compiled_sql: str = "\n".join( + path.read_text(encoding="utf-8") + for path in sorted((project_dir / "target" / "compiled" / "tests").rglob("*.sql")) + ) + for fragment in test_case.expected_compiled_fragments: + assert fragment in compiled_sql, compiled_sql + for fragment in test_case.expected_absent_compiled_fragments: + assert fragment not in compiled_sql, compiled_sql + + +@pytest.mark.parametrize( + "test_case", + ( + HelperReferenceErrorE2ETestCase( + description="helpers reading each other on the python engine", + test_name="helper_cycle", + engine="python", + expected_code="P013", + expected_message=( + "SQL test CTE 'first_rows' reads itself through 'first_rows' -> " + "'second_rows' -> 'first_rows', so the test query cannot define its CTEs in " + "dependency order" + ), + expected_line=7, + expected_column=1, + expected_help_fragment="shared_rows AS (SELECT ...)", + ), + HelperReferenceErrorE2ETestCase( + description="helper reading an unknown model on the python engine", + test_name="helper_reads_unknown_model", + engine="python", + expected_code="P013", + expected_message=( + "SQL test helper CTE 'missing_rows' references unknown model 'item_archive'" + ), + expected_line=7, + expected_column=38, + expected_help_fragment='__ref("")', + ), + HelperReferenceErrorE2ETestCase( + description="mock reading a helper that reads a model on the python engine", + test_name="mock_reads_referencing_helper", + engine="python", + expected_code="P013", + expected_message=( + "SQL test mock '__ref__items' reads helper CTE 'base_rows', which calls " + '__ref("item_totals"); mocks and fixtures are defined before the models the ' + "test runs, so the helper cannot be resolved for them" + ), + expected_line=7, + expected_column=35, + expected_help_fragment='FROM __ref__item_totals rather than FROM __ref("item_totals")', + ), + HelperReferenceErrorE2ETestCase( + description="helpers reading each other on the native engine", + test_name="helper_cycle", + engine="native", + expected_code="P013", + expected_message=( + "SQL test CTE 'first_rows' reads itself through 'first_rows' -> " + "'second_rows' -> 'first_rows', so the test query cannot define its CTEs in " + "dependency order" + ), + expected_line=7, + expected_column=1, + expected_help_fragment="shared_rows AS (SELECT ...)", + ), + HelperReferenceErrorE2ETestCase( + description="helper reading an unknown model on the native engine", + test_name="helper_reads_unknown_model", + engine="native", + expected_code="P013", + expected_message=( + "SQL test helper CTE 'missing_rows' references unknown model 'item_archive'" + ), + expected_line=7, + expected_column=38, + expected_help_fragment='__ref("")', + ), + HelperReferenceErrorE2ETestCase( + description="mock reading a helper that reads a model on the native engine", + test_name="mock_reads_referencing_helper", + engine="native", + expected_code="P013", + expected_message=( + "SQL test mock '__ref__items' reads helper CTE 'base_rows', which calls " + '__ref("item_totals"); mocks and fixtures are defined before the models the ' + "test runs, so the helper cannot be resolved for them" + ), + expected_line=7, + expected_column=35, + expected_help_fragment='FROM __ref__item_totals rather than FROM __ref("item_totals")', + ), + ), + ids=lambda case: case.description, +) +def test_given_unresolvable_helper_references_when_compiling_then_compile_reports_them( + test_case: HelperReferenceErrorE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="helper_reference_project", + repo_files=build_helper_reference_project_files(tests=(test_case.test_name,)), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 1, output + payload: dict[str, Any] = json.loads(result.stdout) + diagnostic: dict[str, Any] = payload["diagnostics"][0] + assert diagnostic["code"] == test_case.expected_code, output + assert diagnostic["message"] == test_case.expected_message, output + assert diagnostic["location"]["path"] == f"tests/unit/test_{test_case.test_name}.sql" + assert diagnostic["location"]["line"] == test_case.expected_line, output + assert diagnostic["location"]["column"] == test_case.expected_column, output + assert test_case.expected_help_fragment in diagnostic["help"], output + + +@pytest.mark.parametrize( + "test_case", + ( + HelperRedefinitionE2ETestCase( + description="nested CTE redefining a helper on the python engine", + engine="python", + expected_output_fragments=("error[P001]", "which redefines helper CTE 'doubled'"), + expected_absent_output_fragments=("P013",), + ), + HelperRedefinitionE2ETestCase( + description="nested CTE redefining a helper on the native engine", + engine="native", + expected_output_fragments=("error[P001]", "which redefines helper CTE 'doubled'"), + expected_absent_output_fragments=("P013",), + ), + ), + ids=lambda case: case.description, +) +def test_given_nested_cte_shadowing_a_helper_when_compiling_then_the_redefinition_is_reported( + test_case: HelperRedefinitionE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="helper_reference_project", + repo_files=build_helper_reference_project_files(tests=("nested_cte_shadows_helper",)), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 1, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + for fragment in test_case.expected_absent_output_fragments: + assert fragment not in output, output diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 0c64348c3a..a416ce8a5c 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -141,6 +141,36 @@ __assert__no_unexpected_order_ids AS ( SQLBuild emits each helper and mock a query needs once, in dependency order, including mocks that a helper reaches through another mock. +Helpers can also call the references an `__assert__` CTE can, such as `__ref()`, `__source()` and +`__seed()`, and they resolve exactly as there: a mocked relation reads its mock, and an unmocked +`__ref("")` reads the model's real SQL, so the test runs that model and +`sqb test --select ` selects the test even when only a helper reads it. SQLBuild emits a +helper after the mocks and models it reads: + +```sql +TEST(); + +WITH +__ref__stg_orders AS ( + SELECT 1 AS order_id, 3 AS quantity +), +order_totals AS ( + SELECT order_id, quantity FROM __ref("orders") +), +__assert__quantities_are_positive AS ( + SELECT order_id FROM order_totals WHERE quantity <= 0 +) +``` + +Only helpers the test reads count: those its `__expected__` and `__assert__` CTEs reach, directly +or through other helpers and mocks. A helper nothing reads is left out of the test query, and its +references are neither checked nor run. Compile rejects references the test query cannot resolve +with `P013`, at the helper or call that causes it: a `__ref()` to a model that does not exist, +helpers or mocks that read each other in a cycle, and a mock or `__table_fn__` fixture the test +uses that reads a helper calling a reference. Mocks are defined before the models a test runs, so +a mock reads other mocks by CTE name, such as `FROM __ref__stg_orders`, rather than through +`__ref("stg_orders")`. + ## Multi-model tests Tests can span multiple models in a single file. Mock your sources, define an expected output for the model you care about, and SQLBuild automatically resolves every intermediate model using its real SQL.