From 1723c7f540f3d8a734393b57b814c5b883dcbfca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 03:13:25 +0100 Subject: [PATCH 1/2] fix(test): rewrite references in helper CTEs and order them after their dependencies Helper CTEs in SQL tests now resolve __ref(), __source(), __seed() and the other reference calls exactly like __assert__ CTEs, are emitted after the mocks and models they read, and count their unmocked __ref() targets when selecting the test. Helper references compile cannot resolve (unknown models, cycles, and mocks or fixtures reading a referencing helper) fail at compile with P013. --- .../_helpers/sql_tests/helper_scope.rs | 69 +++- .../compiler/_helpers/sql_tests/planning.rs | 239 +++++++++---- .../sqlbuild-analysis/src/compiler/tests.rs | 2 + .../src/compiler/tests/helpers.rs | 95 ++++- .../tests/test_sql_test_helper_references.rs | 145 ++++++++ .../src/compiler/tests/test_types.rs | 11 + .../_helpers/corpus/render_failure_cases.py | 17 + .../references/docs/concepts/testing.md | 27 ++ .../compile/_helpers/attachment/sql_tests.py | 26 ++ .../compile/_helpers/sql_tests/helper_ctes.py | 332 ++++++++++++++++++ src/sqlbuild/compiler/compile/constants.py | 1 + .../cli/commands/main/test/_test_types.py | 24 ++ .../cli/commands/main/test/helpers.py | 89 +++++ .../main/test/test_helper_references.py | 233 ++++++++++++ .../content/docs/docs/concepts/testing.mdx | 27 ++ 15 files changed, 1264 insertions(+), 73 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs create mode 100644 src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs index b5501d357a..e0d16187f3 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs @@ -6,7 +6,7 @@ use polyglot_sql::Dialect; use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::SliceDialect; -use crate::compiler::_helpers::sql_tests::cte_sql::with_leading_ctes; +use crate::compiler::_helpers::sql_tests::cte_sql::{with_leading_ctes, with_unique_ctes}; use crate::compiler::_helpers::sql_tests::helper_names::{ helper_cte_name, rename_helper_references, }; @@ -41,6 +41,16 @@ struct ScopeHelper { cte_name: String, sql: String, tokens: Vec, + resolved: Option, +} + +/// A helper whose relation references were resolved like an assertion's, after the model chain. +pub(crate) struct ResolvedHelper { + /// The helper body with every reference replaced by the CTE that stands in for it. + pub(crate) sql: String, + /// The mock and model CTEs the resolved body reads, dependencies first. + pub(crate) lifted_ctes: Vec<(String, String)>, + pub(crate) reached_mocks: HashSet, } /// One test CTE another test CTE reads: a mock, or a helper when `mock_name` is `None`. @@ -161,6 +171,7 @@ impl ScopeGraph { cte_name: cte.name.clone(), sql: cte.sql_body.clone(), tokens: identifier_tokens(&cte.sql_body, patterns), + resolved: None, }); } let readers = test_readers(fixtures); @@ -281,6 +292,40 @@ impl ScopeGraph { order.ordered } + /// Helpers whose SQL, as placed in the test query, calls a reference such as `__ref()`. + pub(crate) fn referencing_helpers(&self, patterns: &SqlTestPatterns) -> Vec<(usize, String)> { + self.helpers + .iter() + .enumerate() + .filter(|(_, helper)| helper.resolved.is_none()) + .filter_map(|(index, _)| { + let sql = self.placed_helper_sql(index); + (patterns.test_reference.is_match(sql) || patterns.udf.is_match(sql)) + .then(|| (index, sql.to_string())) + }) + .collect() + } + + /// Replace a helper's references with the CTEs a reader must define ahead of it. + pub(crate) fn resolve_helper(&mut self, index: usize, resolved: ResolvedHelper) { + let placement = self.placement; + let Some(helper) = self.helpers.get_mut(index) else { + return; + }; + match placement { + HelperPlacement::TopLevel => helper.sql.clone_from(&resolved.sql), + HelperPlacement::Inline => helper.original.clone_from(&resolved.sql), + } + helper.resolved = Some(resolved); + } + + fn placed_helper_sql(&self, index: usize) -> &str { + match self.placement { + HelperPlacement::TopLevel => &self.helpers[index].sql, + HelperPlacement::Inline => &self.helpers[index].original, + } + } + /// Whether a CTE name is a test helper placed at the top level of the test query. pub(crate) fn is_top_level_helper(&self, name: &str) -> bool { self.placement == HelperPlacement::TopLevel @@ -350,11 +395,25 @@ impl ScopeGraph { .reached_mocks .insert(self.mocks[index].mock_name.clone()); } + if let ScopeNode::Helper(index) = node + && let Some(resolved) = &self.helpers[index].resolved + { + scope + .reached_mocks + .extend(resolved.reached_mocks.iter().cloned()); + scope.ctes = with_unique_ctes( + std::mem::take(&mut scope.ctes), + resolved.lifted_ctes.iter().cloned(), + ); + } if let Some(dependency) = self.dependency(node, None) { - scope.ctes.push(( - dependency.generated_name.to_string(), - dependency.sql.to_string(), - )); + scope.ctes = with_unique_ctes( + std::mem::take(&mut scope.ctes), + [( + dependency.generated_name.to_string(), + dependency.sql.to_string(), + )], + ); } } scope diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index 801b526570..e111b50197 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -17,7 +17,7 @@ use crate::compiler::_helpers::sql_tests::cte_sql::{ use crate::compiler::_helpers::sql_tests::expected_columns::expected_columns; use crate::compiler::_helpers::sql_tests::helper_names::HELPER_PREFIX; use crate::compiler::_helpers::sql_tests::helper_scope::{ - ScopeGraph, ScopeRequest, helper_scope_ctes, merged_scoped_ctes, + ResolvedHelper, ScopeGraph, ScopeRequest, helper_scope_ctes, merged_scoped_ctes, }; use crate::compiler::_helpers::sql_tests::markers::{ ProtectedRanges, marker_names, marker_names_in, replace_callable_markers, @@ -223,10 +223,10 @@ pub(crate) struct SqlTestPatterns { reference: Regex, source: Regex, seed: Regex, - udf: Regex, + pub(crate) udf: Regex, table_function: Regex, dbt_reference: Regex, - test_reference: Regex, + pub(crate) test_reference: Regex, pub(crate) lexical: LexicalSyntax, pub(crate) identifier: Regex, } @@ -602,6 +602,7 @@ fn chain_root_names( patterns: &SqlTestPatterns, ) -> Vec { expected_names.extend(assertion_ref_targets(&fixtures.assertions, patterns)); + expected_names.extend(helper_ref_targets(fixtures, patterns)); dedupe(expected_names) } @@ -930,73 +931,30 @@ fn plan_model_test( }); } + let mut readers = ReaderContext { + context, + ordered_names: &ordered_names, + overrides: &plan.model_query_overrides, + file_label: &plan.file_label, + analysis_resolved: &analysis_resolved, + textual_chain: None, + }; + resolve_referencing_helpers(HelperResolution { + readers: &mut readers, + fixtures: &mut fixtures, + chain: &mut chain, + reachable_mocks: &mut reachable_mocks, + })?; + let mut assertions: Vec = Vec::new(); - let mut textual_assertion_chain: Option = None; for (assertion_name, assertion_sql) in &fixtures.assertions { let placed_sql = fixtures.scope.reader_sql(assertion_sql, &context.patterns); - let table_functions = - resolve_table_function_fixtures(&placed_sql, &fixtures, &context.patterns)?; - reachable_mocks.extend(table_functions.reached.iter().cloned()); - let fixture_resolved_sql = &table_functions.sql; - let analyzed = - if context.sql_analysis_enabled && analysis_resolved.len() == ordered_names.len() { - analyze_and_resolve_sql(AnalysisResolutionRequest { - query_sql: fixture_resolved_sql, - fixture_ctes: &table_functions.ctes, - fixtures: &fixtures, - resolved_chain: &analysis_resolved, - functions: &context.functions, - file_label: &plan.file_label, - dialect_name: &context.dialect, - templates: &context.analysis_templates, - patterns: &context.patterns, - })? - } else { - None - }; - let (mut resolved_sql, mut lifted_ctes, comparison_body_sql) = match analyzed { - Some(value) - if !has_unresolved_test_reference(&value.resolved_sql, &context.patterns) => - { - reachable_mocks.extend(value.reachable_mock_names.iter().cloned()); - ( - value.resolved_sql, - value.generated_ctes, - Some(value.cte_body_sql), - ) - } - _ => { - if textual_assertion_chain.is_none() { - let (chain, reached) = build_textual_chain( - &ordered_names, - &plan.model_query_overrides, - &fixtures, - context, - )?; - reachable_mocks.extend(reached); - textual_assertion_chain = Some(chain); - } - let chain = textual_assertion_chain - .as_ref() - .ok_or_else(|| planner_error("textual assertion chain is unavailable"))?; - let (resolved, reached) = - resolve_assertion_textual_sql(AssertionResolutionRequest { - assertion_sql: fixture_resolved_sql, - fixtures: &fixtures, - chain, - functions: &context.functions, - requires_flat_ctes: context.requires_derived_table_aliases, - patterns: &context.patterns, - })?; - reachable_mocks.extend(reached); - let lifted = with_unique_ctes(table_functions.ctes.clone(), resolved.lifted_ctes); - ( - with_leading_ctes(&lifted, &resolved.body_sql), - lifted, - Some(resolved.body_sql), - ) - } - }; + let resolved = readers.resolve(&placed_sql, &fixtures)?; + reachable_mocks.extend(resolved.chain_reached); + reachable_mocks.extend(resolved.reached); + let mut resolved_sql = resolved.resolved_sql; + let mut lifted_ctes = resolved.lifted_ctes; + let comparison_body_sql = Some(resolved.body_sql); let scope = helper_scope_ctes( assertion_sql, &fixtures, @@ -1040,6 +998,140 @@ fn plan_model_test( }) } +/// The planned chain and fixtures whose helpers are resolved once every chain model is known. +struct HelperResolution<'r, 'a> { + readers: &'r mut ReaderContext<'a>, + fixtures: &'r mut TestFixtures, + chain: &'r mut [ChainStep], + reachable_mocks: &'r mut HashSet, +} + +/// Resolve the references helpers call, then give expected rows the CTEs their helpers now read. +fn resolve_referencing_helpers(request: HelperResolution<'_, '_>) -> Result<(), String> { + let HelperResolution { + readers, + fixtures, + chain, + reachable_mocks, + } = request; + let patterns = &readers.context.patterns; + let mut resolved_helpers: Vec<(usize, ResolvedHelper)> = Vec::new(); + for (index, helper_sql) in fixtures.scope.referencing_helpers(patterns) { + let resolved = readers.resolve(&helper_sql, fixtures)?; + reachable_mocks.extend(resolved.chain_reached); + resolved_helpers.push(( + index, + ResolvedHelper { + sql: resolved.body_sql, + lifted_ctes: resolved.lifted_ctes, + reached_mocks: resolved.reached, + }, + )); + } + if resolved_helpers.is_empty() { + return Ok(()); + } + for (index, resolved) in resolved_helpers { + fixtures.scope.resolve_helper(index, resolved); + } + for step in chain { + if let Some(sql) = fixtures.expected.get(&step.model_name) { + let scope = helper_scope_ctes(sql, fixtures, patterns, readers.file_label)?; + reachable_mocks.extend(scope.reached_mocks); + step.expected_lifted_ctes = scope.ctes; + } + } + Ok(()) +} + +/// What resolving one test-authored query's relation references needs once the chain is planned. +struct ReaderContext<'a> { + context: &'a ProjectContext, + ordered_names: &'a [String], + overrides: &'a BTreeMap, + file_label: &'a str, + analysis_resolved: &'a HashMap, + textual_chain: Option, +} + +/// One assertion or helper query with its references resolved to the CTEs standing in for them. +struct ResolvedReader { + resolved_sql: String, + lifted_ctes: Vec<(String, String)>, + body_sql: String, + reached: HashSet, + /// Mocks the shared textual chain reached when this query first needed it. + chain_reached: HashSet, +} + +impl ReaderContext<'_> { + /// Resolve an assertion or helper query against the planned chain, mocks and functions. + fn resolve(&mut self, sql: &str, fixtures: &TestFixtures) -> Result { + let context = self.context; + let table_functions = resolve_table_function_fixtures(sql, fixtures, &context.patterns)?; + let mut reached: HashSet = table_functions.reached.clone(); + let fixture_resolved_sql = &table_functions.sql; + let analyzed = if context.sql_analysis_enabled + && self.analysis_resolved.len() == self.ordered_names.len() + { + analyze_and_resolve_sql(AnalysisResolutionRequest { + query_sql: fixture_resolved_sql, + fixture_ctes: &table_functions.ctes, + fixtures, + resolved_chain: self.analysis_resolved, + functions: &context.functions, + file_label: self.file_label, + dialect_name: &context.dialect, + templates: &context.analysis_templates, + patterns: &context.patterns, + })? + } else { + None + }; + if let Some(value) = analyzed + && !has_unresolved_test_reference(&value.resolved_sql, &context.patterns) + { + reached.extend(value.reachable_mock_names); + return Ok(ResolvedReader { + resolved_sql: value.resolved_sql, + lifted_ctes: value.generated_ctes, + body_sql: value.cte_body_sql, + reached, + chain_reached: HashSet::new(), + }); + } + let mut chain_reached: HashSet = HashSet::new(); + if self.textual_chain.is_none() { + let (chain, chain_mocks) = + build_textual_chain(self.ordered_names, self.overrides, fixtures, context)?; + chain_reached = chain_mocks; + self.textual_chain = Some(chain); + } + let chain = self + .textual_chain + .as_ref() + .ok_or_else(|| planner_error("textual assertion chain is unavailable"))?; + let (resolved, textual_reached) = + resolve_assertion_textual_sql(AssertionResolutionRequest { + assertion_sql: fixture_resolved_sql, + fixtures, + chain, + functions: &context.functions, + requires_flat_ctes: context.requires_derived_table_aliases, + patterns: &context.patterns, + })?; + reached.extend(textual_reached); + let lifted = with_unique_ctes(table_functions.ctes, resolved.lifted_ctes); + Ok(ResolvedReader { + resolved_sql: with_leading_ctes(&lifted, &resolved.body_sql), + lifted_ctes: lifted, + body_sql: resolved.body_sql, + reached, + chain_reached, + }) + } +} + /// Drop SQL from chain steps the renderer never emits, keeping plan output linear in chain length. fn omit_unrendered_step_sql(chain: Vec, assertions: &[AssertionStep]) -> Vec { let rendered_steps = rendered_chain_steps(&chain, assertions); @@ -1745,6 +1837,19 @@ fn assertion_ref_targets( dedupe(targets) } +/// Unmocked models helper CTEs read, which the test runs so helpers can read them. +fn helper_ref_targets(fixtures: &TestFixtures, patterns: &SqlTestPatterns) -> Vec { + let mut targets: Vec = Vec::new(); + for helper in &fixtures.helpers { + targets.extend( + marker_names(&patterns.reference, &patterns.lexical, &helper.sql_body) + .into_iter() + .filter(|name| !fixtures.mock_refs.contains_key(name)), + ); + } + dedupe(targets) +} + fn has_unresolved_test_reference(sql: &str, patterns: &SqlTestPatterns) -> bool { if !patterns.test_reference.is_match(sql) { return false; diff --git a/crates/sqlbuild-analysis/src/compiler/tests.rs b/crates/sqlbuild-analysis/src/compiler/tests.rs index 9b28b3cb13..b3c3a19419 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests.rs @@ -8,6 +8,8 @@ mod sql_test_cte_slices; mod sql_test_expected_columns; #[path = "tests/test_sql_test_extraction.rs"] mod sql_test_extraction; +#[path = "tests/test_sql_test_helper_references.rs"] +mod sql_test_helper_references; #[path = "tests/test_sql_test_markers.rs"] mod sql_test_markers; #[path = "tests/test_sql_test_name_isolation.rs"] diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index a006e32514..a54830f1a7 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -9,7 +9,9 @@ use crate::compiler::_helpers::sql_tests::cte_rename::defined_cte_keys; use crate::compiler::_helpers::sql_tests::cte_slices::{SliceDialect, split_top_level_with}; use crate::compiler::_helpers::sql_tests::relation_markers::relation_marker_calls; use crate::compiler::main::sql_test_extraction::extract_batch_json; -use crate::compiler::tests::test_types::{PlanShape, RenderedShapeTestCase, SharedNameTestCase}; +use crate::compiler::tests::test_types::{ + HelperReferenceTestCase, PlanShape, RenderedShapeTestCase, SharedNameTestCase, +}; pub(crate) fn mixed_expanded_tests_preserve_order_and_payloads() -> bool { let response = extract_batch_json(r#"{"tests":[{"sql":"WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT id FROM helper), __expected__orders AS (SELECT id FROM helper), __assert__positive AS (SELECT id FROM helper WHERE id < 0) SELECT 1","fileLabel":"tests/orders.sql","mode":"model"},{"sql":"WITH input AS (SELECT 1 AS value), __udf_actual__ AS (SELECT __udf(\"increment\")(value) AS value FROM input), __udf_expected__ AS (SELECT 2 AS value) SELECT 1","fileLabel":"tests/increment.sql","mode":"udf"}]}"#).expect("batch succeeds"); @@ -1801,3 +1803,94 @@ fn assert_names_isolated(sql: &str, dialect: &str, helper_name: &str, label: &st ); } } + +fn helper_reference_test(case: &HelperReferenceTestCase) -> Value { + let named = |ctes: &[(&str, &str)]| -> Vec { + ctes.iter() + .map(|(name, sql)| json!({"name": name, "sqlBody": sql})) + .collect() + }; + let mut authored = named(&[ + ( + "__source__raw_orders", + "SELECT 1 AS order_id, 10 AS amount, 7 AS region_id", + ), + ( + "__seed__regions", + "SELECT 7 AS region_id, 'north' AS region_name", + ), + ]); + authored.extend(named(case.helpers)); + json!({ + "name": "orders_case", + "fileLabel": "tests/orders.sql", + "payload": { + "kind": "model", + "authoredCtes": authored, + "expectedCtes": named(case.expected), + "expectedModelNames": case + .expected + .iter() + .map(|(name, _)| name.trim_start_matches("__expected__")) + .collect::>(), + "assertionCtes": named(case.assertions) + } + }) +} + +fn helper_reference_models() -> Value { + json!([ + { + "name": "stg_orders", + "querySql": "SELECT order_id, amount, region_id FROM __source(\"raw_orders\")", + "modelDependencies": [] + }, + { + "name": "orders", + "querySql": "SELECT order_id, amount * 2 AS amount_doubled FROM __ref(\"stg_orders\")", + "modelDependencies": ["stg_orders"] + } + ]) +} + +/// Plan and render one helper-reference case, returning its SQL, chain and warnings. +pub(crate) fn plan_helper_reference_case(case: &HelperReferenceTestCase) -> Value { + let response: Value = serde_json::from_str( + &crate::compiler::main::sql_test_planning::plan_and_render_json( + &json!({ + "lexicalSyntax": generic_lexical_syntax(), + "models": helper_reference_models(), + "tests": [helper_reference_test(case)], + "sqlAnalysisEnabled": case.sql_analysis_enabled, + "sqlAnalysisDialect": "duckdb", + "setDifferenceOperator": "EXCEPT" + }) + .to_string(), + ) + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + response["artifacts"][0].clone() +} + +/// Resolve one helper-reference case's model chain without planning SQL. +pub(crate) fn chain_helper_reference_case(case: &HelperReferenceTestCase) -> Value { + let response: Value = serde_json::from_str( + &crate::compiler::main::sql_test_chain_resolution::resolve_chains_json( + &json!({ + "lexicalSyntax": generic_lexical_syntax(), + "models": helper_reference_models(), + "tests": [helper_reference_test(case)] + }) + .to_string(), + ) + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + response["chains"][0].clone() +} + +/// Whether `first` is defined before `second` at the top level of a rendered test query. +pub(crate) fn defined_before(sql: &str, first: &str, second: &str) -> bool { + top_level_cte_position(sql, first) < top_level_cte_position(sql, second) +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs new file mode 100644 index 0000000000..758cba3d3c --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs @@ -0,0 +1,145 @@ +use serde_json::{Value, json}; + +use crate::compiler::tests::helpers::{ + chain_helper_reference_case, defined_before, plan_helper_reference_case, +}; +use crate::compiler::tests::test_types::HelperReferenceTestCase; + +const HELPER_READS_MODEL: (&str, &str) = ( + "doubled", + "SELECT order_id, amount_doubled FROM __ref(\"orders\")", +); +const ASSERT_DOUBLED: (&str, &str) = ( + "__assert__doubles_amount", + "SELECT order_id FROM doubled WHERE amount_doubled <> 20", +); + +#[test] +fn given_helper_cte_references_when_planning_then_references_resolve_in_dependency_order() { + let test_cases = [ + HelperReferenceTestCase { + description: "helper reading a model is emitted after the model and its mocks", + sql_analysis_enabled: true, + helpers: &[HELPER_READS_MODEL], + expected: &[], + assertions: &[ASSERT_DOUBLED], + expected_chain: &["stg_orders", "orders"], + expected_order: &[ + ("__source__raw_orders", "__ref__stg_orders"), + ("__ref__stg_orders", "__ref__orders"), + ("__ref__orders", "__helper__doubled"), + ("__helper__doubled", "__assert__doubles_amount"), + ], + expected_fragments: &[ + "__helper__doubled AS (SELECT order_id, amount_doubled FROM __ref__orders)", + ], + }, + HelperReferenceTestCase { + description: "helper inlined without sql analysis reads the model CTE", + sql_analysis_enabled: false, + helpers: &[HELPER_READS_MODEL], + expected: &[], + assertions: &[ASSERT_DOUBLED], + expected_chain: &["stg_orders", "orders"], + expected_order: &[ + ("__source__raw_orders", "__ref__orders"), + ("__ref__orders", "__assert__doubles_amount"), + ], + expected_fragments: &[ + "WITH doubled AS (SELECT order_id, amount_doubled FROM __ref__orders)", + ], + }, + HelperReferenceTestCase { + description: "helper reading another helper, a model and a seed is ordered after all three", + sql_analysis_enabled: true, + helpers: &[ + ( + "with_region", + "SELECT b.order_id, r.region_name FROM base AS b \ + JOIN __ref(\"stg_orders\") AS s USING (order_id) \ + JOIN __seed(\"regions\") AS r USING (region_id)", + ), + ( + "base", + "SELECT order_id, amount_doubled FROM __ref(\"orders\")", + ), + ], + expected: &[], + assertions: &[( + "__assert__has_region", + "SELECT order_id FROM with_region WHERE region_name IS NULL", + )], + expected_chain: &["stg_orders", "orders"], + expected_order: &[ + ("__ref__orders", "__helper__base"), + ("__helper__base", "__helper__with_region"), + ("__ref__stg_orders", "__helper__with_region"), + ("__seed__regions", "__helper__with_region"), + ("__helper__with_region", "__assert__has_region"), + ], + expected_fragments: &[ + "FROM __helper__base AS b JOIN __ref__stg_orders AS s USING (order_id) \ + JOIN __seed__regions AS r USING (region_id)", + ], + }, + HelperReferenceTestCase { + description: "helper read by expected rows resolves sources and seeds to their mocks", + sql_analysis_enabled: true, + helpers: &[( + "expected_rows", + "SELECT s.order_id, s.amount * 2 AS amount_doubled FROM __source(\"raw_orders\") AS s \ + JOIN __seed(\"regions\") AS r USING (region_id)", + )], + expected: &[( + "__expected__orders", + "SELECT order_id, amount_doubled FROM expected_rows", + )], + assertions: &[], + expected_chain: &["stg_orders", "orders"], + expected_order: &[ + ("__source__raw_orders", "__helper__expected_rows"), + ("__seed__regions", "__helper__expected_rows"), + ("__helper__expected_rows", "__expected__orders"), + ], + expected_fragments: &[ + "FROM __source__raw_orders AS s JOIN __seed__regions AS r USING (region_id)", + ], + }, + ]; + for test_case in test_cases { + let artifact: Value = plan_helper_reference_case(&test_case); + let chain: Value = chain_helper_reference_case(&test_case); + let sql = artifact["sql"].as_str().expect("test assumption must hold"); + let errors: Vec<&Value> = artifact["warnings"] + .as_array() + .expect("test assumption must hold") + .iter() + .filter(|warning| warning["severity"] == json!("error")) + .collect(); + + assert_eq!( + chain, + json!(test_case.expected_chain), + "{}", + test_case.description + ); + for (first, second) in test_case.expected_order { + assert!( + defined_before(sql, first, second), + "{}: {first} before {second}: {sql}", + test_case.description + ); + } + for fragment in test_case.expected_fragments { + assert!( + sql.contains(fragment), + "{}: {fragment}: {sql}", + test_case.description + ); + } + for marker in ["__ref(", "__source(", "__seed("] { + assert!(!sql.contains(marker), "{}: {sql}", test_case.description); + } + assert!(errors.is_empty(), "{}: {errors:?}", test_case.description); + } +} diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 0dbecaf8ec..2cbbbfa7fc 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -118,3 +118,14 @@ pub(crate) struct PlanShape<'a> { pub(crate) helper_name: &'a str, pub(crate) expected_sql: &'a str, } + +pub(crate) struct HelperReferenceTestCase { + pub(crate) description: &'static str, + pub(crate) sql_analysis_enabled: bool, + pub(crate) helpers: &'static [(&'static str, &'static str)], + pub(crate) expected: &'static [(&'static str, &'static str)], + pub(crate) assertions: &'static [(&'static str, &'static str)], + pub(crate) expected_chain: &'static [&'static str], + pub(crate) expected_order: &'static [(&'static str, &'static str)], + pub(crate) expected_fragments: &'static [&'static str], +} diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index cfe2c9ebcd..257a265ffa 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -245,6 +245,23 @@ def _reference_cases() -> tuple[FailureCase, ...]: ) }, ), + failure_case( + name="test-helper-reads-unknown-model", + expected_code="P013", + expected_message="SQL test helper CTE 'archived_orders' references unknown model", + files={ + "tests/unit/test_stg_orders.sql": ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status\n),\n" + 'archived_orders AS (\n SELECT order_id FROM __ref("archived_orders")\n),\n' + "__assert__no_archived_orders AS (\n" + ' SELECT order_id FROM __ref("stg_orders")\n' + " JOIN archived_orders USING (order_id)\n" + ")\nSELECT 1\n" + ) + }, + ), failure_case( name="dbt-reference-without-manifest", expected_code="C214", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 0e912fe427..0666e7de5d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -165,6 +165,33 @@ __assert__no_unexpected_order_ids AS ( SQLBuild emits each helper and mock a query needs once, in dependency order, including mocks that a helper reaches through another mock. +Helpers can also call the references an `__assert__` CTE can, such as `__ref()`, `__source()` and +`__seed()`, and they resolve exactly as there: a mocked relation reads its mock, and an unmocked +`__ref("")` reads the model's real SQL, so the test runs that model and +`sqb test --select ` selects the test even when only a helper reads it. SQLBuild emits a +helper after the mocks and models it reads: + +```sql +TEST(); + +WITH +__ref__stg_orders AS ( + SELECT 1 AS order_id, 3 AS quantity +), +order_totals AS ( + SELECT order_id, quantity FROM __ref("orders") +), +__assert__quantities_are_positive AS ( + SELECT order_id FROM order_totals WHERE quantity <= 0 +) +``` + +Compile rejects helper references the test query cannot resolve with `P013`, at the helper or call +that causes it: a `__ref()` to a model that does not exist, helpers or mocks that read each other +in a cycle, and a mock or `__table_fn__` fixture that reads a helper calling a reference. Mocks +are defined before the models a test runs, so a mock reads other mocks by CTE name, such as +`FROM __ref__stg_orders`, rather than through `__ref("stg_orders")`. + ## Multi-model tests Tests can span multiple models in a single file. Mock your sources, define an expected output for the model you care about, and SQLBuild automatically resolves every intermediate model using its real SQL. diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index 8c62622748..33bdc97616 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -33,6 +33,11 @@ extract_assertion_target_model_names, extract_sql_test_ctes, ) +from sqlbuild.compiler.compile._helpers.sql_tests.helper_ctes import ( + extract_helper_target_model_names, + report_unresolvable_helper_ctes, + sql_test_helper_ctes, +) from sqlbuild.compiler.compile._helpers.sql_tests.native import ( extract_expanded_sql_tests_cached, ) @@ -317,6 +322,27 @@ def build_test_inputs( loaded_macros=loaded_macros, assertion_target_model_names=assertion_target_model_names, ) + if isinstance(test_ctes.payload, CompileModelSqlTestCtes): + if report_unresolvable_helper_ctes( + model_payload=test_ctes.payload, + test_file=test.test_file, + test_block=test.test_block, + known_model_names=known_model_names, + syntax=sql_lexical_syntax, + ): + continue + assertion_target_model_names = tuple( + dict.fromkeys( + ( + *assertion_target_model_names, + *extract_helper_target_model_names( + helper_ctes=sql_test_helper_ctes(test_ctes.payload.authored_ctes), + mock_model_names=test_ctes.payload.mock_model_names, + syntax=sql_lexical_syntax, + ), + ) + ) + ) test_inputs.append( CompileSqlTestInput( test_file=test.test_file, diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py new file mode 100644 index 0000000000..0226cadad4 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py @@ -0,0 +1,332 @@ +"""Relation references in SQL-test helper CTEs: the models they read and what cannot resolve.""" + +from __future__ import annotations + +import re +from typing import Any + +from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic +from sqlbuild.compiler.compile._helpers.refs.references import ( + reference_call_location, + scan_sql_reference_calls, +) +from sqlbuild.compiler.compile.constants import ( + DBT_REF_TEST_CTE_PREFIX, + REF_TEST_CTE_PREFIX, + SEED_TEST_CTE_PREFIX, + SOURCE_TEST_CTE_PREFIX, + SQL_TEST_HELPER_REFERENCE_CODE, + TABLE_FN_TEST_CTE_PREFIX, +) +from sqlbuild.compiler.compile.models import ( + CompileModelSqlTestCtes, + CompilerDiagnostic, + CompileSqlReference, + CompileSqlTestCte, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.spec.contracts.models import SourceLocation + +_MOCK_CTE_PREFIXES: tuple[str, ...] = ( + REF_TEST_CTE_PREFIX, + SOURCE_TEST_CTE_PREFIX, + SEED_TEST_CTE_PREFIX, + DBT_REF_TEST_CTE_PREFIX, + TABLE_FN_TEST_CTE_PREFIX, +) +_RELATION_REFERENCE_KINDS: frozenset[SqlReferenceKind] = frozenset( + { + SqlReferenceKind.REF, + SqlReferenceKind.SOURCE, + SqlReferenceKind.SEED, + SqlReferenceKind.DBT_REF, + SqlReferenceKind.TABLE_FUNCTION, + } +) + + +def sql_test_helper_ctes( + authored_ctes: tuple[CompileSqlTestCte, ...], +) -> tuple[CompileSqlTestCte, ...]: + """Return the authored CTEs that are helpers rather than mocks or fixtures.""" + + return tuple(cte for cte in authored_ctes if not cte.name.startswith(_MOCK_CTE_PREFIXES)) + + +def extract_helper_target_model_names( + *, + helper_ctes: tuple[CompileSqlTestCte, ...], + mock_model_names: tuple[str, ...], + syntax: SqlLexicalSyntax, +) -> tuple[str, ...]: + """Return unmocked models helper CTEs read with ``__ref()``, which the test must run.""" + + mocked: frozenset[str] = frozenset(mock_model_names) + targets: list[str] = [] + for cte in helper_ctes: + targets.extend( + reference.ref_name + for reference in _references(sql=cte.sql_body, syntax=syntax) + if reference.ref_kind == SqlReferenceKind.REF and reference.ref_name not in mocked + ) + return tuple(dict.fromkeys(targets)) + + +def report_unresolvable_helper_ctes( + *, + model_payload: CompileModelSqlTestCtes, + test_file: DiscoveredSqlTestFile, + test_block: DiscoveredSqlTestBlock, + known_model_names: set[str], + syntax: SqlLexicalSyntax, +) -> bool: + """Report helper CTEs the test query cannot place or resolve; return whether any was found.""" + + helper_ctes: tuple[CompileSqlTestCte, ...] = sql_test_helper_ctes(model_payload.authored_ctes) + if not helper_ctes: + return False + reporter: _HelperDiagnostics = _HelperDiagnostics(test_file=test_file, test_block=test_block) + mocked: frozenset[str] = frozenset(model_payload.mock_model_names) + for cte in helper_ctes: + for reference in _references(sql=cte.sql_body, syntax=syntax): + if ( + reference.ref_kind == SqlReferenceKind.REF + and reference.ref_name not in mocked + and reference.ref_name not in known_model_names + ): + reporter.report( + cte_name=cte.name, + call=SqlReferenceKind.REF.example_call(reference.ref_name, quote='"'), + message=( + f"SQL test helper CTE '{cte.name}' references unknown model " + f"'{reference.ref_name}'" + ), + help=( + 'Read a model defined in the project, for example __ref(""), ' + f"or remove the reference from helper CTE '{cte.name}'." + ), + ) + dependencies: dict[str, tuple[str, ...]] = _authored_cte_dependencies( + authored_ctes=model_payload.authored_ctes + ) + names_by_key: dict[str, str] = { + cte.name.casefold(): cte.name for cte in model_payload.authored_ctes + } + cycle: tuple[str, ...] | None = _first_cycle(dependencies) + if cycle is not None: + path: str = " -> ".join(f"'{names_by_key[key]}'" for key in cycle) + reporter.report( + cte_name=names_by_key[cycle[0]], + call=None, + message=( + f"SQL test CTE '{names_by_key[cycle[0]]}' reads itself through {path}, so the " + "test query cannot define its CTEs in dependency order" + ), + help=( + "Break the cycle by moving the shared rows into a helper CTE that reads neither, " + "for example shared_rows AS (SELECT ...), and read shared_rows from both CTEs." + ), + ) + _report_mocks_reading_references( + reporter=reporter, + model_payload=model_payload, + helper_ctes=helper_ctes, + dependencies=dependencies, + syntax=syntax, + ) + return reporter.reported + + +class _HelperDiagnostics: + """Located P013 diagnostics for one SQL test block.""" + + def __init__( + self, *, test_file: DiscoveredSqlTestFile, test_block: DiscoveredSqlTestBlock + ) -> None: + self._test_file: DiscoveredSqlTestFile = test_file + self._block_offset: int = max(test_file.contents.find(test_block.sql_body), 0) + self._test_name: str = test_block.name or test_file.relative_path.stem + self.reported: bool = False + + def report(self, *, cte_name: str, call: str | None, message: str, help: str) -> None: + location: SourceLocation = self._location(cte_name=cte_name, call=call) + self.reported = True + report_compile_diagnostic( + key=( + SQL_TEST_HELPER_REFERENCE_CODE, + self._test_file.relative_path.as_posix(), + f"{location.line:09d}:{location.column:09d}", + message, + ), + diagnostic=CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=SQL_TEST_HELPER_REFERENCE_CODE, + message=message, + resource_type=CompiledResourceType.SQL_TEST, + resource_name=self._test_name, + location=location, + help=help, + ), + ) + + def _location(self, *, cte_name: str, call: str | None) -> SourceLocation: + contents: str = self._test_file.contents + header: re.Match[str] | None = re.compile( + rf"(? None: + referencing: dict[str, tuple[CompileSqlTestCte, CompileSqlReference]] = {} + for cte in helper_ctes: + reference: CompileSqlReference | None = next( + ( + reference + for reference in _references(sql=cte.sql_body, syntax=syntax) + if reference.ref_kind in _RELATION_REFERENCE_KINDS + ), + None, + ) + if reference is not None: + referencing[cte.name.casefold()] = (cte, reference) + if not referencing: + return + for mock in model_payload.authored_ctes: + if not mock.name.startswith(_MOCK_CTE_PREFIXES): + continue + helper_key: str | None = _first_reachable( + origin=mock.name.casefold(), targets=referencing, dependencies=dependencies + ) + if helper_key is None: + continue + helper, reference = referencing[helper_key] + call: str = _reference_call(reference) + reporter.report( + cte_name=helper.name, + call=call, + message=( + f"SQL test mock '{mock.name}' reads helper CTE '{helper.name}', which calls " + f"{call}; mocks and fixtures are defined before the models the test runs, so " + "the helper cannot be resolved for them" + ), + help=_mock_reference_help(mock_name=mock.name, call=call, reference=reference), + ) + + +def _mock_reference_help(*, mock_name: str, call: str, reference: CompileSqlReference) -> str: + kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) + if kind is not SqlReferenceKind.TABLE_FUNCTION and reference.ref_package is None: + mock_cte: str = f"{kind.fixture_cte_prefix}{reference.ref_name}" + return ( + f"Read a mock by its CTE name instead, for example FROM {mock_cte} rather than " + f"FROM {call}, defining {mock_cte} AS (SELECT ...) if the test does not mock it, " + f"or write the rows of '{mock_name}' directly." + ) + return f"Write the rows of '{mock_name}' directly, for example {mock_name} AS (SELECT ...)." + + +def _reference_call(reference: CompileSqlReference) -> str: + kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) + if kind is SqlReferenceKind.DBT_REF and reference.ref_package is not None: + return kind.example_call(reference.ref_package, reference.ref_name, quote='"') + return kind.example_call(reference.ref_name, quote='"') + + +def _references(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[CompileSqlReference, ...]: + return scan_sql_reference_calls(sql=sql, syntax=syntax).references + + +def _authored_cte_dependencies( + *, authored_ctes: tuple[CompileSqlTestCte, ...] +) -> dict[str, tuple[str, ...]]: + """Return each authored CTE's unqualified table reads of other authored CTEs.""" + + keys: frozenset[str] = frozenset(cte.name.casefold() for cte in authored_ctes) + dependencies: dict[str, tuple[str, ...]] = {} + for cte in authored_ctes: + key: str = cte.name.casefold() + references: tuple[str, ...] = _table_reference_keys(sql=cte.sql_body, keys=keys) + dependencies[key] = tuple(reference for reference in references if reference != key) + return dependencies + + +def _table_reference_keys(*, sql: str, keys: frozenset[str]) -> tuple[str, ...]: + folded: str = sql.casefold() + if not any(key in folded for key in keys): + return () + polyglot_module: Any = import_polyglot_sql() + try: + parsed: Any = polyglot_module.parse_one(sql, dialect="generic") + except polyglot_module.PolyglotError: + return () + references: list[str] = [] + for table in parsed.find_all("table"): + if table.arg("schema") is not None or table.arg("catalog") is not None: + continue + key: str = str(getattr(table, "name", "") or "").casefold() + if key in keys and key not in references: + references.append(key) + return tuple(references) + + +def _first_cycle(dependencies: dict[str, tuple[str, ...]]) -> tuple[str, ...] | None: + """Return the first dependency cycle in authored order, closed on its first node.""" + + for origin in dependencies: + pending: list[tuple[str, ...]] = [(origin,)] + visited: set[str] = set() + while pending: + path: tuple[str, ...] = pending.pop(0) + for dependency in dependencies.get(path[-1], ()): + if dependency == origin: + return (*path, origin) + if dependency not in visited: + visited.add(dependency) + pending.append((*path, dependency)) + return None + + +def _first_reachable( + *, + origin: str, + targets: dict[str, tuple[CompileSqlTestCte, CompileSqlReference]], + dependencies: dict[str, tuple[str, ...]], +) -> str | None: + pending: list[str] = list(dependencies.get(origin, ())) + visited: set[str] = set(pending) + while pending: + key: str = pending.pop(0) + if key in targets: + return key + for dependency in dependencies.get(key, ()): + if dependency not in visited: + visited.add(dependency) + pending.append(dependency) + return None diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index ff40e10148..9be4a7b2f3 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -22,6 +22,7 @@ MISSING_DESCRIPTION_CODE: str = "P010" CURSOR_MODEL_WITHOUT_INPUTS_CODE: str = "P011" REFERENCE_CALL_SYNTAX_CODE: str = "P012" +SQL_TEST_HELPER_REFERENCE_CODE: str = "P013" DESCRIPTION_REQUIRED_INPUT_KINDS: dict[str, str] = { "model_files": "model", "scenario_files": "scenario", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 36b8fea57b..7ea18492c4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -321,3 +321,27 @@ class SelectTestByNameE2ETestCase: expected_exit_code: int expected_output_fragments: tuple[str, ...] unexpected_output_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class HelperReferenceE2ETestCase: + """Test case for relation references inside SQL-test helper CTEs.""" + + description: str + tests: tuple[str, ...] + command: tuple[str, ...] + engine: str + expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class HelperReferenceErrorE2ETestCase: + """Test case for helper CTE references compile cannot resolve.""" + + description: str + test_name: str + engine: str + expected_message: str + expected_line: int + expected_column: int + expected_help_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index a79a641f32..b86135d29d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -1725,3 +1725,92 @@ def prepare_optional_ceremony_project(*, tmp_path: Path, test_files: dict[str, s sql="CREATE TABLE raw_orders AS SELECT 1 AS order_id, 'paid' AS status", ) return project_dir + + +_HELPER_REFERENCE_TESTS: dict[str, str] = { + "helper_and_assertion_read_model": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "doubled AS (\n" + ' SELECT item_id, amount_doubled FROM __ref("item_totals")\n' + "),\n" + "__assert__doubles_amount AS (\n" + " SELECT item_id FROM doubled WHERE amount_doubled <> 20\n" + "),\n" + "__assert__has_one_row AS (\n" + ' SELECT COUNT(*) AS row_count FROM __ref("item_totals") HAVING COUNT(*) <> 1\n' + ")\n" + ), + "only_helper_reads_model": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "doubled AS (\n" + ' SELECT item_id, amount_doubled FROM __ref("item_totals")\n' + "),\n" + "__assert__doubles_amount AS (\n" + " SELECT item_id FROM doubled WHERE amount_doubled <> 20\n" + ")\n" + ), + "helper_reads_helper_and_mock": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "joined AS (\n" + " SELECT d.item_id, d.amount_doubled, i.amount FROM doubled AS d\n" + ' JOIN __ref("items") AS i USING (item_id)\n' + "),\n" + "doubled AS (\n" + ' SELECT item_id, amount_doubled FROM __ref("item_totals")\n' + "),\n" + "__expected__item_totals AS (\n" + " SELECT item_id, amount_doubled FROM joined\n" + "),\n" + "__assert__doubles_mocked_amount AS (\n" + " SELECT item_id FROM joined WHERE amount_doubled <> amount * 2\n" + ")\n" + ), + "helper_cycle": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "first_rows AS (SELECT item_id FROM second_rows),\n" + "second_rows AS (SELECT item_id FROM first_rows),\n" + "__assert__no_rows AS (\n" + ' SELECT item_id FROM __ref("item_totals") JOIN first_rows USING (item_id)\n' + ")\n" + ), + "helper_reads_unknown_model": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + 'missing_rows AS (SELECT item_id FROM __ref("item_archive")),\n' + "__assert__no_rows AS (\n" + ' SELECT item_id FROM __ref("item_totals") JOIN missing_rows USING (item_id)\n' + ")\n" + ), + "mock_reads_referencing_helper": ( + "__ref__items AS (\n SELECT item_id, 10 AS amount FROM base_rows\n),\n" + 'base_rows AS (SELECT item_id FROM __ref("item_totals")),\n' + "__assert__no_rows AS (\n" + ' SELECT item_id FROM __ref("item_totals") WHERE item_id IS NULL\n' + ")\n" + ), +} + + +def build_helper_reference_project_files(*, tests: tuple[str, ...]) -> dict[str, str]: + """Build an items project whose SQL tests read models through helper CTEs.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "helper_reference_demo"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "helper_reference_demo.duckdb"\n' + ), + "models/items.sql": ( + "MODEL (description 'Base items', materialized table);\n\n" + "SELECT 1 AS item_id, 10 AS amount\n" + ), + "models/item_totals.sql": ( + "MODEL (description 'Doubled item amounts', materialized table);\n\n" + 'SELECT item_id, amount * 2 AS amount_doubled FROM __ref("items")\n' + ), + } + for test_name in tests: + files[f"tests/unit/test_{test_name}.sql"] = ( + f'TEST (name "{test_name}");\n\nWITH\n{_HELPER_REFERENCE_TESTS[test_name]}\nSELECT 1\n' + ) + return files diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py new file mode 100644 index 0000000000..a603bd2441 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py @@ -0,0 +1,233 @@ +"""E2E tests for `__ref()` and mock references inside SQL-test helper CTEs.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + HelperReferenceE2ETestCase, + HelperReferenceErrorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import ( + build_helper_reference_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" +_HELPER_ERROR_CODE: str = "P013" + + +@pytest.mark.parametrize( + "test_case", + ( + HelperReferenceE2ETestCase( + description="helper and assertion read the model on the python engine", + tests=("helper_and_assertion_read_model",), + command=("--no-color", "test"), + engine="python", + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + ), + HelperReferenceE2ETestCase( + description="only a helper reads the model on the python engine", + tests=("only_helper_reads_model",), + command=("--no-color", "test", "--select", "item_totals"), + engine="python", + expected_output_fragments=( + "1 selected", + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_amount", + ), + ), + HelperReferenceE2ETestCase( + description="helper reads a helper and a mock on the python engine", + tests=("helper_reads_helper_and_mock",), + command=("--no-color", "test"), + engine="python", + expected_output_fragments=( + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_mocked_amount", + ), + ), + HelperReferenceE2ETestCase( + description="helper reads a helper and a mock without sql analysis on the python engine", + tests=("helper_reads_helper_and_mock",), + command=("--no-color", "test", "--no-sql-analysis"), + engine="python", + expected_output_fragments=( + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_mocked_amount", + ), + ), + HelperReferenceE2ETestCase( + description="helper and assertion read the model on the native engine", + tests=("helper_and_assertion_read_model",), + command=("--no-color", "test"), + engine="native", + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + ), + HelperReferenceE2ETestCase( + description="only a helper reads the model on the native engine", + tests=("only_helper_reads_model",), + command=("--no-color", "test", "--select", "item_totals"), + engine="native", + expected_output_fragments=( + "1 selected", + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_amount", + ), + ), + HelperReferenceE2ETestCase( + description="helper reads a helper and a mock on the native engine", + tests=("helper_reads_helper_and_mock",), + command=("--no-color", "test"), + engine="native", + expected_output_fragments=( + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_mocked_amount", + ), + ), + HelperReferenceE2ETestCase( + description="helper reads a helper and a mock without sql analysis on the native engine", + tests=("helper_reads_helper_and_mock",), + command=("--no-color", "test", "--no-sql-analysis"), + engine="native", + expected_output_fragments=( + "PASS=1 FAIL=0 TOTAL=1", + "assertion doubles_mocked_amount", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_models( + test_case: HelperReferenceE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="helper_reference_project", + repo_files=build_helper_reference_project_files(tests=test_case.tests), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 0, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + HelperReferenceErrorE2ETestCase( + description="helpers reading each other on the python engine", + test_name="helper_cycle", + engine="python", + expected_message=( + "SQL test CTE 'first_rows' reads itself through 'first_rows' -> " + "'second_rows' -> 'first_rows', so the test query cannot define its CTEs in " + "dependency order" + ), + expected_line=7, + expected_column=1, + expected_help_fragment="shared_rows AS (SELECT ...)", + ), + HelperReferenceErrorE2ETestCase( + description="helper reading an unknown model on the python engine", + test_name="helper_reads_unknown_model", + engine="python", + expected_message=( + "SQL test helper CTE 'missing_rows' references unknown model 'item_archive'" + ), + expected_line=7, + expected_column=38, + expected_help_fragment='__ref("")', + ), + HelperReferenceErrorE2ETestCase( + description="mock reading a helper that reads a model on the python engine", + test_name="mock_reads_referencing_helper", + engine="python", + expected_message=( + "SQL test mock '__ref__items' reads helper CTE 'base_rows', which calls " + '__ref("item_totals"); mocks and fixtures are defined before the models the ' + "test runs, so the helper cannot be resolved for them" + ), + expected_line=7, + expected_column=35, + expected_help_fragment='FROM __ref__item_totals rather than FROM __ref("item_totals")', + ), + HelperReferenceErrorE2ETestCase( + description="helpers reading each other on the native engine", + test_name="helper_cycle", + engine="native", + expected_message=( + "SQL test CTE 'first_rows' reads itself through 'first_rows' -> " + "'second_rows' -> 'first_rows', so the test query cannot define its CTEs in " + "dependency order" + ), + expected_line=7, + expected_column=1, + expected_help_fragment="shared_rows AS (SELECT ...)", + ), + HelperReferenceErrorE2ETestCase( + description="helper reading an unknown model on the native engine", + test_name="helper_reads_unknown_model", + engine="native", + expected_message=( + "SQL test helper CTE 'missing_rows' references unknown model 'item_archive'" + ), + expected_line=7, + expected_column=38, + expected_help_fragment='__ref("")', + ), + HelperReferenceErrorE2ETestCase( + description="mock reading a helper that reads a model on the native engine", + test_name="mock_reads_referencing_helper", + engine="native", + expected_message=( + "SQL test mock '__ref__items' reads helper CTE 'base_rows', which calls " + '__ref("item_totals"); mocks and fixtures are defined before the models the ' + "test runs, so the helper cannot be resolved for them" + ), + expected_line=7, + expected_column=35, + expected_help_fragment='FROM __ref__item_totals rather than FROM __ref("item_totals")', + ), + ), + ids=lambda case: case.description, +) +def test_given_unresolvable_helper_references_when_compiling_then_compile_reports_them( + test_case: HelperReferenceErrorE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="helper_reference_project", + repo_files=build_helper_reference_project_files(tests=(test_case.test_name,)), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 1, output + payload: dict[str, Any] = json.loads(result.stdout) + codes: list[str] = [diagnostic["code"] for diagnostic in payload["diagnostics"]] + assert codes == [_HELPER_ERROR_CODE], output + diagnostic: dict[str, Any] = payload["diagnostics"][0] + assert diagnostic["message"] == test_case.expected_message, output + assert diagnostic["location"]["path"] == f"tests/unit/test_{test_case.test_name}.sql" + assert diagnostic["location"]["line"] == test_case.expected_line, output + assert diagnostic["location"]["column"] == test_case.expected_column, output + assert test_case.expected_help_fragment in diagnostic["help"], output diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 0c64348c3a..df6efc0b7c 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -141,6 +141,33 @@ __assert__no_unexpected_order_ids AS ( SQLBuild emits each helper and mock a query needs once, in dependency order, including mocks that a helper reaches through another mock. +Helpers can also call the references an `__assert__` CTE can, such as `__ref()`, `__source()` and +`__seed()`, and they resolve exactly as there: a mocked relation reads its mock, and an unmocked +`__ref("")` reads the model's real SQL, so the test runs that model and +`sqb test --select ` selects the test even when only a helper reads it. SQLBuild emits a +helper after the mocks and models it reads: + +```sql +TEST(); + +WITH +__ref__stg_orders AS ( + SELECT 1 AS order_id, 3 AS quantity +), +order_totals AS ( + SELECT order_id, quantity FROM __ref("orders") +), +__assert__quantities_are_positive AS ( + SELECT order_id FROM order_totals WHERE quantity <= 0 +) +``` + +Compile rejects helper references the test query cannot resolve with `P013`, at the helper or call +that causes it: a `__ref()` to a model that does not exist, helpers or mocks that read each other +in a cycle, and a mock or `__table_fn__` fixture that reads a helper calling a reference. Mocks +are defined before the models a test runs, so a mock reads other mocks by CTE name, such as +`FROM __ref__stg_orders`, rather than through `__ref("stg_orders")`. + ## Multi-model tests Tests can span multiple models in a single file. Mock your sources, define an expected output for the model you care about, and SQLBuild automatically resolves every intermediate model using its real SQL. From 7b3f95864a0eab4a55032fd14fac1c9ab81c0475 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:51:56 +0100 Subject: [PATCH 2/2] fix(test): limit helper CTE references to helpers the test reads Resolve, check and count references only in helpers reachable from the test's expected rows and assertions, so unread helpers and mocks behave as before. Inlined helpers now resolve in expected rows too, relations resolved helpers read win over a step's own copy so assertions and helpers share one mock CTE, nested CTEs shadowing a helper keep the existing redefinition error, and the mock check only covers mocks and fixtures the test uses. --- .../_helpers/sql_tests/helper_scope.rs | 83 +++- .../compiler/_helpers/sql_tests/planning.rs | 53 ++- .../src/compiler/tests/helpers.rs | 3 +- .../tests/test_sql_test_helper_references.rs | 77 ++++ .../src/compiler/tests/test_types.rs | 2 + .../references/docs/concepts/testing.md | 13 +- .../compile/_helpers/assembly/project.py | 24 +- .../compile/_helpers/attachment/sql_tests.py | 32 +- .../compile/_helpers/sql_tests/helper_ctes.py | 414 ++++++++++++------ src/sqlbuild/compiler/compile/models.py | 12 + .../_helpers/sql_tests/native_planning.py | 1 + .../cli/commands/main/test/_test_types.py | 14 + .../cli/commands/main/test/helpers.py | 37 +- .../main/test/test_helper_references.py | 178 +++++++- .../content/docs/docs/concepts/testing.mdx | 13 +- 15 files changed, 737 insertions(+), 219 deletions(-) diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs index e0d16187f3..3dbcec432b 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/helper_scope.rs @@ -24,6 +24,8 @@ use crate::constants::{ pub(crate) struct HelperScope { pub(crate) ctes: Vec<(String, String)>, pub(crate) reached_mocks: HashSet, + /// Mock and model CTEs resolved helpers read, which defer to the reader's own copy. + pub(crate) generated: HashSet, } /// One mock CTE other test CTEs can read by its generated name. @@ -292,12 +294,48 @@ impl ScopeGraph { order.ordered } - /// Helpers whose SQL, as placed in the test query, calls a reference such as `__ref()`. - pub(crate) fn referencing_helpers(&self, patterns: &SqlTestPatterns) -> Vec<(usize, String)> { + /// Helpers the test's expected rows and assertions read, except those a mock it uses reads. + pub(crate) fn read_helpers( + &self, + readers: &[&str], + used_mocks: &HashSet, + patterns: &SqlTestPatterns, + ) -> HashSet { + let helpers = |nodes: Vec| -> HashSet { + nodes + .into_iter() + .filter_map(|node| match node { + ScopeNode::Helper(index) => Some(index), + ScopeNode::Mock(_) => None, + }) + .collect() + }; + let mut read: HashSet = HashSet::new(); + for sql in readers { + read.extend(helpers( + self.closure(&identifier_tokens(sql, patterns), true), + )); + } + for mock in &self.mocks { + if used_mocks.contains(&mock.mock_name) { + for index in helpers(self.closure(&mock.tokens, true)) { + read.remove(&index); + } + } + } + read + } + + /// Read helpers whose SQL, as placed in the test query, calls a reference such as `__ref()`. + pub(crate) fn referencing_helpers( + &self, + read: &HashSet, + patterns: &SqlTestPatterns, + ) -> Vec<(usize, String)> { self.helpers .iter() .enumerate() - .filter(|(_, helper)| helper.resolved.is_none()) + .filter(|(index, helper)| helper.resolved.is_none() && read.contains(index)) .filter_map(|(index, _)| { let sql = self.placed_helper_sql(index); (patterns.test_reference.is_match(sql) || patterns.udf.is_match(sql)) @@ -388,6 +426,7 @@ impl ScopeGraph { let mut scope = HelperScope { ctes: Vec::new(), reached_mocks: HashSet::new(), + generated: HashSet::new(), }; for node in nodes { if let ScopeNode::Mock(index) = node { @@ -401,6 +440,12 @@ impl ScopeGraph { scope .reached_mocks .extend(resolved.reached_mocks.iter().cloned()); + scope.generated.extend( + resolved + .lifted_ctes + .iter() + .map(|(name, _)| name.to_ascii_lowercase()), + ); scope.ctes = with_unique_ctes( std::mem::take(&mut scope.ctes), resolved.lifted_ctes.iter().cloned(), @@ -487,6 +532,7 @@ pub(crate) fn helper_scope_ctes( return Ok(HelperScope { ctes: Vec::new(), reached_mocks: HashSet::new(), + generated: HashSet::new(), }); } let nodes = graph.closure(&identifier_tokens(sql, patterns), true); @@ -503,14 +549,31 @@ pub(crate) fn helper_scope_ctes( Ok(graph.collect_scope(nodes)) } -/// Append scoped CTEs after a step's generated CTEs, keeping one definition per name. +/// Merge scoped CTEs into a step's CTEs; relations resolved helpers read come first and win. pub(crate) fn merged_scoped_ctes( - mut lifted: Vec<(String, String)>, - scoped: Vec<(String, String)>, + lifted: Vec<(String, String)>, + scoped: HelperScope, file_label: &str, ) -> Result, String> { - for (name, sql) in scoped { - match lifted + type Ctes = Vec<(String, String)>; + let (generated, authored): (Ctes, Ctes) = scoped + .ctes + .into_iter() + .partition(|(name, _)| scoped.generated.contains(&name.to_ascii_lowercase())); + let mut merged: Vec<(String, String)> = lifted; + if !generated.is_empty() { + let own = std::mem::replace(&mut merged, generated); + for (name, sql) in own { + if !merged + .iter() + .any(|(existing, _)| existing.eq_ignore_ascii_case(&name)) + { + merged.push((name, sql)); + } + } + } + for (name, sql) in authored { + match merged .iter() .find(|(existing, _)| existing.eq_ignore_ascii_case(&name)) { @@ -520,10 +583,10 @@ pub(crate) fn merged_scoped_ctes( "SQL test '{file_label}' defines CTE '{name}', which conflicts with the generated CTE" ))); } - None => lifted.push((name, sql)), + None => merged.push((name, sql)), } } - Ok(lifted) + Ok(merged) } /// Names the comparison renderer generates, which a scoped helper CTE must not shadow. diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs index e111b50197..a163db7201 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs @@ -154,6 +154,8 @@ enum TestPayload { expected_model_names: Vec, #[serde(default)] assertion_ctes: Vec, + #[serde(default)] + helper_target_model_names: Vec, }, Direct { mode: String, @@ -300,6 +302,7 @@ struct ModelTestPlan { expected_ctes: Vec, expected_model_names: Vec, assertion_ctes: Vec, + helper_target_model_names: Vec, } struct TextualChainRequest<'a> { @@ -579,10 +582,16 @@ pub(crate) fn resolve_chains_json(request_json: &str) -> Result model_query_overrides, expected_model_names, assertion_ctes, + helper_target_model_names, .. } => { let fixtures = classify_fixtures(authored_ctes, Vec::new(), assertion_ctes); - let expected_names = chain_root_names(expected_model_names, &fixtures, &patterns); + let expected_names = chain_root_names( + expected_model_names, + helper_target_model_names, + &fixtures, + &patterns, + ); topo_sort_model_chain(TopoSortRequest { expected_names: &expected_names, models: &models, @@ -596,13 +605,15 @@ pub(crate) fn resolve_chains_json(request_json: &str) -> Result serde_json::to_string(&ChainBatchResponse { chains }).map_err(|error| error.to_string()) } +/// Models the test runs: expected models, assertion targets, and the compiler's helper targets. fn chain_root_names( mut expected_names: Vec, + helper_target_names: Vec, fixtures: &TestFixtures, patterns: &SqlTestPatterns, ) -> Vec { expected_names.extend(assertion_ref_targets(&fixtures.assertions, patterns)); - expected_names.extend(helper_ref_targets(fixtures, patterns)); + expected_names.extend(helper_target_names); dedupe(expected_names) } @@ -736,6 +747,7 @@ fn plan_test(test: TestInput, context: &ProjectContext) -> Result plan_model_test( ModelTestPlan { test_name: test.name, @@ -745,6 +757,7 @@ fn plan_test(test: TestInput, context: &ProjectContext) -> Result) -> Result<(), reachable_mocks, } = request; let patterns = &readers.context.patterns; + let read_sql: Vec<&str> = fixtures + .expected + .values() + .chain(fixtures.assertions.iter().map(|(_, sql)| sql)) + .map(String::as_str) + .collect(); + let read_helpers = fixtures + .scope + .read_helpers(&read_sql, reachable_mocks, patterns); let mut resolved_helpers: Vec<(usize, ResolvedHelper)> = Vec::new(); - for (index, helper_sql) in fixtures.scope.referencing_helpers(patterns) { + for (index, helper_sql) in fixtures.scope.referencing_helpers(&read_helpers, patterns) { let resolved = readers.resolve(&helper_sql, fixtures)?; reachable_mocks.extend(resolved.chain_reached); resolved_helpers.push(( @@ -1039,6 +1066,7 @@ fn resolve_referencing_helpers(request: HelperResolution<'_, '_>) -> Result<(), let scope = helper_scope_ctes(sql, fixtures, patterns, readers.file_label)?; reachable_mocks.extend(scope.reached_mocks); step.expected_lifted_ctes = scope.ctes; + step.expected_cte_sql = Some(fixtures.scope.reader_sql(sql, patterns)); } } Ok(()) @@ -1837,19 +1865,6 @@ fn assertion_ref_targets( dedupe(targets) } -/// Unmocked models helper CTEs read, which the test runs so helpers can read them. -fn helper_ref_targets(fixtures: &TestFixtures, patterns: &SqlTestPatterns) -> Vec { - let mut targets: Vec = Vec::new(); - for helper in &fixtures.helpers { - targets.extend( - marker_names(&patterns.reference, &patterns.lexical, &helper.sql_body) - .into_iter() - .filter(|name| !fixtures.mock_refs.contains_key(name)), - ); - } - dedupe(targets) -} - fn has_unresolved_test_reference(sql: &str, patterns: &SqlTestPatterns) -> bool { if !patterns.test_reference.is_match(sql) { return false; diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index a54830f1a7..88b54d4943 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -1833,7 +1833,8 @@ fn helper_reference_test(case: &HelperReferenceTestCase) -> Value { .iter() .map(|(name, _)| name.trim_start_matches("__expected__")) .collect::>(), - "assertionCtes": named(case.assertions) + "assertionCtes": named(case.assertions), + "helperTargetModelNames": case.helper_targets } }) } diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs index 758cba3d3c..f05fd17ccf 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_helper_references.rs @@ -23,6 +23,7 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen helpers: &[HELPER_READS_MODEL], expected: &[], assertions: &[ASSERT_DOUBLED], + helper_targets: &["orders"], expected_chain: &["stg_orders", "orders"], expected_order: &[ ("__source__raw_orders", "__ref__stg_orders"), @@ -33,6 +34,7 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen expected_fragments: &[ "__helper__doubled AS (SELECT order_id, amount_doubled FROM __ref__orders)", ], + expected_absent_fragments: &[], }, HelperReferenceTestCase { description: "helper inlined without sql analysis reads the model CTE", @@ -40,6 +42,7 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen helpers: &[HELPER_READS_MODEL], expected: &[], assertions: &[ASSERT_DOUBLED], + helper_targets: &["orders"], expected_chain: &["stg_orders", "orders"], expected_order: &[ ("__source__raw_orders", "__ref__orders"), @@ -48,6 +51,7 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen expected_fragments: &[ "WITH doubled AS (SELECT order_id, amount_doubled FROM __ref__orders)", ], + expected_absent_fragments: &[], }, HelperReferenceTestCase { description: "helper reading another helper, a model and a seed is ordered after all three", @@ -69,6 +73,7 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen "__assert__has_region", "SELECT order_id FROM with_region WHERE region_name IS NULL", )], + helper_targets: &["stg_orders", "orders"], expected_chain: &["stg_orders", "orders"], expected_order: &[ ("__ref__orders", "__helper__base"), @@ -81,6 +86,7 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen "FROM __helper__base AS b JOIN __ref__stg_orders AS s USING (order_id) \ JOIN __seed__regions AS r USING (region_id)", ], + expected_absent_fragments: &[], }, HelperReferenceTestCase { description: "helper read by expected rows resolves sources and seeds to their mocks", @@ -95,6 +101,7 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen "SELECT order_id, amount_doubled FROM expected_rows", )], assertions: &[], + helper_targets: &[], expected_chain: &["stg_orders", "orders"], expected_order: &[ ("__source__raw_orders", "__helper__expected_rows"), @@ -104,6 +111,69 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen expected_fragments: &[ "FROM __source__raw_orders AS s JOIN __seed__regions AS r USING (region_id)", ], + expected_absent_fragments: &[], + }, + HelperReferenceTestCase { + description: "inlined helper read by expected rows reads the model CTE", + sql_analysis_enabled: false, + helpers: &[HELPER_READS_MODEL], + expected: &[( + "__expected__orders", + "SELECT order_id, amount_doubled FROM doubled", + )], + assertions: &[], + helper_targets: &["orders"], + expected_chain: &["stg_orders", "orders"], + expected_order: &[("__ref__orders", "__expected__orders")], + expected_fragments: &[ + "__expected__orders AS (WITH doubled AS (SELECT order_id, amount_doubled FROM __ref__orders)", + ], + expected_absent_fragments: &["__helper__"], + }, + HelperReferenceTestCase { + description: "helper and assertion reading a mocked model through different paths share one mock CTE", + sql_analysis_enabled: true, + helpers: &[ + ( + "__ref__stg_orders", + "SELECT order_id, 10 AS amount FROM base_ids", + ), + ("base_ids", "SELECT 1 AS order_id"), + ( + "doubled", + "SELECT o.order_id, o.amount_doubled FROM __ref(\"orders\") AS o", + ), + ], + expected: &[], + assertions: &[( + "__assert__doubles_amount", + "SELECT order_id FROM doubled JOIN __ref(\"stg_orders\") USING (order_id) \ + WHERE amount_doubled <> 20", + )], + helper_targets: &["orders"], + expected_chain: &["orders", "stg_orders"], + expected_order: &[ + ("__helper__base_ids", "__ref__orders"), + ("__ref__orders", "__helper__doubled"), + ("__helper__doubled", "__assert__doubles_amount"), + ], + expected_fragments: &[], + expected_absent_fragments: &[], + }, + HelperReferenceTestCase { + description: "helper nothing reads is neither resolved nor emitted", + sql_analysis_enabled: true, + helpers: &[("unused_rows", "SELECT order_id FROM __ref(\"orders\")")], + expected: &[( + "__expected__stg_orders", + "SELECT 1 AS order_id, 10 AS amount, 7 AS region_id", + )], + assertions: &[], + helper_targets: &[], + expected_chain: &["stg_orders"], + expected_order: &[("__source__raw_orders", "__expected__stg_orders")], + expected_fragments: &[], + expected_absent_fragments: &["unused_rows", "__ref__orders"], }, ]; for test_case in test_cases { @@ -130,6 +200,13 @@ fn given_helper_cte_references_when_planning_then_references_resolve_in_dependen test_case.description ); } + for fragment in test_case.expected_absent_fragments { + assert!( + !sql.contains(fragment), + "{}: {fragment}: {sql}", + test_case.description + ); + } for fragment in test_case.expected_fragments { assert!( sql.contains(fragment), diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 2cbbbfa7fc..39a9489144 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -125,7 +125,9 @@ pub(crate) struct HelperReferenceTestCase { pub(crate) helpers: &'static [(&'static str, &'static str)], pub(crate) expected: &'static [(&'static str, &'static str)], pub(crate) assertions: &'static [(&'static str, &'static str)], + pub(crate) helper_targets: &'static [&'static str], pub(crate) expected_chain: &'static [&'static str], pub(crate) expected_order: &'static [(&'static str, &'static str)], pub(crate) expected_fragments: &'static [&'static str], + pub(crate) expected_absent_fragments: &'static [&'static str], } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index 0666e7de5d..5eadc2239a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -186,11 +186,14 @@ __assert__quantities_are_positive AS ( ) ``` -Compile rejects helper references the test query cannot resolve with `P013`, at the helper or call -that causes it: a `__ref()` to a model that does not exist, helpers or mocks that read each other -in a cycle, and a mock or `__table_fn__` fixture that reads a helper calling a reference. Mocks -are defined before the models a test runs, so a mock reads other mocks by CTE name, such as -`FROM __ref__stg_orders`, rather than through `__ref("stg_orders")`. +Only helpers the test reads count: those its `__expected__` and `__assert__` CTEs reach, directly +or through other helpers and mocks. A helper nothing reads is left out of the test query, and its +references are neither checked nor run. Compile rejects references the test query cannot resolve +with `P013`, at the helper or call that causes it: a `__ref()` to a model that does not exist, +helpers or mocks that read each other in a cycle, and a mock or `__table_fn__` fixture the test +uses that reads a helper calling a reference. Mocks are defined before the models a test runs, so +a mock reads other mocks by CTE name, such as `FROM __ref__stg_orders`, rather than through +`__ref("stg_orders")`. ## Multi-model tests diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 36529c2ea9..0d8e1b8c9d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -93,6 +93,9 @@ ) from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile._helpers.sharing.binding import shareable_binding_prekeys +from sqlbuild.compiler.compile._helpers.sql_tests.helper_ctes import ( + report_mocks_reading_referencing_helpers, +) from sqlbuild.compiler.compile._helpers.sql_tests.identity import build_sql_test_case_fingerprint from sqlbuild.compiler.compile.classes.stored_model_analyses import StoredModelAnalyses from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME @@ -1746,9 +1749,23 @@ def _assemble_compiled_sql_test( model_payload: CompileModelSqlTestInputPayload = test_input.payload target_model_names = tuple( dict.fromkeys( - (*model_payload.expected_model_names, *model_payload.assertion_target_model_names) + ( + *model_payload.expected_model_names, + *model_payload.assertion_target_model_names, + *model_payload.helper_target_model_names, + ) ) ) + report_mocks_reading_referencing_helpers( + authored_ctes=model_payload.authored_ctes, + reader_ctes=(*model_payload.expected_ctes, *model_payload.assertion_ctes), + model_inputs=model_inputs, + target_model_names=target_model_names, + mock_model_names=model_payload.mock_model_names, + test_file=test_input.test_file, + test_block=test_input.test_block, + syntax=inputs.sql_lexical_syntax, + ) scope_deps = sql_test_scope_deps(expected_model_names=target_model_names) compiled_payload = CompiledModelSqlTestPayload( authored_ctes=model_payload.authored_ctes, @@ -1824,6 +1841,11 @@ def _assemble_compiled_sql_test( if isinstance(test_input.payload, CompileModelSqlTestInputPayload) else () ), + helper_target_model_names=( + test_input.payload.helper_target_model_names + if isinstance(test_input.payload, CompileModelSqlTestInputPayload) + else () + ), target_model_names=target_model_names, tested_resources=tested_resources, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index 33bdc97616..e471b8d787 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -34,9 +34,8 @@ extract_sql_test_ctes, ) from sqlbuild.compiler.compile._helpers.sql_tests.helper_ctes import ( - extract_helper_target_model_names, + helper_target_model_names, report_unresolvable_helper_ctes, - sql_test_helper_ctes, ) from sqlbuild.compiler.compile._helpers.sql_tests.native import ( extract_expanded_sql_tests_cached, @@ -53,6 +52,7 @@ CompileSqlScenarioCte, CompileSqlScenarioCtes, CompileSqlScenarioInput, + CompileSqlTestCte, CompileSqlTestCtes, CompileSqlTestInput, DeclarationExpansionContext, @@ -322,26 +322,27 @@ def build_test_inputs( loaded_macros=loaded_macros, assertion_target_model_names=assertion_target_model_names, ) + helper_targets: tuple[str, ...] = () if isinstance(test_ctes.payload, CompileModelSqlTestCtes): + reader_ctes: tuple[CompileSqlTestCte, ...] = ( + *test_ctes.payload.expected_ctes, + *test_ctes.payload.assertion_ctes, + ) if report_unresolvable_helper_ctes( - model_payload=test_ctes.payload, + authored_ctes=test_ctes.payload.authored_ctes, + reader_ctes=reader_ctes, + mock_model_names=test_ctes.payload.mock_model_names, test_file=test.test_file, test_block=test.test_block, known_model_names=known_model_names, syntax=sql_lexical_syntax, ): continue - assertion_target_model_names = tuple( - dict.fromkeys( - ( - *assertion_target_model_names, - *extract_helper_target_model_names( - helper_ctes=sql_test_helper_ctes(test_ctes.payload.authored_ctes), - mock_model_names=test_ctes.payload.mock_model_names, - syntax=sql_lexical_syntax, - ), - ) - ) + helper_targets = helper_target_model_names( + authored_ctes=test_ctes.payload.authored_ctes, + reader_ctes=reader_ctes, + mock_model_names=test_ctes.payload.mock_model_names, + syntax=sql_lexical_syntax, ) test_inputs.append( CompileSqlTestInput( @@ -353,6 +354,7 @@ def build_test_inputs( test_ctes=test_ctes, tested_resource_names=test.tested_resource_names, assertion_target_model_names=assertion_target_model_names, + helper_target_model_names=helper_targets, ), declaration_usages=test.declaration_usages, parent_name=test.parent_name, @@ -414,6 +416,7 @@ def _build_test_input_payload( test_ctes: CompileSqlTestCtes, tested_resource_names: tuple[str, ...], assertion_target_model_names: tuple[str, ...], + helper_target_model_names: tuple[str, ...], ) -> CompileModelSqlTestInputPayload | CompileDirectLogicSqlTestInputPayload: match test_ctes.payload: case CompileModelSqlTestCtes() as model_payload: @@ -430,6 +433,7 @@ def _build_test_input_payload( assertion_ctes=model_payload.assertion_ctes, assertion_names=model_payload.assertion_names, assertion_target_model_names=assertion_target_model_names, + helper_target_model_names=helper_target_model_names, ) case CompileDirectLogicSqlTestCtes() as direct_logic_payload: return CompileDirectLogicSqlTestInputPayload( diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py index 0226cadad4..1563f60c57 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/helper_ctes.py @@ -1,8 +1,9 @@ -"""Relation references in SQL-test helper CTEs: the models they read and what cannot resolve.""" +"""Relation references in the SQL-test helper CTEs a test reads, and what cannot resolve.""" from __future__ import annotations import re +from collections.abc import Iterable from typing import Any from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic @@ -19,10 +20,11 @@ TABLE_FN_TEST_CTE_PREFIX, ) from sqlbuild.compiler.compile.models import ( - CompileModelSqlTestCtes, + CompileModelInput, CompilerDiagnostic, CompileSqlReference, CompileSqlTestCte, + SqlTestCteGraph, ) from sqlbuild.compiler.compile.types import ( CompiledResourceType, @@ -51,30 +53,81 @@ SqlReferenceKind.TABLE_FUNCTION, } ) +_MOCK_PREFIX_BY_KIND: dict[SqlReferenceKind, str] = { + SqlReferenceKind.REF: REF_TEST_CTE_PREFIX, + SqlReferenceKind.SOURCE: SOURCE_TEST_CTE_PREFIX, + SqlReferenceKind.SEED: SEED_TEST_CTE_PREFIX, + SqlReferenceKind.DBT_REF: DBT_REF_TEST_CTE_PREFIX, + SqlReferenceKind.TABLE_FUNCTION: TABLE_FN_TEST_CTE_PREFIX, +} +_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"[A-Za-z_][\w$]*") -def sql_test_helper_ctes( +def sql_test_cte_graph( + *, authored_ctes: tuple[CompileSqlTestCte, ...], -) -> tuple[CompileSqlTestCte, ...]: - """Return the authored CTEs that are helpers rather than mocks or fixtures.""" + reader_ctes: tuple[CompileSqlTestCte, ...], +) -> SqlTestCteGraph: + """Return the authored CTEs each test CTE reads, ignoring names its own nested CTEs define.""" + + ctes: dict[str, CompileSqlTestCte] = {cte.name.casefold(): cte for cte in authored_ctes} + keys: frozenset[str] = frozenset(ctes) + reads: dict[str, tuple[str, ...]] = {} + parsed_reads: dict[str, tuple[str, ...]] = {} + for key, cte in ctes.items(): + parsed: tuple[str, ...] | None = _parsed_reads(sql=cte.sql_body, keys=keys - {key}) + parsed_reads[key] = parsed or () + reads[key] = ( + parsed if parsed is not None else _token_reads(sql=cte.sql_body, keys=keys - {key}) + ) + reader_reads: list[str] = [] + for reader in reader_ctes: + parsed_reader: tuple[str, ...] | None = _parsed_reads(sql=reader.sql_body, keys=keys) + reader_reads.extend( + parsed_reader + if parsed_reader is not None + else _token_reads(sql=reader.sql_body, keys=keys) + ) + return SqlTestCteGraph( + ctes=ctes, + reads=reads, + parsed_reads=parsed_reads, + reader_reads=tuple(dict.fromkeys(reader_reads)), + ) - return tuple(cte for cte in authored_ctes if not cte.name.startswith(_MOCK_CTE_PREFIXES)) +def reachable_cte_keys(*, graph: SqlTestCteGraph, roots: Iterable[str]) -> tuple[str, ...]: + """Return the authored CTEs reachable from ``roots``, including the roots, in visit order.""" -def extract_helper_target_model_names( + pending: list[str] = [root for root in roots if root in graph.ctes] + visited: list[str] = list(dict.fromkeys(pending)) + while pending: + key: str = pending.pop(0) + for dependency in graph.reads.get(key, ()): + if dependency not in visited: + visited.append(dependency) + pending.append(dependency) + return tuple(visited) + + +def helper_target_model_names( *, - helper_ctes: tuple[CompileSqlTestCte, ...], + authored_ctes: tuple[CompileSqlTestCte, ...], + reader_ctes: tuple[CompileSqlTestCte, ...], mock_model_names: tuple[str, ...], syntax: SqlLexicalSyntax, ) -> tuple[str, ...]: - """Return unmocked models helper CTEs read with ``__ref()``, which the test must run.""" + """Return unmocked models that helpers the test reads call ``__ref()`` on.""" + graph: SqlTestCteGraph = sql_test_cte_graph( + authored_ctes=authored_ctes, reader_ctes=reader_ctes + ) mocked: frozenset[str] = frozenset(mock_model_names) targets: list[str] = [] - for cte in helper_ctes: + for key in _read_helper_keys(graph): targets.extend( reference.ref_name - for reference in _references(sql=cte.sql_body, syntax=syntax) + for reference in _references(sql=graph.ctes[key].sql_body, syntax=syntax) if reference.ref_kind == SqlReferenceKind.REF and reference.ref_name not in mocked ) return tuple(dict.fromkeys(targets)) @@ -82,20 +135,26 @@ def extract_helper_target_model_names( def report_unresolvable_helper_ctes( *, - model_payload: CompileModelSqlTestCtes, + authored_ctes: tuple[CompileSqlTestCte, ...], + reader_ctes: tuple[CompileSqlTestCte, ...], + mock_model_names: tuple[str, ...], test_file: DiscoveredSqlTestFile, test_block: DiscoveredSqlTestBlock, known_model_names: set[str], syntax: SqlLexicalSyntax, ) -> bool: - """Report helper CTEs the test query cannot place or resolve; return whether any was found.""" + """Report read helpers with unknown models or read cycles; return whether any was found.""" - helper_ctes: tuple[CompileSqlTestCte, ...] = sql_test_helper_ctes(model_payload.authored_ctes) - if not helper_ctes: + graph: SqlTestCteGraph = sql_test_cte_graph( + authored_ctes=authored_ctes, reader_ctes=reader_ctes + ) + read_helpers: tuple[str, ...] = _read_helper_keys(graph) + if not read_helpers: return False reporter: _HelperDiagnostics = _HelperDiagnostics(test_file=test_file, test_block=test_block) - mocked: frozenset[str] = frozenset(model_payload.mock_model_names) - for cte in helper_ctes: + mocked: frozenset[str] = frozenset(mock_model_names) + for key in read_helpers: + cte: CompileSqlTestCte = graph.ctes[key] for reference in _references(sql=cte.sql_body, syntax=syntax): if ( reference.ref_kind == SqlReferenceKind.REF @@ -114,98 +173,44 @@ def report_unresolvable_helper_ctes( f"or remove the reference from helper CTE '{cte.name}'." ), ) - dependencies: dict[str, tuple[str, ...]] = _authored_cte_dependencies( - authored_ctes=model_payload.authored_ctes + cycle: tuple[str, ...] | None = _first_cycle( + graph=graph, keys=reachable_cte_keys(graph=graph, roots=graph.reader_reads) ) - names_by_key: dict[str, str] = { - cte.name.casefold(): cte.name for cte in model_payload.authored_ctes - } - cycle: tuple[str, ...] | None = _first_cycle(dependencies) if cycle is not None: - path: str = " -> ".join(f"'{names_by_key[key]}'" for key in cycle) + names: tuple[str, ...] = tuple(graph.ctes[key].name for key in cycle) + path: str = " -> ".join(f"'{name}'" for name in names) reporter.report( - cte_name=names_by_key[cycle[0]], + cte_name=names[0], call=None, message=( - f"SQL test CTE '{names_by_key[cycle[0]]}' reads itself through {path}, so the " - "test query cannot define its CTEs in dependency order" + f"SQL test CTE '{names[0]}' reads itself through {path}, so the test query " + "cannot define its CTEs in dependency order" ), help=( "Break the cycle by moving the shared rows into a helper CTE that reads neither, " "for example shared_rows AS (SELECT ...), and read shared_rows from both CTEs." ), ) - _report_mocks_reading_references( - reporter=reporter, - model_payload=model_payload, - helper_ctes=helper_ctes, - dependencies=dependencies, - syntax=syntax, - ) return reporter.reported -class _HelperDiagnostics: - """Located P013 diagnostics for one SQL test block.""" - - def __init__( - self, *, test_file: DiscoveredSqlTestFile, test_block: DiscoveredSqlTestBlock - ) -> None: - self._test_file: DiscoveredSqlTestFile = test_file - self._block_offset: int = max(test_file.contents.find(test_block.sql_body), 0) - self._test_name: str = test_block.name or test_file.relative_path.stem - self.reported: bool = False - - def report(self, *, cte_name: str, call: str | None, message: str, help: str) -> None: - location: SourceLocation = self._location(cte_name=cte_name, call=call) - self.reported = True - report_compile_diagnostic( - key=( - SQL_TEST_HELPER_REFERENCE_CODE, - self._test_file.relative_path.as_posix(), - f"{location.line:09d}:{location.column:09d}", - message, - ), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=SQL_TEST_HELPER_REFERENCE_CODE, - message=message, - resource_type=CompiledResourceType.SQL_TEST, - resource_name=self._test_name, - location=location, - help=help, - ), - ) - - def _location(self, *, cte_name: str, call: str | None) -> SourceLocation: - contents: str = self._test_file.contents - header: re.Match[str] | None = re.compile( - rf"(? None: - referencing: dict[str, tuple[CompileSqlTestCte, CompileSqlReference]] = {} - for cte in helper_ctes: + """Report mocks the test reads that read a helper calling a reference, which cannot resolve.""" + + referencing: dict[str, CompileSqlReference] = {} + for cte in authored_ctes: + if cte.name.startswith(_MOCK_CTE_PREFIXES): + continue reference: CompileSqlReference | None = next( ( reference @@ -215,31 +220,101 @@ def _report_mocks_reading_references( None, ) if reference is not None: - referencing[cte.name.casefold()] = (cte, reference) + referencing[cte.name.casefold()] = reference if not referencing: return - for mock in model_payload.authored_ctes: - if not mock.name.startswith(_MOCK_CTE_PREFIXES): - continue - helper_key: str | None = _first_reachable( - origin=mock.name.casefold(), targets=referencing, dependencies=dependencies + graph: SqlTestCteGraph = sql_test_cte_graph( + authored_ctes=authored_ctes, reader_ctes=reader_ctes + ) + used_mocks: tuple[str, ...] = _used_mock_keys( + graph=graph, + model_references={ + model_input.model_file.file_path.stem: model_input.references + for model_input in model_inputs + }, + target_model_names=target_model_names, + mocked=frozenset(mock_model_names), + ) + reporter: _HelperDiagnostics = _HelperDiagnostics(test_file=test_file, test_block=test_block) + for mock_key in used_mocks: + helper_key: str | None = next( + ( + key + for key in reachable_cte_keys(graph=graph, roots=graph.reads.get(mock_key, ())) + if key in referencing + ), + None, ) if helper_key is None: continue - helper, reference = referencing[helper_key] - call: str = _reference_call(reference) + mock_name: str = graph.ctes[mock_key].name + helper_name: str = graph.ctes[helper_key].name + call: str = _reference_call(referencing[helper_key]) reporter.report( - cte_name=helper.name, + cte_name=helper_name, call=call, message=( - f"SQL test mock '{mock.name}' reads helper CTE '{helper.name}', which calls " + f"SQL test mock '{mock_name}' reads helper CTE '{helper_name}', which calls " f"{call}; mocks and fixtures are defined before the models the test runs, so " "the helper cannot be resolved for them" ), - help=_mock_reference_help(mock_name=mock.name, call=call, reference=reference), + help=_mock_reference_help( + mock_name=mock_name, call=call, reference=referencing[helper_key] + ), ) +def _read_helper_keys(graph: SqlTestCteGraph) -> tuple[str, ...]: + return tuple( + key + for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads) + if not graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES) + ) + + +def _used_mock_keys( + *, + graph: SqlTestCteGraph, + model_references: dict[str, tuple[CompileSqlReference, ...]], + target_model_names: tuple[str, ...], + mocked: frozenset[str], +) -> tuple[str, ...]: + """Return mocks the test's readers or the models it runs read, in a stable order.""" + + used: list[str] = [ + key + for key in reachable_cte_keys(graph=graph, roots=graph.reader_reads) + if graph.ctes[key].name.startswith(_MOCK_CTE_PREFIXES) + ] + pending: list[str] = [name for name in target_model_names if name not in mocked] + visited: set[str] = set(pending) + while pending: + model_name: str = pending.pop(0) + for reference in model_references.get(model_name, ()): + kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) + for mock_key in _mock_keys(kind=kind, reference=reference): + if mock_key in graph.ctes and mock_key not in used: + used.append(mock_key) + if ( + kind is SqlReferenceKind.REF + and reference.ref_name not in mocked + and reference.ref_name not in visited + ): + visited.add(reference.ref_name) + pending.append(reference.ref_name) + return tuple(used) + + +def _mock_keys(*, kind: SqlReferenceKind, reference: CompileSqlReference) -> tuple[str, ...]: + prefix: str | None = _MOCK_PREFIX_BY_KIND.get(kind) + if prefix is None: + return () + names: list[str] = [reference.ref_name] + if reference.ref_package is not None: + names.append(f"{reference.ref_package}__{reference.ref_name}") + return tuple(f"{prefix}{name}".casefold() for name in names) + + def _mock_reference_help(*, mock_name: str, call: str, reference: CompileSqlReference) -> str: kind: SqlReferenceKind = SqlReferenceKind(reference.ref_kind) if kind is not SqlReferenceKind.TABLE_FUNCTION and reference.ref_package is None: @@ -263,21 +338,9 @@ def _references(*, sql: str, syntax: SqlLexicalSyntax) -> tuple[CompileSqlRefere return scan_sql_reference_calls(sql=sql, syntax=syntax).references -def _authored_cte_dependencies( - *, authored_ctes: tuple[CompileSqlTestCte, ...] -) -> dict[str, tuple[str, ...]]: - """Return each authored CTE's unqualified table reads of other authored CTEs.""" +def _parsed_reads(*, sql: str, keys: frozenset[str]) -> tuple[str, ...] | None: + """Return unqualified table reads of ``keys`` not shadowed by a nested CTE, or None.""" - keys: frozenset[str] = frozenset(cte.name.casefold() for cte in authored_ctes) - dependencies: dict[str, tuple[str, ...]] = {} - for cte in authored_ctes: - key: str = cte.name.casefold() - references: tuple[str, ...] = _table_reference_keys(sql=cte.sql_body, keys=keys) - dependencies[key] = tuple(reference for reference in references if reference != key) - return dependencies - - -def _table_reference_keys(*, sql: str, keys: frozenset[str]) -> tuple[str, ...]: folded: str = sql.casefold() if not any(key in folded for key in keys): return () @@ -285,48 +348,111 @@ def _table_reference_keys(*, sql: str, keys: frozenset[str]) -> tuple[str, ...]: try: parsed: Any = polyglot_module.parse_one(sql, dialect="generic") except polyglot_module.PolyglotError: - return () - references: list[str] = [] + return None + nested: frozenset[str] = _defined_cte_keys(parsed.to_dict()) + reads: list[str] = [] for table in parsed.find_all("table"): if table.arg("schema") is not None or table.arg("catalog") is not None: continue key: str = str(getattr(table, "name", "") or "").casefold() - if key in keys and key not in references: - references.append(key) - return tuple(references) + if key in keys and key not in nested and key not in reads: + reads.append(key) + return tuple(reads) + +def _defined_cte_keys(tree: Any) -> frozenset[str]: + """Return the case-folded names of every CTE a parsed query defines, at any depth.""" -def _first_cycle(dependencies: dict[str, tuple[str, ...]]) -> tuple[str, ...] | None: - """Return the first dependency cycle in authored order, closed on its first node.""" + keys: set[str] = set() + pending: list[Any] = [tree] + while pending: + value: Any = pending.pop() + if isinstance(value, list): + pending.extend(value) + continue + if not isinstance(value, dict): + continue + ctes: Any = value.get("ctes") + for cte in ctes if isinstance(ctes, list) else (): + alias: Any = cte.get("alias") if isinstance(cte, dict) else None + name: Any = alias.get("name") if isinstance(alias, dict) else None + if isinstance(name, str): + keys.add(name.casefold()) + pending.extend(value.values()) + return frozenset(keys) + + +def _token_reads(*, sql: str, keys: frozenset[str]) -> tuple[str, ...]: + tokens: tuple[str, ...] = tuple( + dict.fromkeys(match.casefold() for match in _IDENTIFIER_PATTERN.findall(sql)) + ) + return tuple(token for token in tokens if token in keys) + + +def _first_cycle(*, graph: SqlTestCteGraph, keys: tuple[str, ...]) -> tuple[str, ...] | None: + """Return the first parsed read cycle among ``keys``, closed on its first CTE.""" - for origin in dependencies: + allowed: frozenset[str] = frozenset(keys) + for origin in keys: pending: list[tuple[str, ...]] = [(origin,)] visited: set[str] = set() while pending: path: tuple[str, ...] = pending.pop(0) - for dependency in dependencies.get(path[-1], ()): + for dependency in graph.parsed_reads.get(path[-1], ()): if dependency == origin: return (*path, origin) - if dependency not in visited: + if dependency in allowed and dependency not in visited: visited.add(dependency) pending.append((*path, dependency)) return None -def _first_reachable( - *, - origin: str, - targets: dict[str, tuple[CompileSqlTestCte, CompileSqlReference]], - dependencies: dict[str, tuple[str, ...]], -) -> str | None: - pending: list[str] = list(dependencies.get(origin, ())) - visited: set[str] = set(pending) - while pending: - key: str = pending.pop(0) - if key in targets: - return key - for dependency in dependencies.get(key, ()): - if dependency not in visited: - visited.add(dependency) - pending.append(dependency) - return None +class _HelperDiagnostics: + """Located P013 diagnostics for one SQL test block.""" + + def __init__( + self, *, test_file: DiscoveredSqlTestFile, test_block: DiscoveredSqlTestBlock + ) -> None: + self._test_file: DiscoveredSqlTestFile = test_file + self._block_offset: int = max(test_file.contents.find(test_block.sql_body), 0) + self._test_name: str = test_block.name or test_file.relative_path.stem + self.reported: bool = False + + def report(self, *, cte_name: str, call: str | None, message: str, help: str) -> None: + location: SourceLocation = self._location(cte_name=cte_name, call=call) + self.reported = True + report_compile_diagnostic( + key=( + SQL_TEST_HELPER_REFERENCE_CODE, + self._test_file.relative_path.as_posix(), + f"{location.line:09d}:{location.column:09d}", + message, + ), + diagnostic=CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=SQL_TEST_HELPER_REFERENCE_CODE, + message=message, + resource_type=CompiledResourceType.SQL_TEST, + resource_name=self._test_name, + location=location, + help=help, + ), + ) + + def _location(self, *, cte_name: str, call: str | None) -> SourceLocation: + contents: str = self._test_file.contents + header: re.Match[str] | None = re.compile( + rf"(? amount * 2\n" ")\n" ), + "mocked_model_read_by_helper_and_assertion": ( + "__ref__items AS (\n SELECT item_id, 10 AS amount FROM base_ids\n),\n" + "base_ids AS (SELECT 1 AS item_id),\n" + 'doubled AS (SELECT t.item_id, t.amount_doubled FROM __ref("item_totals") AS t),\n' + "__assert__doubles_amount AS (\n" + ' SELECT item_id FROM doubled JOIN __ref("items") USING (item_id)\n' + " WHERE amount_doubled <> 20\n" + ")\n" + ), + "unread_helpers_and_mocks": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + 'archived_rows AS (SELECT item_id FROM __ref("item_archive")),\n' + 'other_rows AS (SELECT item_id, amount FROM __ref("item_extras")),\n' + "__ref__item_extras AS (SELECT item_id, amount FROM other_rows),\n" + "__expected__item_totals AS (\n SELECT 1 AS item_id, 20 AS amount_doubled\n)\n" + ), + "nested_cte_shadows_helper": ( + "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" + "expected_rows AS (\n" + " WITH doubled AS (SELECT 1 AS item_id, 20 AS amount_doubled)\n" + " SELECT item_id, amount_doubled FROM doubled\n" + "),\n" + "doubled AS (SELECT item_id, amount_doubled FROM expected_rows),\n" + "__expected__item_totals AS (\n SELECT item_id, amount_doubled FROM doubled\n)\n" + ), "helper_cycle": ( "__ref__items AS (\n SELECT 1 AS item_id, 10 AS amount\n),\n" "first_rows AS (SELECT item_id FROM second_rows),\n" @@ -1790,7 +1815,9 @@ def prepare_optional_ceremony_project(*, tmp_path: Path, test_files: dict[str, s } -def build_helper_reference_project_files(*, tests: tuple[str, ...]) -> dict[str, str]: +def build_helper_reference_project_files( + *, tests: tuple[str, ...], sql_analysis: bool = True +) -> dict[str, str]: """Build an items project whose SQL tests read models through helper CTEs.""" files: dict[str, str] = { @@ -1798,7 +1825,13 @@ def build_helper_reference_project_files(*, tests: tuple[str, ...]) -> dict[str, 'name = "helper_reference_demo"\n' 'adapter = "duckdb"\n\n' "[connection]\n" - 'database = "helper_reference_demo.duckdb"\n' + 'database = "helper_reference_demo.duckdb"\n\n' + "[settings]\n" + f"sql_analysis = {str(sql_analysis).lower()}\n" + ), + "models/item_extras.sql": ( + "MODEL (description 'Extra item amounts', materialized table);\n\n" + 'SELECT item_id, amount FROM __ref("items")\n' ), "models/items.sql": ( "MODEL (description 'Base items', materialized table);\n\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py index a603bd2441..badcfe9370 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_references.py @@ -10,6 +10,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + HelperRedefinitionE2ETestCase, HelperReferenceE2ETestCase, HelperReferenceErrorE2ETestCase, ) @@ -19,7 +20,6 @@ from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb _ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" -_HELPER_ERROR_CODE: str = "P013" @pytest.mark.parametrize( @@ -30,76 +30,156 @@ tests=("helper_and_assertion_read_model",), command=("--no-color", "test"), engine="python", + sql_analysis=True, expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + expected_compiled_fragments=("\n__helper__doubled AS (",), + expected_absent_compiled_fragments=("__ref(",), ), HelperReferenceE2ETestCase( description="only a helper reads the model on the python engine", tests=("only_helper_reads_model",), command=("--no-color", "test", "--select", "item_totals"), engine="python", - expected_output_fragments=( - "1 selected", - "PASS=1 FAIL=0 TOTAL=1", - "assertion doubles_amount", - ), + sql_analysis=True, + expected_output_fragments=("1 selected", "PASS=1 FAIL=0 TOTAL=1"), + expected_compiled_fragments=("\n__ref__item_totals AS (",), + expected_absent_compiled_fragments=("__ref(",), ), HelperReferenceE2ETestCase( description="helper reads a helper and a mock on the python engine", tests=("helper_reads_helper_and_mock",), command=("--no-color", "test"), engine="python", + sql_analysis=True, expected_output_fragments=( "PASS=1 FAIL=0 TOTAL=1", "assertion doubles_mocked_amount", ), + expected_compiled_fragments=("\n__helper__joined AS (",), + expected_absent_compiled_fragments=("__ref(",), ), HelperReferenceE2ETestCase( - description="helper reads a helper and a mock without sql analysis on the python engine", + description="inlined helpers read by expected rows and assertions on the python engine", tests=("helper_reads_helper_and_mock",), - command=("--no-color", "test", "--no-sql-analysis"), + command=("--no-color", "test"), engine="python", + sql_analysis=False, expected_output_fragments=( "PASS=1 FAIL=0 TOTAL=1", "assertion doubles_mocked_amount", ), + expected_compiled_fragments=( + "(WITH doubled AS (SELECT item_id, amount_doubled FROM __ref__item_totals)", + ), + expected_absent_compiled_fragments=("__ref(", "__helper__"), + ), + HelperReferenceE2ETestCase( + description="helper and assertion read a mocked model on the python engine", + tests=("mocked_model_read_by_helper_and_assertion",), + command=("--no-color", "test"), + engine="python", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + expected_compiled_fragments=("\n__helper__doubled AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="unread helpers and mocks are ignored on the python engine", + tests=("unread_helpers_and_mocks",), + command=("--no-color", "test"), + engine="python", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1",), + expected_compiled_fragments=(), + expected_absent_compiled_fragments=("archived_rows", "other_rows", "__ref("), + ), + HelperReferenceE2ETestCase( + description="unread helpers add no tested models on the python engine", + tests=("unread_helpers_and_mocks",), + command=("--no-color", "test", "--select", "item_extras"), + engine="python", + sql_analysis=True, + expected_output_fragments=("0 selected",), + expected_compiled_fragments=(), + expected_absent_compiled_fragments=("archived_rows", "other_rows"), ), HelperReferenceE2ETestCase( description="helper and assertion read the model on the native engine", tests=("helper_and_assertion_read_model",), command=("--no-color", "test"), engine="native", + sql_analysis=True, expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + expected_compiled_fragments=("\n__helper__doubled AS (",), + expected_absent_compiled_fragments=("__ref(",), ), HelperReferenceE2ETestCase( description="only a helper reads the model on the native engine", tests=("only_helper_reads_model",), command=("--no-color", "test", "--select", "item_totals"), engine="native", - expected_output_fragments=( - "1 selected", - "PASS=1 FAIL=0 TOTAL=1", - "assertion doubles_amount", - ), + sql_analysis=True, + expected_output_fragments=("1 selected", "PASS=1 FAIL=0 TOTAL=1"), + expected_compiled_fragments=("\n__ref__item_totals AS (",), + expected_absent_compiled_fragments=("__ref(",), ), HelperReferenceE2ETestCase( description="helper reads a helper and a mock on the native engine", tests=("helper_reads_helper_and_mock",), command=("--no-color", "test"), engine="native", + sql_analysis=True, expected_output_fragments=( "PASS=1 FAIL=0 TOTAL=1", "assertion doubles_mocked_amount", ), + expected_compiled_fragments=("\n__helper__joined AS (",), + expected_absent_compiled_fragments=("__ref(",), ), HelperReferenceE2ETestCase( - description="helper reads a helper and a mock without sql analysis on the native engine", + description="inlined helpers read by expected rows and assertions on the native engine", tests=("helper_reads_helper_and_mock",), - command=("--no-color", "test", "--no-sql-analysis"), + command=("--no-color", "test"), engine="native", + sql_analysis=False, expected_output_fragments=( "PASS=1 FAIL=0 TOTAL=1", "assertion doubles_mocked_amount", ), + expected_compiled_fragments=( + "(WITH doubled AS (SELECT item_id, amount_doubled FROM __ref__item_totals)", + ), + expected_absent_compiled_fragments=("__ref(", "__helper__"), + ), + HelperReferenceE2ETestCase( + description="helper and assertion read a mocked model on the native engine", + tests=("mocked_model_read_by_helper_and_assertion",), + command=("--no-color", "test"), + engine="native", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1", "assertion doubles_amount"), + expected_compiled_fragments=("\n__helper__doubled AS (",), + expected_absent_compiled_fragments=("__ref(",), + ), + HelperReferenceE2ETestCase( + description="unread helpers and mocks are ignored on the native engine", + tests=("unread_helpers_and_mocks",), + command=("--no-color", "test"), + engine="native", + sql_analysis=True, + expected_output_fragments=("PASS=1 FAIL=0 TOTAL=1",), + expected_compiled_fragments=(), + expected_absent_compiled_fragments=("archived_rows", "other_rows", "__ref("), + ), + HelperReferenceE2ETestCase( + description="unread helpers add no tested models on the native engine", + tests=("unread_helpers_and_mocks",), + command=("--no-color", "test", "--select", "item_extras"), + engine="native", + sql_analysis=True, + expected_output_fragments=("0 selected",), + expected_compiled_fragments=(), + expected_absent_compiled_fragments=("archived_rows", "other_rows"), ), ), ids=lambda case: case.description, @@ -110,7 +190,9 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="helper_reference_project", - repo_files=build_helper_reference_project_files(tests=test_case.tests), + repo_files=build_helper_reference_project_files( + tests=test_case.tests, sql_analysis=test_case.sql_analysis + ), ) result: subprocess.CompletedProcess[str] = run_sqb( @@ -118,11 +200,25 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m project_dir=project_dir, env={_ENGINE_ENV_VAR: test_case.engine}, ) + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine}, + ) output: str = result.stdout + result.stderr assert result.returncode == 0, output + assert compiled.returncode == 0, compiled.stdout + compiled.stderr for fragment in test_case.expected_output_fragments: assert fragment in output, output + compiled_sql: str = "\n".join( + path.read_text(encoding="utf-8") + for path in sorted((project_dir / "target" / "compiled" / "tests").rglob("*.sql")) + ) + for fragment in test_case.expected_compiled_fragments: + assert fragment in compiled_sql, compiled_sql + for fragment in test_case.expected_absent_compiled_fragments: + assert fragment not in compiled_sql, compiled_sql @pytest.mark.parametrize( @@ -132,6 +228,7 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m description="helpers reading each other on the python engine", test_name="helper_cycle", engine="python", + expected_code="P013", expected_message=( "SQL test CTE 'first_rows' reads itself through 'first_rows' -> " "'second_rows' -> 'first_rows', so the test query cannot define its CTEs in " @@ -145,6 +242,7 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m description="helper reading an unknown model on the python engine", test_name="helper_reads_unknown_model", engine="python", + expected_code="P013", expected_message=( "SQL test helper CTE 'missing_rows' references unknown model 'item_archive'" ), @@ -156,6 +254,7 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m description="mock reading a helper that reads a model on the python engine", test_name="mock_reads_referencing_helper", engine="python", + expected_code="P013", expected_message=( "SQL test mock '__ref__items' reads helper CTE 'base_rows', which calls " '__ref("item_totals"); mocks and fixtures are defined before the models the ' @@ -169,6 +268,7 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m description="helpers reading each other on the native engine", test_name="helper_cycle", engine="native", + expected_code="P013", expected_message=( "SQL test CTE 'first_rows' reads itself through 'first_rows' -> " "'second_rows' -> 'first_rows', so the test query cannot define its CTEs in " @@ -182,6 +282,7 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m description="helper reading an unknown model on the native engine", test_name="helper_reads_unknown_model", engine="native", + expected_code="P013", expected_message=( "SQL test helper CTE 'missing_rows' references unknown model 'item_archive'" ), @@ -193,6 +294,7 @@ def test_given_helper_cte_references_when_testing_then_helpers_read_the_tested_m description="mock reading a helper that reads a model on the native engine", test_name="mock_reads_referencing_helper", engine="native", + expected_code="P013", expected_message=( "SQL test mock '__ref__items' reads helper CTE 'base_rows', which calls " '__ref("item_totals"); mocks and fixtures are defined before the models the ' @@ -223,11 +325,51 @@ def test_given_unresolvable_helper_references_when_compiling_then_compile_report output: str = result.stdout + result.stderr assert result.returncode == 1, output payload: dict[str, Any] = json.loads(result.stdout) - codes: list[str] = [diagnostic["code"] for diagnostic in payload["diagnostics"]] - assert codes == [_HELPER_ERROR_CODE], output diagnostic: dict[str, Any] = payload["diagnostics"][0] + assert diagnostic["code"] == test_case.expected_code, output assert diagnostic["message"] == test_case.expected_message, output assert diagnostic["location"]["path"] == f"tests/unit/test_{test_case.test_name}.sql" assert diagnostic["location"]["line"] == test_case.expected_line, output assert diagnostic["location"]["column"] == test_case.expected_column, output assert test_case.expected_help_fragment in diagnostic["help"], output + + +@pytest.mark.parametrize( + "test_case", + ( + HelperRedefinitionE2ETestCase( + description="nested CTE redefining a helper on the python engine", + engine="python", + expected_output_fragments=("error[P001]", "which redefines helper CTE 'doubled'"), + expected_absent_output_fragments=("P013",), + ), + HelperRedefinitionE2ETestCase( + description="nested CTE redefining a helper on the native engine", + engine="native", + expected_output_fragments=("error[P001]", "which redefines helper CTE 'doubled'"), + expected_absent_output_fragments=("P013",), + ), + ), + ids=lambda case: case.description, +) +def test_given_nested_cte_shadowing_a_helper_when_compiling_then_the_redefinition_is_reported( + test_case: HelperRedefinitionE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="helper_reference_project", + repo_files=build_helper_reference_project_files(tests=("nested_cte_shadows_helper",)), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), + project_dir=project_dir, + env={_ENGINE_ENV_VAR: test_case.engine}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 1, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + for fragment in test_case.expected_absent_output_fragments: + assert fragment not in output, output diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index df6efc0b7c..a416ce8a5c 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -162,11 +162,14 @@ __assert__quantities_are_positive AS ( ) ``` -Compile rejects helper references the test query cannot resolve with `P013`, at the helper or call -that causes it: a `__ref()` to a model that does not exist, helpers or mocks that read each other -in a cycle, and a mock or `__table_fn__` fixture that reads a helper calling a reference. Mocks -are defined before the models a test runs, so a mock reads other mocks by CTE name, such as -`FROM __ref__stg_orders`, rather than through `__ref("stg_orders")`. +Only helpers the test reads count: those its `__expected__` and `__assert__` CTEs reach, directly +or through other helpers and mocks. A helper nothing reads is left out of the test query, and its +references are neither checked nor run. Compile rejects references the test query cannot resolve +with `P013`, at the helper or call that causes it: a `__ref()` to a model that does not exist, +helpers or mocks that read each other in a cycle, and a mock or `__table_fn__` fixture the test +uses that reads a helper calling a reference. Mocks are defined before the models a test runs, so +a mock reads other mocks by CTE name, such as `FROM __ref__stg_orders`, rather than through +`__ref("stg_orders")`. ## Multi-model tests