diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 664dd44e9b..1ddfa641fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,6 +40,14 @@ For focused iteration, run the narrowest relevant pytest target, for example: pytest tests/unit/src/sqlbuild/compiler/compile/test_main.py ``` +To check whether a branch introduces copied or drifting code in Python or Rust, run: + +```bash +uv run dupscore clones --since origin/main +``` + +`make dupscore` reports the whole repository. See `scripts/dupscore/README.md` for options. + ## Tests - Mirror source layout under `tests//src/sqlbuild/...`. diff --git a/Makefile b/Makefile index b8d25c48b4..8542b00209 100644 --- a/Makefile +++ b/Makefile @@ -449,7 +449,7 @@ check-fensu: dupscore: - uv run dupscore report + uv run dupscore clones check: diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_sql.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_sql.rs new file mode 100644 index 0000000000..83e0e26683 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_sql.rs @@ -0,0 +1,76 @@ +//! CTE definition and leading-WITH assembly shared by SQL-test planning and rendering. + +use crate::compiler::_helpers::sql_tests::sql_scan::{comment_end, skip_whitespace}; + +/// Render one `name AS (body)` definition, closing a trailing line comment first. +pub(crate) fn cte_definition_sql(name: &str, sql: &str) -> String { + let body = sql.trim_end(); + let final_line = body.rsplit_once('\n').map_or(body, |(_, line)| line); + let terminator = if final_line.contains("--") { "\n" } else { "" }; + format!("{name} AS ({body}{terminator})") +} + +/// Append CTEs whose names are not already present, keeping first definitions and order. +pub(crate) fn with_unique_ctes( + mut ctes: Vec<(String, String)>, + extra: impl IntoIterator, +) -> Vec<(String, String)> { + for (name, sql) in extra { + if !ctes.iter().any(|(existing, _)| *existing == name) { + ctes.push((name, sql)); + } + } + ctes +} + +/// Prefix CTEs to a query, merging into its own leading WITH clause when present. +pub(crate) fn with_leading_ctes(ctes: &[(String, String)], body: &str) -> String { + if ctes.is_empty() { + return body.to_string(); + } + let definitions: String = ctes + .iter() + .map(|(name, sql)| cte_definition_sql(name, sql)) + .collect::>() + .join(", "); + match leading_with_prefix_end(body) { + Some(end) => format!("{}{definitions}, {}", &body[..end], &body[end..]), + None => format!("WITH {definitions} {body}"), + } +} + +/// Return the offset just after a leading `WITH` (and optional `RECURSIVE`) keyword. +pub(crate) fn leading_with_prefix_end(sql: &str) -> Option { + let mut index = skip_leading_ignorable(sql, 0); + index = keyword_end(sql, index, "WITH")?; + index = skip_leading_ignorable(sql, index); + if let Some(recursive_end) = keyword_end(sql, index, "RECURSIVE") { + index = skip_leading_ignorable(sql, recursive_end); + } + Some(index) +} + +fn skip_leading_ignorable(sql: &str, mut index: usize) -> usize { + loop { + index = skip_whitespace(sql, index); + match comment_end(sql, index) { + Ok(Some(end)) => index = end, + Ok(None) | Err(_) => return index, + } + } +} + +fn keyword_end(sql: &str, start: usize, keyword: &str) -> Option { + let end = start + keyword.len(); + if !sql.get(start..end)?.eq_ignore_ascii_case(keyword) { + return None; + } + if sql + .as_bytes() + .get(end) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + { + return None; + } + Some(end) +} diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs index b91c086eab..625ab2d292 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/extraction.rs @@ -4,6 +4,9 @@ use std::collections::{HashMap, HashSet, VecDeque}; use serde::{Deserialize, Serialize}; +use crate::compiler::_helpers::sql_tests::sql_scan::{ + self, Unclosed, comment_end, quoted_end, skip_whitespace, +}; use crate::constants::{ DIRECT_DEPENDENCY_PATH_LENGTH, MACRO_TEST_MODE, TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE, }; @@ -442,7 +445,7 @@ fn validate_expected( Ok(()) } -fn empty_fixture_marker_matches(sql: &str) -> Result { +pub(crate) fn empty_fixture_marker_matches(sql: &str) -> Result { let mut index = skip_ignorable(sql, 0)?; let Some(select_end) = consume_keyword(sql, index, "SELECT") else { return Ok(false); @@ -1054,48 +1057,21 @@ fn byte_at(sql: &str, index: usize) -> Option { fn char_len(sql: &str, index: usize) -> usize { sql[index..].chars().next().map_or(1, char::len_utf8) } -fn skip_whitespace(sql: &str, mut index: usize) -> usize { - while index < sql.len() { - let Some(c) = sql[index..].chars().next() else { - break; - }; - if !c.is_whitespace() { - break; - } - index += c.len_utf8(); - } - index -} - fn skip_ignorable(sql: &str, mut index: usize) -> Result { loop { index = skip_whitespace(sql, index); - if sql[index..].starts_with("--") { - index = sql[index..] - .find('\n') - .map_or(sql.len(), |value| index + value + 1); - } else if sql[index..].starts_with("/*") { - index = sql[index + 2..].find("*/").map_or_else( - || Err("SQL test contains an unclosed block comment".to_owned()), - |value| Ok(index + 2 + value + 2), - )?; - } else { - return Ok(index); + match comment_end(sql, index).map_err(|error| scan_error_message(error, "SQL test"))? { + Some(end) => index = end, + None => return Ok(index), } } } fn skip_non_code(sql: &str, index: usize) -> Result { - if sql[index..].starts_with("--") { - return Ok(sql[index..] - .find('\n') - .map_or(sql.len(), |value| index + value + 1)); - } - if sql[index..].starts_with("/*") { - return sql[index + 2..] - .find("*/") - .map(|value| index + 2 + value + 2) - .ok_or_else(|| "SQL test contains an unclosed block comment".to_owned()); + if let Some(end) = + comment_end(sql, index).map_err(|error| scan_error_message(error, "SQL test"))? + { + return Ok(end); } if matches!(byte_at(sql, index), Some(b'\'') | Some(b'"') | Some(b'`')) { return skip_quote(sql, index, "SQL test"); @@ -1104,46 +1080,25 @@ fn skip_non_code(sql: &str, index: usize) -> Result { } fn skip_quote(sql: &str, start: usize, context: &str) -> Result { - let Some(quote) = byte_at(sql, start) else { + if byte_at(sql, start).is_none() { return Err(format!("{context} expected a quote")); - }; - let mut index = start + 1; - while index < sql.len() { - if byte_at(sql, index) == Some(quote) { - if byte_at(sql, index + 1) == Some(quote) { - index += 2; - } else { - return Ok(index + 1); - } - } else { - index += char_len(sql, index) - } } - Err(format!("{context} contains an unclosed quoted string")) + quoted_end(sql, start).map_err(|error| scan_error_message(error, context)) } fn matching_paren(sql: &str, open: usize, context: &str) -> Result { - let mut depth = 0; - let mut index = open; - while index < sql.len() { - let next = skip_non_code(sql, index)?; - if next != index { - index = next; - continue; - } - match byte_at(sql, index) { - Some(b'(') => depth += 1, - Some(b')') => { - depth -= 1; - if depth == 0 { - return Ok(index); - } - } - _ => {} - } - index += char_len(sql, index); + sql_scan::matching_paren(sql, open).map_err(|error| match error { + Unclosed::Parenthesis => scan_error_message(error, context), + _ => scan_error_message(error, "SQL test"), + }) +} + +fn scan_error_message(error: Unclosed, context: &str) -> String { + match error { + Unclosed::BlockComment => "SQL test contains an unclosed block comment".to_owned(), + Unclosed::Quote => format!("{context} contains an unclosed quoted string"), + Unclosed::Parenthesis => format!("{context} contains an unclosed parenthesis"), } - Err(format!("{context} contains an unclosed parenthesis")) } fn read_string(sql: &str, start: usize) -> Result<(String, usize), String> { diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs new file mode 100644 index 0000000000..4d59240106 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs @@ -0,0 +1,276 @@ +//! Helper and mock CTEs placed in scope for SQL-test queries, in dependency order. + +use std::collections::{BTreeMap, HashMap, HashSet}; + +use crate::compiler::_helpers::sql_tests::markers::{in_protected_range, protected_ranges}; +use crate::compiler::_helpers::sql_tests::planning::{ + DBT_REF_PREFIX, REF_PREFIX, SEED_PREFIX, SOURCE_PREFIX, SqlTestPatterns, TestFixtures, + compile_error, with_helper_ctes, +}; +use crate::constants::{ + QUOTED_IDENTIFIER_DELIMITER_BYTES, SQL_TEST_ACTUAL_CTE, SQL_TEST_ACTUAL_CTE_PREFIX, + SQL_TEST_EXPECTED_CTE, +}; + +/// Helper CTEs, and the mocks they read by CTE name, that one test query needs in scope. +pub(crate) struct HelperScope { + pub(crate) ctes: Vec<(String, String)>, + pub(crate) reached_mocks: HashSet, +} + +/// One mock CTE other test CTEs can read by its generated name. +pub(crate) struct ScopeMock { + pub(crate) generated_name: String, + pub(crate) mock_name: String, + pub(crate) sql: String, + tokens: Vec, +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +enum ScopeNode { + Helper(usize), + Mock(usize), +} + +/// Name references between one test's helper and mock CTEs, tokenized once per test. +#[derive(Default)] +pub(crate) struct ScopeGraph { + helper_tokens: Vec>, + helper_indexes: HashMap, + mocks: Vec, + mock_indexes: HashMap, + mocks_by_generated_name: HashMap, +} + +/// Graph nodes in dependency-first order, built by depth-first traversal. +struct ScopeOrder<'a> { + graph: &'a ScopeGraph, + visited: HashSet, + ordered: Vec, +} + +impl ScopeOrder<'_> { + fn visit_tokens(&mut self, tokens: &[String]) { + for token in tokens { + if let Some(node) = self.graph.node(token) { + self.visit(node); + } + } + } + + fn visit(&mut self, node: ScopeNode) { + if !self.visited.insert(node) { + return; + } + let graph = self.graph; + self.visit_tokens(graph.tokens(node)); + self.ordered.push(node); + } +} + +impl ScopeGraph { + /// Tokenize every helper body and authored mock body of one test. + pub(crate) fn new(fixtures: &TestFixtures, patterns: &SqlTestPatterns) -> Self { + let mock_groups: [(&str, &BTreeMap); 4] = [ + (REF_PREFIX, &fixtures.mock_refs), + (SOURCE_PREFIX, &fixtures.mock_sources), + (SEED_PREFIX, &fixtures.mock_seeds), + (DBT_REF_PREFIX, &fixtures.mock_dbt_refs), + ]; + let mut mocks: Vec = Vec::new(); + for (prefix, group) in mock_groups { + for (name, body) in group { + mocks.push(ScopeMock { + generated_name: format!("{prefix}{name}"), + mock_name: name.clone(), + sql: with_helper_ctes(body, &fixtures.helpers), + tokens: identifier_tokens(body, patterns), + }); + } + } + Self { + helper_tokens: fixtures + .helpers + .iter() + .map(|cte| identifier_tokens(&cte.sql_body, patterns)) + .collect(), + helper_indexes: fixtures + .helpers + .iter() + .enumerate() + .map(|(index, cte)| (cte.name.to_ascii_lowercase(), index)) + .collect(), + mock_indexes: mocks + .iter() + .enumerate() + .map(|(index, mock)| (mock.generated_name.to_ascii_lowercase(), index)) + .collect(), + mocks_by_generated_name: mocks + .iter() + .enumerate() + .map(|(index, mock)| (mock.generated_name.clone(), index)) + .collect(), + mocks, + } + } + + fn node(&self, token: &str) -> Option { + self.helper_indexes + .get(token) + .map(|index| ScopeNode::Helper(*index)) + .or_else(|| { + self.mock_indexes + .get(token) + .map(|index| ScopeNode::Mock(*index)) + }) + } + + fn tokens(&self, node: ScopeNode) -> &[String] { + match node { + ScopeNode::Helper(index) => &self.helper_tokens[index], + ScopeNode::Mock(index) => &self.mocks[index].tokens, + } + } + + fn closure(&self, tokens: &[String]) -> Vec { + let mut order = ScopeOrder { + graph: self, + visited: HashSet::new(), + ordered: Vec::new(), + }; + order.visit_tokens(tokens); + order.ordered + } + + /// The generated CTE body of one mock, with every helper CTE in scope. + pub(crate) fn mock_sql(&self, generated_name: &str) -> Option<&str> { + self.mocks_by_generated_name + .get(generated_name) + .map(|index| self.mocks[*index].sql.as_str()) + } + + /// The parenthesized body of one mock inlined in place of its relation marker. + pub(crate) fn inlined_mock_sql(&self, generated_name: &str) -> Option { + self.mock_sql(generated_name).map(|sql| format!("({sql})")) + } + + /// Mocks that one mock's body reads through helpers or by name, dependencies first. + pub(crate) fn mock_dependencies(&self, generated_name: &str) -> Vec<&ScopeMock> { + let Some(root) = self.mocks_by_generated_name.get(generated_name).copied() else { + return Vec::new(); + }; + self.closure(&self.mocks[root].tokens) + .into_iter() + .filter_map(|node| match node { + ScopeNode::Mock(index) if index != root => Some(&self.mocks[index]), + _ => None, + }) + .collect() + } +} + +/// Return the helper and mock CTEs `sql` reads, transitively and dependencies first. +pub(crate) fn helper_scope_ctes( + sql: &str, + fixtures: &TestFixtures, + patterns: &SqlTestPatterns, + file_label: &str, +) -> Result { + let mut scope = HelperScope { + ctes: Vec::new(), + reached_mocks: HashSet::new(), + }; + if fixtures.helpers.is_empty() && fixtures.scope.mocks.is_empty() { + return Ok(scope); + } + for node in fixtures.scope.closure(&identifier_tokens(sql, patterns)) { + match node { + ScopeNode::Mock(index) => { + let mock = &fixtures.scope.mocks[index]; + scope.reached_mocks.insert(mock.mock_name.clone()); + scope + .ctes + .push((mock.generated_name.clone(), mock.sql.clone())); + } + ScopeNode::Helper(index) => { + let helper = &fixtures.helpers[index]; + if is_generated_cte_name(&helper.name) { + return Err(compile_error(&format!( + "SQL test '{file_label}' defines CTE '{}', which conflicts with the generated CTE", + helper.name + ))); + } + scope + .ctes + .push((helper.name.clone(), helper.sql_body.clone())); + } + } + } + Ok(scope) +} + +/// Append scoped CTEs after a step's generated CTEs, keeping one definition per name. +pub(crate) fn merged_scoped_ctes( + mut lifted: Vec<(String, String)>, + scoped: Vec<(String, String)>, + file_label: &str, +) -> Result, String> { + for (name, sql) in scoped { + match lifted + .iter() + .find(|(existing, _)| existing.eq_ignore_ascii_case(&name)) + { + Some((_, existing_sql)) if *existing_sql == sql => {} + Some(_) => { + return Err(compile_error(&format!( + "SQL test '{file_label}' defines CTE '{name}', which conflicts with the generated CTE" + ))); + } + None => lifted.push((name, sql)), + } + } + Ok(lifted) +} + +/// Names the comparison renderer generates, which a scoped helper CTE must not shadow. +fn is_generated_cte_name(name: &str) -> bool { + let lowered = name.to_ascii_lowercase(); + lowered == SQL_TEST_ACTUAL_CTE + || lowered == SQL_TEST_EXPECTED_CTE + || lowered.starts_with(SQL_TEST_ACTUAL_CTE_PREFIX) +} + +/// Distinct lowercase identifiers outside strings and comments, in first-appearance order. +fn identifier_tokens(sql: &str, patterns: &SqlTestPatterns) -> Vec { + let protected = protected_ranges(&patterns.protected, sql); + let mut seen: HashSet = HashSet::new(); + let mut tokens: Vec = Vec::new(); + let mut push = |token: &str| { + let lowered = token.to_ascii_lowercase(); + if seen.insert(lowered.clone()) { + tokens.push(lowered); + } + }; + for found in patterns.identifier.find_iter(sql) { + let preceded_by_identifier = sql[..found.start()] + .chars() + .next_back() + .is_some_and(|character| character.is_alphanumeric() || character == '_'); + if !preceded_by_identifier && !in_protected_range(found.start(), &protected) { + push(found.as_str()); + } + } + for (start, end) in protected { + let text = &sql[start..end]; + if let Some(quote) = text + .chars() + .next() + .filter(|quote| matches!(quote, '"' | '`')) + && text.len() >= QUOTED_IDENTIFIER_DELIMITER_BYTES + && text.ends_with(quote) + { + push(&text[1..text.len() - 1]); + } + } + tokens +} diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs new file mode 100644 index 0000000000..da1babc312 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs @@ -0,0 +1,156 @@ +//! Marker and call-site text replacement shared by SQL-test planning. + +use std::collections::HashSet; + +use regex::{Captures, Regex}; + +use crate::compiler::_helpers::sql_tests::planning::compile_error; +use crate::compiler::_helpers::sql_tests::sql_scan::{Unclosed, matching_paren, skip_whitespace}; + +pub(crate) fn replace_callable_markers( + sql: &str, + pattern: &Regex, + protected_pattern: &Regex, + mut replacement: F, +) -> Result<(String, HashSet), String> +where + F: FnMut(&str, &str) -> Option<(String, bool)>, +{ + let protected = protected_ranges(protected_pattern, sql); + let mut output = String::with_capacity(sql.len()); + let mut reached: HashSet = HashSet::new(); + let mut cursor = 0; + for captures in pattern.captures_iter(sql) { + let Some(full) = captures.get(0) else { + continue; + }; + if full.start() < cursor || in_protected_range(full.start(), &protected) { + continue; + } + let suffix_start = skip_whitespace(sql, full.end()); + if !sql[suffix_start..].starts_with('(') { + continue; + } + let suffix_end = matching_paren_end(sql, suffix_start)?; + let Some(name) = captures.get(1).map(|value| value.as_str()) else { + continue; + }; + let call_suffix = &sql[suffix_start..suffix_end]; + let Some((value, records_reach)) = replacement(name, call_suffix) else { + continue; + }; + output.push_str(&sql[cursor..full.start()]); + output.push_str(&value); + cursor = suffix_end; + if records_reach { + reached.insert(name.to_string()); + } + } + output.push_str(&sql[cursor..]); + Ok((output, reached)) +} + +fn matching_paren_end(sql: &str, open: usize) -> Result { + matching_paren(sql, open) + .map(|close| close + 1) + .map_err(|error| { + compile_error(match error { + Unclosed::BlockComment => "SQL function call contains an unclosed block comment", + Unclosed::Quote => "SQL function call contains an unclosed quoted string", + Unclosed::Parenthesis => "SQL function call contains an unclosed parenthesis", + }) + }) +} + +pub(crate) fn replace_named_markers( + sql: &str, + pattern: &Regex, + protected_pattern: &Regex, + mut replacement: F, +) -> String +where + F: FnMut(&str) -> Option, +{ + replace_marker_captures(sql, pattern, protected_pattern, |captures| { + replacement(captures.get(1)?.as_str()) + }) +} + +pub(crate) fn replace_dbt_ref_markers( + sql: &str, + pattern: &Regex, + protected_pattern: &Regex, + mut replacement: F, +) -> String +where + F: FnMut(&str) -> Option, +{ + replace_marker_captures(sql, pattern, protected_pattern, |captures| { + let first = captures.get(1)?.as_str(); + let name = captures.get(2).map_or_else( + || first.to_string(), + |second| format!("{first}__{}", second.as_str()), + ); + replacement(&name) + }) +} + +fn replace_marker_captures( + sql: &str, + pattern: &Regex, + protected_pattern: &Regex, + mut replacement: F, +) -> String +where + F: FnMut(&Captures<'_>) -> Option, +{ + let protected = protected_ranges(protected_pattern, sql); + let mut output = String::with_capacity(sql.len()); + let mut cursor = 0; + for captures in pattern.captures_iter(sql) { + let Some(full) = captures.get(0) else { + continue; + }; + if in_protected_range(full.start(), &protected) { + continue; + } + let Some(value) = replacement(&captures) else { + continue; + }; + output.push_str(&sql[cursor..full.start()]); + output.push_str(&value); + cursor = full.end(); + } + output.push_str(&sql[cursor..]); + output +} + +pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str) -> Vec { + let protected = protected_ranges(protected_pattern, sql); + let mut names: Vec = Vec::new(); + for captures in pattern.captures_iter(sql) { + let Some(full) = captures.get(0) else { + continue; + }; + if in_protected_range(full.start(), &protected) { + continue; + } + if let Some(name) = captures.get(1) { + names.push(name.as_str().to_string()); + } + } + names +} + +pub(crate) fn protected_ranges(pattern: &Regex, sql: &str) -> Vec<(usize, usize)> { + pattern + .find_iter(sql) + .map(|value| (value.start(), value.end())) + .collect() +} + +pub(crate) fn in_protected_range(index: usize, ranges: &[(usize, usize)]) -> bool { + ranges + .iter() + .any(|(start, end)| index >= *start && index < *end) +} diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs index 271ed3c3ae..7969a2dd71 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs @@ -1,4 +1,8 @@ +pub(crate) mod cte_sql; pub(crate) mod expected_columns; pub(crate) mod extraction; +pub(crate) mod helper_scope; +pub(crate) mod markers; pub(crate) mod planning; pub(crate) mod rendering; +pub(crate) mod sql_scan; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs index 174cd6fafd..58f157ebad 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs @@ -10,7 +10,17 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::Value; +use crate::compiler::_helpers::sql_tests::cte_sql::{ + cte_definition_sql, leading_with_prefix_end, with_leading_ctes, with_unique_ctes, +}; use crate::compiler::_helpers::sql_tests::expected_columns::expected_columns; +use crate::compiler::_helpers::sql_tests::helper_scope::{ + ScopeGraph, helper_scope_ctes, merged_scoped_ctes, +}; +use crate::compiler::_helpers::sql_tests::markers::{ + in_protected_range, marker_names, protected_ranges, replace_callable_markers, + replace_dbt_ref_markers, replace_named_markers, +}; use crate::compiler::_helpers::sql_tests::rendering::{ AssertionStep, ChainStep, RenderRequest, render_comparison_sql, render_dialect, rendered_chain_steps, @@ -20,11 +30,18 @@ use crate::constants::{TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE}; const DEFAULT_WORKERS: usize = 4; const MAX_WORKERS: usize = 4; const WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; -const REF_PREFIX: &str = "__ref__"; -const SOURCE_PREFIX: &str = "__source__"; -const SEED_PREFIX: &str = "__seed__"; -const DBT_REF_PREFIX: &str = "__dbt_ref__"; +pub(crate) const REF_PREFIX: &str = "__ref__"; +pub(crate) const SOURCE_PREFIX: &str = "__source__"; +pub(crate) const SEED_PREFIX: &str = "__seed__"; +pub(crate) const DBT_REF_PREFIX: &str = "__dbt_ref__"; const TABLE_FUNCTION_PREFIX: &str = "__table_fn__"; +pub(crate) const MOCK_CTE_PREFIXES: [&str; 5] = [ + REF_PREFIX, + SOURCE_PREFIX, + SEED_PREFIX, + DBT_REF_PREFIX, + TABLE_FUNCTION_PREFIX, +]; const EXPECTED_PREFIX: &str = "__expected__"; const ASSERT_PREFIX: &str = "__assert__"; const REF_FUNCTION: &str = "__ref"; @@ -131,9 +148,9 @@ enum TestPayload { #[derive(Clone, Debug, Deserialize)] #[serde(rename_all = "camelCase")] -struct CteInput { - name: String, - sql_body: String, +pub(crate) struct CteInput { + pub(crate) name: String, + pub(crate) sql_body: String, } /// One planned SQL test: the executable chain and assertion steps plus optional rendered SQL. @@ -183,7 +200,7 @@ struct ProjectContext { } #[derive(Clone)] -struct SqlTestPatterns { +pub(crate) struct SqlTestPatterns { reference: Regex, source: Regex, seed: Regex, @@ -191,7 +208,8 @@ struct SqlTestPatterns { table_function: Regex, dbt_reference: Regex, test_reference: Regex, - protected: Regex, + pub(crate) protected: Regex, + pub(crate) identifier: Regex, } impl SqlTestPatterns { @@ -209,6 +227,7 @@ impl SqlTestPatterns { protected: compile_pattern( r#"(?s)'(?:''|[^'])*'|\"(?:\"\"|[^\"])*\"|`(?:``|[^`])*`|\$\$.*?\$\$|--[^\n]*|/\*.*?\*/"#, )?, + identifier: compile_pattern(r"[A-Za-z_][A-Za-z0-9_$]*")?, }) } } @@ -236,13 +255,14 @@ pub(crate) struct AnalysisTemplate { type AnalysisTemplateCell = Arc>>; type AnalysisTemplateCache = Mutex>; -struct TestFixtures { - mock_refs: BTreeMap, - mock_sources: BTreeMap, - mock_seeds: BTreeMap, - mock_dbt_refs: BTreeMap, +pub(crate) struct TestFixtures { + pub(crate) mock_refs: BTreeMap, + pub(crate) mock_sources: BTreeMap, + pub(crate) mock_seeds: BTreeMap, + pub(crate) mock_dbt_refs: BTreeMap, mock_table_functions: BTreeMap, - helpers: Vec, + pub(crate) helpers: Vec, + pub(crate) scope: ScopeGraph, expected: BTreeMap, assertions: Vec<(String, String)>, } @@ -279,6 +299,7 @@ struct TextualChain { cte_names: HashMap, bodies: HashMap, dependencies: HashMap>, + mock_ctes: HashMap>, order: Vec, } @@ -307,7 +328,7 @@ impl TextualChain { }; let query_sql = inputs.overrides.get(model_name).unwrap_or(&model.query_sql); let mut chain_references: Vec = Vec::new(); - let (resolved_sql, reached) = resolve_textual_sql(TextualResolutionRequest { + let resolution = resolve_textual_sql(TextualResolutionRequest { query_sql, fixtures: inputs.fixtures, resolved_chain: &self.cte_names, @@ -315,8 +336,10 @@ impl TextualChain { functions: &inputs.context.functions, patterns: &inputs.context.patterns, })?; - reachable.extend(reached); - self.insert(model_name, resolved_sql, chain_references); + reachable.extend(resolution.reached); + self.mock_ctes + .insert(model_name.to_string(), resolution.mock_ctes); + self.insert(model_name, resolution.sql, chain_references); } Ok(reachable) } @@ -349,45 +372,40 @@ impl TextualChain { stack.extend(dependencies.iter().cloned()); } } - self.order - .iter() - .filter(|name| needed.contains(*name)) - .filter_map(|name| { - Some(( - self.cte_names.get(name)?.clone(), - self.bodies.get(name)?.clone(), - )) - }) - .collect() + let mut ctes: Vec<(String, String)> = with_unique_ctes( + Vec::new(), + self.order + .iter() + .filter(|name| needed.contains(*name) || roots.contains(name)) + .filter_map(|name| self.mock_ctes.get(name)) + .flatten() + .cloned(), + ); + ctes.extend( + self.order + .iter() + .filter(|name| needed.contains(*name)) + .filter_map(|name| { + Some(( + self.cte_names.get(name)?.clone(), + self.bodies.get(name)?.clone(), + )) + }), + ); + ctes } fn step(&self, model_name: &str) -> Option { let body_sql = self.bodies.get(model_name)?.clone(); let lifted_ctes = self.closure_ctes(&[model_name.to_string()], false); Some(TextualStep { - resolved_sql: with_generated_ctes(&lifted_ctes, &body_sql), + resolved_sql: with_leading_ctes(&lifted_ctes, &body_sql), lifted_ctes, body_sql, }) } } -/// Prefix generated CTEs to a query, merging into its own leading WITH clause when present. -fn with_generated_ctes(ctes: &[(String, String)], body: &str) -> String { - if ctes.is_empty() { - return body.to_string(); - } - let definitions: String = ctes - .iter() - .map(|(name, sql)| cte_definition_sql(name, sql)) - .collect::>() - .join(", "); - match leading_with_prefix_end(body) { - Some(end) => format!("{}{definitions}, {}", &body[..end], &body[end..]), - None => format!("WITH {definitions} {body}"), - } -} - struct AssertionResolutionRequest<'a> { assertion_sql: &'a str, fixtures: &'a TestFixtures, @@ -463,9 +481,9 @@ impl GeneratedCteState { } fn mock_target(&mut self, request: MockTargetRequest<'_>) -> Result, String> { - let Some(mock_body) = request.mocks.get(request.referenced_name) else { + if !request.mocks.contains_key(request.referenced_name) { return Ok(None); - }; + } self.reachable.insert(request.referenced_name.to_string()); let generated_name = format!("{}{}", request.prefix, request.referenced_name); if self.names.contains(&generated_name) @@ -480,14 +498,20 @@ impl GeneratedCteState { request.file_label, request.referenced_name ))); } - let sql = if request.fixtures.helpers.is_empty() { - mock_body.clone() - } else { - format!( - "{} {mock_body}", - helper_with_clause(&request.fixtures.helpers) - ) - }; + for dependency in request.fixtures.scope.mock_dependencies(&generated_name) { + self.reachable.insert(dependency.mock_name.clone()); + self.insert( + &dependency.generated_name, + &dependency.sql, + request.file_label, + )?; + } + let sql = request + .fixtures + .scope + .mock_sql(&generated_name) + .ok_or_else(|| planner_error("SQL-test mock scope is incomplete"))? + .to_string(); self.insert(&generated_name, &sql, request.file_label)?; Ok(Some(generated_name)) } @@ -680,7 +704,6 @@ fn plan_direct_test( plan: DirectTestPlan, context: &ProjectContext, ) -> Result { - let helper_with = helper_with_clause(&plan.helpers); let mut actual_sql = plan.actual_cte.sql_body; if plan.mode == UDF_TEST_MODE { actual_sql = @@ -693,8 +716,9 @@ fn plan_direct_test( let request = RenderRequest { chain: vec![ChainStep { model_name: model_name.clone(), - resolved_sql: wrap_direct_sql(&actual_sql, &helper_with), - expected_cte_sql: Some(wrap_direct_sql(&plan.expected_cte.sql_body, &helper_with)), + resolved_sql: with_helper_ctes(&actual_sql, &plan.helpers), + expected_cte_sql: Some(with_helper_ctes(&plan.expected_cte.sql_body, &plan.helpers)), + expected_lifted_ctes: Vec::new(), lifted_ctes: Vec::new(), comparison_body_sql: None, expected_columns: None, @@ -716,7 +740,9 @@ fn plan_model_test( plan: ModelTestPlan, context: &ProjectContext, ) -> Result { - let fixtures = classify_fixtures(plan.authored_ctes, plan.expected_ctes, plan.assertion_ctes); + let mut fixtures = + classify_fixtures(plan.authored_ctes, plan.expected_ctes, plan.assertion_ctes); + fixtures.scope = ScopeGraph::new(&fixtures, &context.patterns); let expected_names = chain_root_names(plan.expected_model_names, &fixtures, &context.patterns); let ordered_names = topo_sort_model_chain(TopoSortRequest { expected_names: &expected_names, @@ -797,6 +823,14 @@ fn plan_model_test( reported: &mut reported_missing_mocks, })); let expected_cte_sql = fixtures.expected.get(model_name).cloned(); + let expected_lifted_ctes = match expected_cte_sql.as_deref() { + Some(sql) => { + let scope = helper_scope_ctes(sql, &fixtures, &context.patterns, &plan.file_label)?; + reachable_mocks.extend(scope.reached_mocks); + scope.ctes + } + None => Vec::new(), + }; chain.push(ChainStep { model_name: model_name.clone(), resolved_sql, @@ -804,6 +838,7 @@ fn plan_model_test( .as_deref() .and_then(|sql| expected_columns(sql, &context.render_dialect)), expected_cte_sql, + expected_lifted_ctes, lifted_ctes, comparison_body_sql, }); @@ -834,7 +869,7 @@ fn plan_model_test( } else { None }; - let (resolved_sql, lifted_ctes, comparison_body_sql) = match analyzed { + let (mut resolved_sql, mut lifted_ctes, comparison_body_sql) = match analyzed { Some(value) if !has_unresolved_test_reference(&value.resolved_sql, &context.patterns) => { @@ -876,6 +911,20 @@ fn plan_model_test( ) } }; + let scope = helper_scope_ctes( + &fixture_resolved_sql, + &fixtures, + &context.patterns, + &plan.file_label, + )?; + if !scope.ctes.is_empty() { + reachable_mocks.extend(scope.reached_mocks); + lifted_ctes = merged_scoped_ctes(lifted_ctes, scope.ctes, &plan.file_label)?; + resolved_sql = with_leading_ctes( + &lifted_ctes, + comparison_body_sql.as_deref().unwrap_or(&resolved_sql), + ); + } assertions.push(AssertionStep { name: assertion_name.clone(), resolved_sql, @@ -919,6 +968,7 @@ fn omit_unrendered_step_sql(chain: Vec, assertions: &[AssertionStep]) model_name: step.model_name, resolved_sql: String::new(), expected_cte_sql: step.expected_cte_sql, + expected_lifted_ctes: step.expected_lifted_ctes, lifted_ctes: Vec::new(), comparison_body_sql: None, expected_columns: step.expected_columns, @@ -992,8 +1042,7 @@ fn resolve_assertion_textual_sql( } referenced.push(name); } - let lifted_ctes: Vec<(String, String)> = request.chain.closure_ctes(&referenced, true); - let (body_sql, reached) = resolve_textual_sql(TextualResolutionRequest { + let resolution = resolve_textual_sql(TextualResolutionRequest { query_sql: request.assertion_sql, fixtures: request.fixtures, resolved_chain: &request.chain.cte_names, @@ -1001,6 +1050,12 @@ fn resolve_assertion_textual_sql( functions: request.functions, patterns: request.patterns, })?; + let lifted_ctes: Vec<(String, String)> = with_unique_ctes( + resolution.mock_ctes, + request.chain.closure_ctes(&referenced, true), + ); + let body_sql = resolution.sql; + let reached = resolution.reached; let resolved_sql = if lifted_ctes.is_empty() { body_sql.clone() } else { @@ -1032,6 +1087,7 @@ fn classify_fixtures( mock_dbt_refs: BTreeMap::new(), mock_table_functions: BTreeMap::new(), helpers: Vec::new(), + scope: ScopeGraph::default(), expected: BTreeMap::new(), assertions: Vec::new(), }; @@ -1204,7 +1260,7 @@ fn analyze_and_resolve_sql( } } let cte_body_sql = replace_relation_markers(request.query_sql, &replacements, request.patterns); - let resolved_sql = assemble_resolved_sql(&cte_body_sql, &generated_state.generated); + let resolved_sql = with_leading_ctes(&generated_state.generated, &cte_body_sql); Ok(Some(AnalysisResolvedSql { resolved_sql, cte_body_sql, @@ -1259,11 +1315,16 @@ where Ok(cached.get_or_init(initialize).clone()) } -fn resolve_textual_sql( - request: TextualResolutionRequest<'_>, -) -> Result<(String, HashSet), String> { - let helper_with = helper_with_clause(&request.fixtures.helpers); +/// Textually resolved SQL plus the mocks it reached and the mock CTEs its inlined mocks read. +struct TextualResolution { + sql: String, + reached: HashSet, + mock_ctes: Vec<(String, String)>, +} + +fn resolve_textual_sql(request: TextualResolutionRequest<'_>) -> Result { let mut reached: HashSet = HashSet::new(); + let mut inlined: Vec = Vec::new(); let mut chain_references = request.chain_references; let mut result = replace_named_markers( request.query_sql, @@ -1278,10 +1339,12 @@ fn resolve_textual_sql( } return Some(sql.clone()); } - request.fixtures.mock_refs.get(name).map(|sql| { - reached.insert(name.to_string()); - wrap_mock(sql, &helper_with) - }) + request.fixtures.mock_refs.get(name)?; + let generated_name = format!("{REF_PREFIX}{name}"); + let sql = request.fixtures.scope.inlined_mock_sql(&generated_name)?; + reached.insert(name.to_string()); + inlined.push(generated_name); + Some(sql) }, ); result = replace_named_markers( @@ -1289,10 +1352,12 @@ fn resolve_textual_sql( &request.patterns.source, &request.patterns.protected, |name| { - request.fixtures.mock_sources.get(name).map(|sql| { - reached.insert(name.to_string()); - wrap_mock(sql, &helper_with) - }) + request.fixtures.mock_sources.get(name)?; + let generated_name = format!("{SOURCE_PREFIX}{name}"); + let sql = request.fixtures.scope.inlined_mock_sql(&generated_name)?; + reached.insert(name.to_string()); + inlined.push(generated_name); + Some(sql) }, ); result = replace_named_markers( @@ -1300,10 +1365,12 @@ fn resolve_textual_sql( &request.patterns.seed, &request.patterns.protected, |name| { - request.fixtures.mock_seeds.get(name).map(|sql| { - reached.insert(name.to_string()); - wrap_mock(sql, &helper_with) - }) + request.fixtures.mock_seeds.get(name)?; + let generated_name = format!("{SEED_PREFIX}{name}"); + let sql = request.fixtures.scope.inlined_mock_sql(&generated_name)?; + reached.insert(name.to_string()); + inlined.push(generated_name); + Some(sql) }, ); result = replace_dbt_ref_markers( @@ -1311,10 +1378,12 @@ fn resolve_textual_sql( &request.patterns.dbt_reference, &request.patterns.protected, |name| { - request.fixtures.mock_dbt_refs.get(name).map(|sql| { - reached.insert(name.to_string()); - wrap_mock(sql, &helper_with) - }) + request.fixtures.mock_dbt_refs.get(name)?; + let generated_name = format!("{DBT_REF_PREFIX}{name}"); + let sql = request.fixtures.scope.inlined_mock_sql(&generated_name)?; + reached.insert(name.to_string()); + inlined.push(generated_name); + Some(sql) }, ); let (table_resolved, table_reached) = resolve_table_function_fixtures( @@ -1326,7 +1395,22 @@ fn resolve_textual_sql( reached.extend(table_reached); result = resolve_function_calls(&table_resolved, request.functions, false, request.patterns)?; result = resolve_function_calls(&result, request.functions, true, request.patterns)?; - Ok((result, reached)) + let mut mock_ctes: Vec<(String, String)> = Vec::new(); + for generated_name in inlined { + let dependencies = request.fixtures.scope.mock_dependencies(&generated_name); + reached.extend(dependencies.iter().map(|mock| mock.mock_name.clone())); + mock_ctes = with_unique_ctes( + mock_ctes, + dependencies + .into_iter() + .map(|mock| (mock.generated_name.clone(), mock.sql.clone())), + ); + } + Ok(TextualResolution { + sql: result, + reached, + mock_ctes, + }) } fn resolve_table_function_fixtures( @@ -1338,7 +1422,6 @@ fn resolve_table_function_fixtures( if fixtures.is_empty() { return Ok((sql.to_string(), HashSet::new())); } - let helper_with = helper_with_clause(helpers); replace_callable_markers( sql, &patterns.table_function, @@ -1346,7 +1429,7 @@ fn resolve_table_function_fixtures( |name, _call_suffix| { fixtures .get(name) - .map(|body| (wrap_mock(body, &helper_with), true)) + .map(|body| (format!("({})", with_helper_ctes(body, helpers)), true)) }, ) } @@ -1381,108 +1464,6 @@ fn resolve_function_calls( Ok(result) } -fn replace_callable_markers( - sql: &str, - pattern: &Regex, - protected_pattern: &Regex, - mut replacement: F, -) -> Result<(String, HashSet), String> -where - F: FnMut(&str, &str) -> Option<(String, bool)>, -{ - let protected = protected_ranges(protected_pattern, sql); - let mut output = String::with_capacity(sql.len()); - let mut reached: HashSet = HashSet::new(); - let mut cursor = 0; - for captures in pattern.captures_iter(sql) { - let Some(full) = captures.get(0) else { - continue; - }; - if full.start() < cursor || in_protected_range(full.start(), &protected) { - continue; - } - let suffix_start = skip_whitespace(sql, full.end()); - if !sql[suffix_start..].starts_with('(') { - continue; - } - let suffix_end = matching_paren_end(sql, suffix_start)?; - let Some(name) = captures.get(1).map(|value| value.as_str()) else { - continue; - }; - let call_suffix = &sql[suffix_start..suffix_end]; - let Some((value, records_reach)) = replacement(name, call_suffix) else { - continue; - }; - output.push_str(&sql[cursor..full.start()]); - output.push_str(&value); - cursor = suffix_end; - if records_reach { - reached.insert(name.to_string()); - } - } - output.push_str(&sql[cursor..]); - Ok((output, reached)) -} - -fn matching_paren_end(sql: &str, open: usize) -> Result { - let bytes = sql.as_bytes(); - let mut depth = 0usize; - let mut index = open; - while index < bytes.len() { - match bytes[index] { - b'\'' | b'"' | b'`' => index = skip_quoted(sql, index)?, - b'-' if bytes.get(index + 1) == Some(&b'-') => { - index = sql[index..] - .find('\n') - .map_or(bytes.len(), |offset| index + offset + 1) - } - b'/' if bytes.get(index + 1) == Some(&b'*') => { - let Some(offset) = sql[index + 2..].find("*/") else { - return Err(compile_error( - "SQL function call contains an unclosed block comment", - )); - }; - index += offset + 4; - } - b'(' => { - depth += 1; - index += 1; - } - b')' => { - depth -= 1; - index += 1; - if depth == 0 { - return Ok(index); - } - } - _ => index += 1, - } - } - Err(compile_error( - "SQL function call contains an unclosed parenthesis", - )) -} - -fn skip_quoted(sql: &str, start: usize) -> Result { - let quote = sql.as_bytes()[start]; - let bytes = sql.as_bytes(); - let mut index = start + 1; - while index < bytes.len() { - if bytes[index] != quote { - index += 1; - continue; - } - if bytes.get(index + 1) == Some("e) { - index += 2; - continue; - } - return Ok(index + 1); - } - Err(compile_error( - "SQL function call contains an unclosed quoted string", - )) -} - fn relation_marker_calls(value: &Value) -> Vec<(String, String)> { fn marker(expression: &Value) -> Option<(String, String)> { let object = expression.as_object()?; @@ -1589,172 +1570,6 @@ fn replace_relation_markers( ) } -fn replace_named_markers( - sql: &str, - pattern: &Regex, - protected_pattern: &Regex, - mut replacement: F, -) -> String -where - F: FnMut(&str) -> Option, -{ - let protected = protected_ranges(protected_pattern, sql); - let mut output = String::with_capacity(sql.len()); - let mut cursor = 0; - for captures in pattern.captures_iter(sql) { - let Some(full) = captures.get(0) else { - continue; - }; - if in_protected_range(full.start(), &protected) { - continue; - } - let Some(name) = captures.get(1).map(|value| value.as_str()) else { - continue; - }; - let Some(value) = replacement(name) else { - continue; - }; - output.push_str(&sql[cursor..full.start()]); - output.push_str(&value); - cursor = full.end(); - } - output.push_str(&sql[cursor..]); - output -} - -fn replace_dbt_ref_markers( - sql: &str, - pattern: &Regex, - protected_pattern: &Regex, - mut replacement: F, -) -> String -where - F: FnMut(&str) -> Option, -{ - let protected = protected_ranges(protected_pattern, sql); - let mut output = String::with_capacity(sql.len()); - let mut cursor = 0; - for captures in pattern.captures_iter(sql) { - let Some(full) = captures.get(0) else { - continue; - }; - if in_protected_range(full.start(), &protected) { - continue; - } - let Some(first) = captures.get(1).map(|value| value.as_str()) else { - continue; - }; - let name = captures.get(2).map_or_else( - || first.to_string(), - |second| format!("{first}__{}", second.as_str()), - ); - let Some(value) = replacement(&name) else { - continue; - }; - output.push_str(&sql[cursor..full.start()]); - output.push_str(&value); - cursor = full.end(); - } - output.push_str(&sql[cursor..]); - output -} - -fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str) -> Vec { - let protected = protected_ranges(protected_pattern, sql); - let mut names: Vec = Vec::new(); - for captures in pattern.captures_iter(sql) { - let Some(full) = captures.get(0) else { - continue; - }; - if in_protected_range(full.start(), &protected) { - continue; - } - if let Some(name) = captures.get(1) { - names.push(name.as_str().to_string()); - } - } - names -} - -fn protected_ranges(pattern: &Regex, sql: &str) -> Vec<(usize, usize)> { - pattern - .find_iter(sql) - .map(|value| (value.start(), value.end())) - .collect() -} - -fn in_protected_range(index: usize, ranges: &[(usize, usize)]) -> bool { - ranges - .iter() - .any(|(start, end)| index >= *start && index < *end) -} - -fn assemble_resolved_sql(cte_body_sql: &str, generated: &[(String, String)]) -> String { - if generated.is_empty() { - return cte_body_sql.to_string(); - } - let generated_sql = generated - .iter() - .map(|(name, sql)| cte_definition_sql(name, sql)) - .collect::>() - .join(", "); - if let Some(with_end) = leading_with_prefix_end(cte_body_sql) { - format!( - "{}{generated_sql}, {}", - &cte_body_sql[..with_end], - &cte_body_sql[with_end..] - ) - } else { - format!("WITH {generated_sql} {cte_body_sql}") - } -} - -pub(crate) fn leading_with_prefix_end(sql: &str) -> Option { - let mut index = skip_leading_ignorable(sql, 0); - index = keyword_end(sql, index, "WITH")?; - index = skip_leading_ignorable(sql, index); - if let Some(recursive_end) = keyword_end(sql, index, "RECURSIVE") { - index = skip_leading_ignorable(sql, recursive_end); - } - Some(index) -} - -fn skip_leading_ignorable(sql: &str, mut index: usize) -> usize { - while index < sql.len() { - let byte = sql.as_bytes()[index]; - if byte.is_ascii_whitespace() { - index += 1; - } else if sql[index..].starts_with("--") { - index = sql[index..] - .find('\n') - .map_or(sql.len(), |offset| index + offset + 1); - } else if sql[index..].starts_with("/*") { - let Some(offset) = sql[index + 2..].find("*/") else { - return index; - }; - index += offset + 4; - } else { - break; - } - } - index -} - -fn keyword_end(sql: &str, start: usize, keyword: &str) -> Option { - let end = start + keyword.len(); - if !sql.get(start..end)?.eq_ignore_ascii_case(keyword) { - return None; - } - if sql - .as_bytes() - .get(end) - .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') - { - return None; - } - Some(end) -} - struct UnresolvedReferenceRequest<'a> { sql: &'a str, test_name: &'a str, @@ -1894,38 +1709,13 @@ fn helper_with_clause(helpers: &[CteInput]) -> String { ) } -fn cte_definition_sql(name: &str, sql: &str) -> String { - let body = sql.trim_end(); - let final_line = body.rsplit_once('\n').map_or(body, |(_, line)| line); - let terminator = if final_line.contains("--") { "\n" } else { "" }; - format!("{name} AS ({body}{terminator})") -} - -fn wrap_mock(sql: &str, helper_with: &str) -> String { - if helper_with.is_empty() { - format!("({sql})") - } else { - format!("({helper_with} {sql})") - } -} - -fn wrap_direct_sql(sql: &str, helper_with: &str) -> String { - if helper_with.is_empty() { +/// Prefix every helper CTE to a query; mock and direct-test bodies all see helpers this way. +pub(crate) fn with_helper_ctes(sql: &str, helpers: &[CteInput]) -> String { + if helpers.is_empty() { sql.to_string() } else { - format!("{helper_with} {sql}") - } -} - -fn skip_whitespace(sql: &str, mut index: usize) -> usize { - while sql - .as_bytes() - .get(index) - .is_some_and(u8::is_ascii_whitespace) - { - index += 1; + format!("{} {sql}", helper_with_clause(helpers)) } - index } fn dedupe(mut values: Vec) -> Vec { @@ -1951,7 +1741,7 @@ fn default_set_difference() -> String { "EXCEPT".to_string() } -fn compile_error(message: &str) -> String { +pub(crate) fn compile_error(message: &str) -> String { format!("compile_input:{message}") } diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs index 8c48343d3e..d6d787066b 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs @@ -4,7 +4,9 @@ use std::collections::HashMap; use polyglot_sql::{Dialect, DialectType, Expression}; -use crate::compiler::_helpers::sql_tests::planning::leading_with_prefix_end; +use crate::compiler::_helpers::sql_tests::cte_sql::{ + cte_definition_sql, leading_with_prefix_end, with_leading_ctes, +}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::{Deserialize, Serialize}; @@ -68,6 +70,8 @@ pub(crate) struct ChainStep { #[serde(default)] pub(crate) expected_cte_sql: Option, #[serde(default)] + pub(crate) expected_lifted_ctes: Vec<(String, String)>, + #[serde(default)] pub(crate) lifted_ctes: Vec<(String, String)>, #[serde(default)] pub(crate) comparison_body_sql: Option, @@ -141,6 +145,17 @@ impl<'a> RenderCteState<'a> { ) } + /// Place an expected step's helper CTEs at top level and return its comparison body. + fn expected_step_sql(&mut self, step: &ChainStep, expected_sql: &str, enabled: bool) -> String { + if step.expected_lifted_ctes.is_empty() { + return self.lift(expected_sql, enabled); + } + if !self.merge(&step.expected_lifted_ctes) { + return with_leading_ctes(&step.expected_lifted_ctes, expected_sql); + } + self.lift(expected_sql, enabled) + } + fn merge(&mut self, ctes: &[(String, String)]) -> bool { if ctes.iter().any(|(name, body)| { existing_cte(&self.lifted, name).is_some_and(|(_, existing)| existing != body) @@ -246,7 +261,8 @@ fn render_difference_sample_sql(request: &DifferenceSampleRequest, dialect: &Dia }; let mut cte_state = RenderCteState::new(dialect); let actual_sql = cte_state.actual_step_sql(step, request.sql_analysis_enabled); - let expected_sql = cte_state.lift(expected_input, request.sql_analysis_enabled); + let expected_sql = + cte_state.expected_step_sql(step, expected_input, request.sql_analysis_enabled); let mut cte_parts: Vec = cte_state .lifted .iter() @@ -309,7 +325,8 @@ pub(crate) fn render_comparison_sql(request: &RenderRequest, dialect: &Dialect) let Some(expected_input) = step.expected_cte_sql.as_deref() else { continue; }; - let expected_sql = cte_state.lift(expected_input, request.sql_analysis_enabled); + let expected_sql = + cte_state.expected_step_sql(step, expected_input, request.sql_analysis_enabled); comparison_ctes.push(cte_definition_sql(&expected_cte, &expected_sql)); let projection = compared_projection(step); select_parts.push(format!( @@ -485,13 +502,6 @@ fn sanitize_cte_suffix(model_name: &str) -> String { } } -fn cte_definition_sql(name: &str, sql: &str) -> String { - let body = sql.trim_end(); - let final_line = body.rsplit_once('\n').map_or(body, |(_, line)| line); - let terminator = if final_line.contains("--") { "\n" } else { "" }; - format!("{name} AS ({body}{terminator})") -} - fn assertions_use_actual(assertions: &[AssertionStep], actual_cte: &str) -> bool { let needle = actual_cte.to_ascii_lowercase(); assertions.iter().any(|assertion| { diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/sql_scan.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/sql_scan.rs new file mode 100644 index 0000000000..c1ba225599 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/sql_scan.rs @@ -0,0 +1,83 @@ +//! Quote- and comment-aware SQL scanning shared by SQL-test extraction and planning. + +/// The SQL construct a scan reached the end of input inside. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Unclosed { + BlockComment, + Quote, + Parenthesis, +} + +/// Return the first index at or after `index` that is not whitespace. +pub(crate) fn skip_whitespace(sql: &str, mut index: usize) -> usize { + while let Some(character) = sql.get(index..).and_then(|rest| rest.chars().next()) { + if !character.is_whitespace() { + break; + } + index += character.len_utf8(); + } + index +} + +/// Return the end of a line or block comment starting at `index`, if one starts there. +pub(crate) fn comment_end(sql: &str, index: usize) -> Result, Unclosed> { + let bytes = sql.as_bytes(); + match (bytes.get(index), bytes.get(index + 1)) { + (Some(b'-'), Some(b'-')) => Ok(Some( + sql[index..] + .find('\n') + .map_or(sql.len(), |offset| index + offset + 1), + )), + (Some(b'/'), Some(b'*')) => sql[index + 2..] + .find("*/") + .map(|offset| Some(index + 2 + offset + 2)) + .ok_or(Unclosed::BlockComment), + _ => Ok(None), + } +} + +/// Return the index after the quoted text starting at `start`, treating doubled quotes as escapes. +pub(crate) fn quoted_end(sql: &str, start: usize) -> Result { + let bytes = sql.as_bytes(); + let quote = bytes[start]; + let mut index = start + 1; + while index < bytes.len() { + if bytes[index] != quote { + index += 1; + } else if bytes.get(index + 1) == Some("e) { + index += 2; + } else { + return Ok(index + 1); + } + } + Err(Unclosed::Quote) +} + +/// Return the index of the parenthesis closing the one at `open`, skipping quotes and comments. +pub(crate) fn matching_paren(sql: &str, open: usize) -> Result { + let bytes = sql.as_bytes(); + let mut depth = 0usize; + let mut index = open; + while index < bytes.len() { + if let Some(end) = comment_end(sql, index)? { + index = end; + continue; + } + match bytes[index] { + b'\'' | b'"' | b'`' => { + index = quoted_end(sql, index)?; + continue; + } + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Ok(index); + } + } + _ => {} + } + index += 1; + } + Err(Unclosed::Parenthesis) +} diff --git a/crates/sqlbuild-rules-native/src/compiler/main/mod.rs b/crates/sqlbuild-rules-native/src/compiler/main/mod.rs index 8e6cc4ba3a..761d058af1 100644 --- a/crates/sqlbuild-rules-native/src/compiler/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/compiler/main/mod.rs @@ -5,5 +5,6 @@ pub(crate) mod sql_references; pub(crate) mod sql_test_chain_resolution; pub(crate) mod sql_test_difference_sampling; pub(crate) mod sql_test_extraction; +pub(crate) mod sql_test_fixture_facts; pub(crate) mod sql_test_planning; pub(crate) mod sql_test_rendering; diff --git a/crates/sqlbuild-rules-native/src/compiler/main/sql_test_fixture_facts.rs b/crates/sqlbuild-rules-native/src/compiler/main/sql_test_fixture_facts.rs new file mode 100644 index 0000000000..95d6f216c7 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/main/sql_test_fixture_facts.rs @@ -0,0 +1,12 @@ +use crate::compiler::_helpers::sql_tests::{extraction, planning}; +use crate::compiler::models::SqlTestFixtureFacts; + +pub(crate) fn fixture_facts(name: &str, sql: &str) -> SqlTestFixtureFacts { + SqlTestFixtureFacts { + mock: planning::MOCK_CTE_PREFIXES.iter().any(|prefix| { + name.strip_prefix(prefix) + .is_some_and(|target| !target.is_empty()) + }), + empty_fixture_marker: matches!(extraction::empty_fixture_marker_matches(sql), Ok(true)), + } +} diff --git a/crates/sqlbuild-rules-native/src/compiler/models.rs b/crates/sqlbuild-rules-native/src/compiler/models.rs index d0603350cb..ef144dab88 100644 --- a/crates/sqlbuild-rules-native/src/compiler/models.rs +++ b/crates/sqlbuild-rules-native/src/compiler/models.rs @@ -13,3 +13,9 @@ pub(crate) enum AuthoredValue { NamedSqlHook(String, Vec<(String, AuthoredValue)>), PythonHook(String, Vec<(String, AuthoredValue)>), } + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SqlTestFixtureFacts { + pub(crate) mock: bool, + pub(crate) empty_fixture_marker: bool, +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs index 278b84de7f..ab82c272bd 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs @@ -1011,3 +1011,188 @@ pub(crate) fn snowflake_plan_keeps_quoted_expected_columns() -> bool { ), "{sql}"); true } + +fn plan_helper_scope_request( + sql_analysis_enabled: bool, + helper_name: &str, +) -> Result { + crate::compiler::main::sql_test_planning::plan_and_render_json( + &json!({ + "models": [ + { + "name": "orders", + "querySql": "SELECT order_id, amount FROM __ref(\"stg_orders\")", + "modelDependencies": ["stg_orders"] + } + ], + "tests": [{ + "name": "orders_case", + "fileLabel": "tests/orders.sql", + "payload": { + "kind": "model", + "authoredCtes": [ + {"name": "__ref__stg_orders", "sqlBody": "SELECT 1 AS order_id, 10 AS amount"}, + {"name": "expected_rows", "sqlBody": "SELECT 1 AS order_id, 10 AS amount"}, + {"name": helper_name, "sqlBody": "SELECT s.order_id FROM __ref__stg_orders AS s JOIN expected_rows AS e ON e.order_id = s.order_id"}, + {"name": "unused_rows", "sqlBody": "SELECT 3 AS order_id"} + ], + "expectedCtes": [{ + "name": "__expected__orders", + "sqlBody": "SELECT order_id, amount FROM expected_rows" + }], + "expectedModelNames": ["orders"], + "assertionCtes": [{ + "name": "__assert__ids_match", + "sqlBody": format!("SELECT order_id FROM __ref(\"orders\") EXCEPT SELECT order_id FROM {helper_name}") + }] + } + }], + "sqlAnalysisEnabled": sql_analysis_enabled, + "sqlAnalysisDialect": "duckdb", + "setDifferenceOperator": "EXCEPT" + }) + .to_string(), + ) +} + +fn top_level_cte_position(sql: &str, name: &str) -> usize { + let needle = format!("{name} AS ("); + let positions: Vec = sql + .match_indices(&needle) + .map(|(index, _)| index) + .filter(|index| *index == 5 || sql[..*index].ends_with(",\n")) + .collect(); + assert_eq!( + positions.len(), + 1, + "{name} must be defined once at top level: {sql}" + ); + positions[0] +} + +pub(crate) fn helper_ctes_are_in_scope_for_assertions_and_expected_rows() -> bool { + for sql_analysis_enabled in [true, false] { + let response: Value = serde_json::from_str( + &plan_helper_scope_request(sql_analysis_enabled, "matched_ids") + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + let sql = response["artifacts"][0]["sql"] + .as_str() + .expect("test assumption must hold"); + let mock = top_level_cte_position(sql, "__ref__stg_orders"); + let expected_rows = top_level_cte_position(sql, "expected_rows"); + let matched_ids = top_level_cte_position(sql, "matched_ids"); + let expected = top_level_cte_position(sql, "__expected__orders"); + let assertion = top_level_cte_position(sql, "__assert__ids_match"); + assert!(mock < matched_ids && expected_rows < matched_ids, "{sql}"); + assert!(expected_rows < expected && matched_ids < assertion, "{sql}"); + assert!( + !sql.contains(",\nunused_rows AS ("), + "unreferenced helpers stay out of top-level scope: {sql}" + ); + assert_eq!(response["artifacts"][0]["warnings"], json!([]), "{sql}"); + } + true +} + +pub(crate) fn scoped_helper_named_like_generated_cte_is_rejected() -> bool { + let error = + plan_helper_scope_request(true, "__actual__orders").expect_err("test assumption must hold"); + assert_eq!( + error, + "compile_input:SQL test 'tests/orders.sql' defines CTE '__actual__orders', which conflicts with the generated CTE" + ); + true +} + +pub(crate) fn difference_sample_lifts_expected_helper_ctes() -> bool { + let response: Value = serde_json::from_str( + &crate::compiler::main::sql_test_difference_sampling::render_difference_sample_json( + &json!({ + "step": { + "modelName": "orders", + "resolvedSql": "SELECT 1 AS order_id", + "expectedCteSql": "SELECT order_id FROM expected_rows", + "expectedLiftedCtes": [["expected_rows", "SELECT 2 AS order_id"]], + "expectedColumns": ["order_id"] + }, + "sqlAnalysisEnabled": false, + "setDifferenceOperator": "EXCEPT", + "sqlAnalysisDialect": "duckdb", + "direction": "missing", + "sampleLimit": 3 + }) + .to_string(), + ) + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + assert_eq!( + response["sql"], + json!( + "WITH expected_rows AS (SELECT 2 AS order_id),\n\ + __actual AS (SELECT 1 AS order_id),\n\ + __expected AS (SELECT order_id FROM expected_rows)\n\ + SELECT * FROM (SELECT order_id FROM __expected EXCEPT SELECT order_id FROM __actual) \ + AS __sqlbuild_difference LIMIT 3" + ) + ); + true +} + +pub(crate) fn mock_read_through_helper_brings_its_mock_dependencies_into_scope() -> bool { + for sql_analysis_enabled in [true, false] { + let response: Value = serde_json::from_str( + &crate::compiler::main::sql_test_planning::plan_and_render_json( + &json!({ + "models": [ + {"name": "raw_orders", "querySql": "SELECT 1 AS order_id", "modelDependencies": []}, + {"name": "stg_orders", "querySql": "SELECT order_id FROM __ref(\"raw_orders\")", "modelDependencies": ["raw_orders"]}, + {"name": "orders", "querySql": "SELECT order_id FROM __ref(\"stg_orders\")", "modelDependencies": ["stg_orders"]} + ], + "tests": [{ + "name": "orders_case", + "fileLabel": "tests/orders.sql", + "payload": { + "kind": "model", + "authoredCtes": [ + {"name": "__ref__raw_orders", "sqlBody": "SELECT 1 AS order_id"}, + {"name": "base_rows", "sqlBody": "SELECT order_id FROM __ref__raw_orders"}, + {"name": "__ref__stg_orders", "sqlBody": "SELECT order_id FROM base_rows"}, + {"name": "expected_rows", "sqlBody": "SELECT order_id FROM __ref__stg_orders"} + ], + "expectedCtes": [{ + "name": "__expected__orders", + "sqlBody": "SELECT order_id FROM expected_rows" + }], + "expectedModelNames": ["orders"], + "assertionCtes": [{ + "name": "__assert__rows_match", + "sqlBody": "SELECT order_id FROM expected_rows EXCEPT SELECT order_id FROM __ref(\"orders\")" + }] + } + }], + "sqlAnalysisEnabled": sql_analysis_enabled, + "sqlAnalysisDialect": "duckdb", + "setDifferenceOperator": "EXCEPT" + }) + .to_string(), + ) + .expect("test assumption must hold"), + ) + .expect("test assumption must hold"); + let sql = response["artifacts"][0]["sql"] + .as_str() + .expect("test assumption must hold"); + let raw_mock = top_level_cte_position(sql, "__ref__raw_orders"); + let stg_mock = top_level_cte_position(sql, "__ref__stg_orders"); + let expected_rows = top_level_cte_position(sql, "expected_rows"); + let actual = top_level_cte_position(sql, "__actual__orders"); + let assertion = top_level_cte_position(sql, "__assert__rows_match"); + assert!(raw_mock < stg_mock && stg_mock < expected_rows, "{sql}"); + assert!(raw_mock < actual && expected_rows < assertion, "{sql}"); + assert_eq!(response["artifacts"][0]["warnings"], json!([]), "{sql}"); + } + true +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs index 27303541d8..b343728391 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_planning.rs @@ -1,7 +1,10 @@ use crate::compiler::tests::helpers::{ chain_resolution_orders_unmocked_models, concurrent_requests_initialize_shared_template_once, - deep_shared_graph_reports_missing_mock_once, long_chain_plan_output_stays_linear, + deep_shared_graph_reports_missing_mock_once, + helper_ctes_are_in_scope_for_assertions_and_expected_rows, long_chain_plan_output_stays_linear, + mock_read_through_helper_brings_its_mock_dependencies_into_scope, model_test_batch_returns_ordered_artifact, plan_without_rendering_returns_executable_steps, + scoped_helper_named_like_generated_cte_is_rejected, shared_textual_chain_renders_each_model_once, unicode_cte_after_leading_with_preserves_identifier, unresolved_reference_fast_rejection_preserves_warning, upstream_fallback_resolves, @@ -73,6 +76,21 @@ fn given_sql_test_planning_cases_when_exercising_native_planner_then_expected_be run: chain_resolution_orders_unmocked_models, expected_success: true, }, + SqlTestPlanningTestCase { + description: "helper CTEs are in scope for assertions and expected rows", + run: helper_ctes_are_in_scope_for_assertions_and_expected_rows, + expected_success: true, + }, + SqlTestPlanningTestCase { + description: "mock read through a helper brings its mock dependencies into scope", + run: mock_read_through_helper_brings_its_mock_dependencies_into_scope, + expected_success: true, + }, + SqlTestPlanningTestCase { + description: "scoped helper named like a generated CTE is rejected", + run: scoped_helper_named_like_generated_cte_is_rejected, + expected_success: true, + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs index 43e2a531b5..ffa81a4d68 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs @@ -1,5 +1,5 @@ use crate::compiler::tests::helpers::{ - actual_probe_selects_zero_rows_from_step, + actual_probe_selects_zero_rows_from_step, difference_sample_lifts_expected_helper_ctes, difference_sample_lifts_generated_ctes_and_bounds_rows, ordered_comparison_batch_preserves_order, partial_expected_columns_project_both_sides, snowflake_plan_keeps_quoted_expected_columns, @@ -40,6 +40,11 @@ fn given_sql_rendering_cases_when_rendering_native_batches_then_expected_behavio run: snowflake_plan_keeps_quoted_expected_columns, expected_success: true, }, + SqlTestRenderingTestCase { + description: "difference samples lift expected helper CTEs", + run: difference_sample_lifts_expected_helper_ctes, + expected_success: true, + }, ]; for test_case in test_cases { diff --git a/crates/sqlbuild-rules-native/src/constants.rs b/crates/sqlbuild-rules-native/src/constants.rs index e562e1efde..f8aa83bd9d 100644 --- a/crates/sqlbuild-rules-native/src/constants.rs +++ b/crates/sqlbuild-rules-native/src/constants.rs @@ -49,3 +49,7 @@ pub(crate) const TABLE_FUNCTION_TEST_MODE: &str = "table_fn"; pub(crate) const THRESHOLDS_KEY: &str = "thresholds"; pub(crate) const TYPE_KEY: &str = "type"; pub(crate) const UDF_TEST_MODE: &str = "udf"; +pub(crate) const SQL_TEST_ACTUAL_CTE: &str = "__actual"; +pub(crate) const SQL_TEST_EXPECTED_CTE: &str = "__expected"; +pub(crate) const SQL_TEST_ACTUAL_CTE_PREFIX: &str = "__actual__"; +pub(crate) const QUOTED_IDENTIFIER_DELIMITER_BYTES: usize = 2; diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs index 40d43b3f9b..8dff5e1504 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs @@ -9,7 +9,7 @@ use crate::models::{ }; use crate::rules::main::{ assemble_catalogue, evaluate as rules, evaluate_project, fingerprint, - resolve_threshold_overrides, select, + resolve_threshold_overrides, select, sql_test_coverage, }; use crate::rules::models::{ ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, @@ -194,6 +194,14 @@ pub(crate) fn evaluate_json(request_json: &str) -> Result { config::validate(&request.config)?; let all_rules = assemble_catalogue::assemble_catalogue(&request.custom_rules)?; let selected = select::select(&all_rules, &request.config.select, &request.config.ignore)?; + let request = if selected + .iter() + .any(|rule| matches!(rule.code.as_str(), "SQBRTEST202" | "SQBRTEST203")) + { + sql_test_coverage::annotate(request) + } else { + request + }; validate_suppression_codes(&request, &all_rules)?; let ruleset_fingerprint = fingerprint::fingerprint(&selected, &request.config, &request.dialect)?; diff --git a/crates/sqlbuild-rules-native/src/engine/tests.rs b/crates/sqlbuild-rules-native/src/engine/tests.rs index 198d344d29..1907da7600 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests.rs @@ -1,5 +1,7 @@ #[path = "tests/test_domain_layout.rs"] mod domain_layout; +#[path = "tests/test_empty_input_tests.rs"] +mod empty_input_tests; #[path = "tests/test_evaluation.rs"] mod evaluation; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs b/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs index 9ff17d86dc..9e43892a47 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/helpers.rs @@ -285,3 +285,96 @@ pub(crate) fn cached_sql_test_rules_request(project_dir: &TempDir, name: Option< }) .to_string() } + +pub(crate) fn empty_input_test_fact( + name: &str, + mocks: &[(&str, &str)], + expected: &[(&str, &str)], + assertions: &[(&str, &str)], +) -> Value { + let ctes = |items: &[(&str, &str)]| -> Value { + Value::Array( + items + .iter() + .map(|(cte_name, sql)| json!({"name": cte_name, "sql": sql})) + .collect(), + ) + }; + let mut fact = sql_test_fact( + &format!("tests/unit/test_{name}.sql"), + Some(name), + json!(["orders"]), + ); + fact["authored_ctes"] = ctes(mocks); + fact["expected_ctes"] = ctes(expected); + fact["assertion_ctes"] = ctes(assertions); + fact +} + +pub(crate) fn empty_input_rule_evaluation( + project_dir: &TempDir, + select: Value, + tests: Value, + allowed_tests: Value, + cache_enabled: bool, +) -> Result, String> { + let request = json!({ + "version": 1, + "project_dir": project_dir.path(), + "dialect": "duckdb", + "config": { + "select": select, + "thresholds": {"min_tests_per_model": 1}, + "rule_options": {"SQBRTEST203": {"allowed_tests": allowed_tests}}, + "cache": {"enabled": cache_enabled} + }, + "models": [{ + "name": "orders", + "relative_path": "models/orders.sql", + "query_sql": "SELECT order_id, amount * 2 AS doubled_amount FROM raw_orders", + "authored_sql": "SELECT order_id, amount * 2 AS doubled_amount FROM raw_orders", + "targeting_test_count": tests.as_array().map_or(0, Vec::len) + }], + "sql_tests": tests, + "sql_scenarios": [], + "scope_index": scope_index() + }); + let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(result["faults"] + .as_array() + .into_iter() + .flatten() + .map(|fault| { + format!( + "{} {}: {}", + fault["code"].as_str().unwrap_or_default(), + fault["path"].as_str().unwrap_or_default(), + fault["message"].as_str().unwrap_or_default() + ) + }) + .collect()) +} + +pub(crate) fn filler_test() -> Value { + empty_input_test_fact( + "orders__empty_inputs_produce_no_rows", + &[ + ( + "__source__raw_orders", + "SELECT NULL AS order_id, NULL AS amount WHERE FALSE", + ), + ("__ref__customers", "SELECT NULL AS customer_id WHERE 1 = 0"), + ], + &[], + &[( + "__assert__empty_inputs_produce_no_rows", + "SELECT 1 AS unexpected_row FROM __ref(\"orders\")", + )], + ) +} + +pub(crate) fn with_field(mut fact: Value, field: &str, value: Value) -> Value { + fact[field] = value; + fact +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs new file mode 100644 index 0000000000..96ee59552f --- /dev/null +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs @@ -0,0 +1,389 @@ +use crate::engine::tests::{helpers, test_types}; +use serde_json::json; +use tempfile::TempDir; + +const EMPTY_ORDERS: (&str, &str) = ( + "__source__raw_orders", + "SELECT NULL AS order_id, NULL AS amount WHERE FALSE", +); +const ORDER_ROWS: (&str, &str) = ( + "__source__raw_orders", + "SELECT 1 AS order_id, 5 AS amount UNION ALL SELECT 2 AS order_id, 7 AS amount", +); +const NO_ROWS_ASSERTION: (&str, &str) = ( + "__assert__empty_inputs_produce_no_rows", + "SELECT 1 AS unexpected_row FROM __ref(\"orders\")", +); +const FLAGGED_MESSAGE: &str = "SQBRTEST203 tests/unit/test_orders__empty_inputs_produce_no_rows.sql: unit test block 1 (\"orders__empty_inputs_produce_no_rows\") mocks only empty inputs and asserts only that no rows are produced"; + +#[test] +fn given_sql_test_shapes_when_evaluating_empty_input_rule_then_flags_only_filler_tests() +-> Result<(), String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let test_cases = [ + test_types::EmptyInputTestRuleTestCase { + description: "all-empty mocks with a no-rows assertion are flagged", + test: helpers::filler_test(), + allowed_tests: json!([]), + expected_messages: &[FLAGGED_MESSAGE], + }, + test_types::EmptyInputTestRuleTestCase { + description: "all-empty mocks with an empty expected output are flagged", + test: helpers::empty_input_test_fact( + "orders__empty_inputs_produce_no_rows", + &[ + ( + "__source__raw_orders", + "SELECT NULL AS order_id, NULL AS amount LIMIT 0", + ), + ("__ref__customers", "SELECT * FROM __EMPTY_FIXTURE()"), + ], + &[("__expected__orders", "SELECT * FROM __empty_fixture()")], + &[], + ), + allowed_tests: json!([]), + expected_messages: &[FLAGGED_MESSAGE], + }, + test_types::EmptyInputTestRuleTestCase { + description: "input rows with expected output pass", + test: helpers::empty_input_test_fact( + "orders__doubles_amounts", + &[ORDER_ROWS], + &[( + "__expected__orders", + "SELECT 1 AS order_id, 10 AS doubled_amount", + )], + &[], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "input rows with an existence assertion pass", + test: helpers::empty_input_test_fact( + "orders__keeps_every_order", + &[ORDER_ROWS], + &[], + &[NO_ROWS_ASSERTION], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "one empty mock alongside a mock with rows passes", + test: helpers::empty_input_test_fact( + "orders__ignores_missing_customers", + &[ + ORDER_ROWS, + ("__ref__customers", "SELECT NULL AS customer_id WHERE 1 = 0"), + ], + &[], + &[NO_ROWS_ASSERTION], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "a union mock whose last branch is filtered is not empty", + test: helpers::empty_input_test_fact( + "orders__keeps_first_branch", + &[( + "__source__raw_orders", + "SELECT 1 AS order_id, 5 AS amount UNION ALL SELECT 2 AS order_id, 7 AS amount WHERE FALSE", + )], + &[], + &[NO_ROWS_ASSERTION], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "all-empty mocks with a summary-row EXCEPT assertion pass", + test: helpers::empty_input_test_fact( + "orders__reports_zero_summary", + &[EMPTY_ORDERS], + &[], + &[( + "__assert__zero_summary_row", + "SELECT order_count, total_amount FROM __ref(\"orders\") EXCEPT SELECT 0 AS order_count, 0 AS total_amount", + )], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "all-empty mocks with a filtered assertion pass", + test: helpers::empty_input_test_fact( + "orders__never_negative", + &[EMPTY_ORDERS], + &[], + &[( + "__assert__no_negative_amounts", + "SELECT 1 FROM __ref(\"orders\") WHERE doubled_amount < 0", + )], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "all-empty mocks with an aggregate-projection assertion pass", + test: helpers::empty_input_test_fact( + "orders__reports_one_zero_summary", + &[EMPTY_ORDERS], + &[], + &[( + "__assert__one_zero_summary_row", + "SELECT UNNEST(CASE WHEN COUNT(*) = 1 AND MIN(order_count) = 0 THEN [] ELSE [1] END) AS failure FROM __ref(\"orders\")", + )], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "all-empty mocks with an offset assertion pass", + test: helpers::empty_input_test_fact( + "orders__reports_at_most_one_summary", + &[EMPTY_ORDERS], + &[], + &[( + "__assert__at_most_one_summary_row", + "SELECT 1 AS unexpected_extra_summary FROM __ref(\"orders\") OFFSET 1", + )], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "a filtered mock ordered by an aggregate is not empty", + test: helpers::empty_input_test_fact( + "orders__drops_first_order", + &[( + "__source__raw_orders", + "SELECT 1 AS order_id, 5 AS amount WHERE FALSE ORDER BY COUNT(*)", + )], + &[], + &[NO_ROWS_ASSERTION], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "an allowlisted filler test passes", + test: helpers::filler_test(), + allowed_tests: json!(["orders__empty_inputs_produce_no_rows"]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "a stale allowlist entry is reported", + test: helpers::empty_input_test_fact( + "orders__doubles_amounts", + &[ORDER_ROWS], + &[( + "__expected__orders", + "SELECT 1 AS order_id, 10 AS doubled_amount", + )], + &[], + ), + allowed_tests: json!(["orders__retired_filler"]), + expected_messages: &[ + "SQBRTEST203 sqlbuild_project.toml: stale allowed_tests entry \"orders__retired_filler\" names no existing SQL test", + ], + }, + test_types::EmptyInputTestRuleTestCase { + description: "an unparseable mock is not flagged", + test: helpers::empty_input_test_fact( + "orders__empty_inputs_produce_no_rows", + &[( + "__source__raw_orders", + "SELECT NULL AS order_id WHERE FALSE LIMIT", + )], + &[], + &[NO_ROWS_ASSERTION], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "a macro-mode test is not flagged", + test: helpers::with_field(helpers::filler_test(), "mode", json!("macro")), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "a test with macro mocks is not flagged", + test: helpers::with_field(helpers::filler_test(), "has_macro_mocks", json!(true)), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "a test with model query overrides is not flagged", + test: helpers::with_field( + helpers::filler_test(), + "has_model_query_overrides", + json!(true), + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "a global aggregate under a false predicate is not an empty mock", + test: helpers::empty_input_test_fact( + "orders__empty_inputs_produce_no_rows", + &[( + "__source__raw_orders", + "SELECT COUNT(*) AS order_id, 0 AS amount WHERE FALSE", + )], + &[], + &[NO_ROWS_ASSERTION], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "an assertion on an untested model is not a bare existence check", + test: helpers::empty_input_test_fact( + "orders__empty_inputs_produce_no_rows", + &[EMPTY_ORDERS], + &[], + &[( + "__assert__no_customers", + "SELECT 1 FROM __ref(\"customers\")", + )], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputTestRuleTestCase { + description: "an assertion with a subquery is not a bare existence check", + test: helpers::empty_input_test_fact( + "orders__empty_inputs_produce_no_rows", + &[EMPTY_ORDERS], + &[], + &[( + "__assert__no_rows", + "SELECT (SELECT 1) AS unexpected_row FROM __ref(\"orders\")", + )], + ), + allowed_tests: json!([]), + expected_messages: &[], + }, + ]; + + for test_case in test_cases { + let messages = helpers::empty_input_rule_evaluation( + &project_dir, + json!(["SQBRTEST203"]), + json!([test_case.test]), + test_case.allowed_tests, + false, + )?; + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_empty_input_only_tests_when_evaluating_minimum_tests_then_they_do_not_count() +-> Result<(), String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let real_test = helpers::empty_input_test_fact( + "orders__doubles_amounts", + &[ORDER_ROWS], + &[( + "__expected__orders", + "SELECT 1 AS order_id, 10 AS doubled_amount", + )], + &[], + ); + let test_cases = [ + test_types::EmptyInputMinimumTestsTestCase { + description: "filler is not counted when only SQBRTEST202 is selected", + select: json!(["SQBRTEST202"]), + tests: json!([helpers::filler_test()]), + allowed_tests: json!([]), + expected_messages: &[ + "SQBRTEST202 models/orders.sql: model \"orders\" has 0 tests; 1 required (1 empty-input-only test not counted; see SQBRTEST203)", + ], + }, + test_types::EmptyInputMinimumTestsTestCase { + description: "a real test satisfies the minimum alongside filler", + select: json!(["SQBRTEST202"]), + tests: json!([helpers::filler_test(), real_test]), + allowed_tests: json!([]), + expected_messages: &[], + }, + test_types::EmptyInputMinimumTestsTestCase { + description: "an allowlisted filler test counts toward the minimum", + select: json!(["SQBRTEST202", "SQBRTEST203"]), + tests: json!([helpers::filler_test()]), + allowed_tests: json!(["orders__empty_inputs_produce_no_rows"]), + expected_messages: &[], + }, + ]; + + for test_case in test_cases { + let messages = helpers::empty_input_rule_evaluation( + &project_dir, + test_case.select, + test_case.tests, + test_case.allowed_tests, + false, + )?; + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_cached_minimum_tests_when_filler_becomes_real_test_then_model_cache_is_invalidated() +-> Result<(), String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let real_test = helpers::empty_input_test_fact( + "orders__empty_inputs_produce_no_rows", + &[ORDER_ROWS], + &[], + &[NO_ROWS_ASSERTION], + ); + let test_cases = [ + test_types::EmptyInputMinimumTestsTestCase { + description: "filler fails the cached minimum", + select: json!(["SQBRTEST202"]), + tests: json!([helpers::filler_test()]), + allowed_tests: json!([]), + expected_messages: &[ + "SQBRTEST202 models/orders.sql: model \"orders\" has 0 tests; 1 required (1 empty-input-only test not counted; see SQBRTEST203)", + ], + }, + test_types::EmptyInputMinimumTestsTestCase { + description: "the same test with input rows satisfies the minimum", + select: json!(["SQBRTEST202"]), + tests: json!([real_test]), + allowed_tests: json!([]), + expected_messages: &[], + }, + ]; + + for test_case in test_cases { + let messages = helpers::empty_input_rule_evaluation( + &project_dir, + test_case.select, + test_case.tests, + test_case.allowed_tests, + true, + )?; + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs index 997dceb20c..c3457bd571 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs @@ -150,3 +150,18 @@ pub(crate) struct SqlTestRulesCacheTestCase { pub(crate) expected_second_hits: u64, pub(crate) expected_second_misses: u64, } + +pub(crate) struct EmptyInputTestRuleTestCase { + pub(crate) description: &'static str, + pub(crate) test: Value, + pub(crate) allowed_tests: Value, + pub(crate) expected_messages: &'static [&'static str], +} + +pub(crate) struct EmptyInputMinimumTestsTestCase { + pub(crate) description: &'static str, + pub(crate) select: Value, + pub(crate) tests: Value, + pub(crate) allowed_tests: Value, + pub(crate) expected_messages: &'static [&'static str], +} diff --git a/crates/sqlbuild-rules-native/src/models.rs b/crates/sqlbuild-rules-native/src/models.rs index dd7dce1ffc..e720b25b28 100644 --- a/crates/sqlbuild-rules-native/src/models.rs +++ b/crates/sqlbuild-rules-native/src/models.rs @@ -176,6 +176,33 @@ impl Default for RulesConfig { } } +impl RulesConfig { + fn rule_option(&self, code: &str, option: &str) -> Option<&Value> { + self.rule_options + .get(code) + .and_then(|options| options.get(option)) + } + + pub(crate) fn rule_option_enabled(&self, code: &str, option: &str) -> bool { + self.rule_option(code, option) + .and_then(Value::as_bool) + .unwrap_or(false) + } + + pub(crate) fn rule_option_strings(&self, code: &str, option: &str) -> Vec { + self.rule_option(code, option) + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default() + } +} + #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] pub(crate) struct Reference { @@ -530,6 +557,8 @@ pub(crate) struct Model { pub constant_declarations: Vec, pub declared_audit_count: u32, pub targeting_test_count: u32, + #[serde(skip_deserializing)] + pub empty_input_only_test_count: u32, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -600,6 +629,25 @@ pub(crate) struct SqlTestFact { pub assertion_target_model_names: Vec, pub target_model_names: Vec, pub tested_resources: Vec, + #[serde(default)] + pub authored_ctes: Vec, + #[serde(default)] + pub expected_ctes: Vec, + #[serde(default)] + pub assertion_ctes: Vec, + #[serde(default)] + pub has_macro_mocks: bool, + #[serde(default)] + pub has_model_query_overrides: bool, + #[serde(default, skip_deserializing)] + pub empty_input_only: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SqlTestCteFact { + pub name: String, + pub sql: String, } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs index 75bf312128..5e19697f5e 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs @@ -39,6 +39,7 @@ macro_rules! rule { | "SQBRTEST103" | "SQBRTEST104" | "SQBRTEST105" + | "SQBRTEST203" | "SQBRTEST301" ), custom: false, @@ -308,6 +309,13 @@ pub(crate) fn catalogue() -> Vec { "Attach concrete not_null, unique, or accepted_values audits to this model's contract; audits gate promotion when bad rows appear.", ), minimum_tests_rule(), + rule!( + "SQBRTEST203", + "tests", + "empty-input-only-test", + "SQL unit tests must not only prove that empty inputs produce no rows", + "Mock representative input rows and assert concrete transformed output in __expected__ or a targeted __assert__ CTE, or delete this test. An empty-input test is legitimate only when it asserts concrete output, for example a global aggregate that must return one zero-valued summary row compared against expected rows. List a reviewed exception by test name in [rules.rule_options.SQBRTEST203] allowed_tests.", + ), rule!( "SQBRTEST301", "tests", @@ -351,7 +359,7 @@ fn minimum_tests_rule() -> RuleMetadata { slug: "minimum-tests".into(), message: "non-passthrough models must have the configured minimum unit tests".into(), remediation: guidance.remediation( - "Add a SQL unit test that mocks each real import and asserts concrete transformed rows.", + "Add a SQL unit test that mocks each real import and asserts concrete transformed rows. Empty-input-only tests flagged by SQBRTEST203 do not count toward this minimum unless they are listed in its allowed_tests option; do not write them to satisfy the minimum.", ), guidance: Some(guidance), implementation_fingerprint: env!("CARGO_PKG_VERSION").into(), diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/contract_name_types.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/contract_name_types.rs index 16f4a90c1d..f505f71681 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/contract_name_types.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/contract_name_types.rs @@ -11,9 +11,9 @@ pub(crate) fn finding(column: &Column, config: &RulesConfig) -> Option<(&'static || column.name.starts_with("has_") || column.name.starts_with("can_") { - let allowed_numeric = - rule_option_enabled(config, BOOLEAN_COLUMN_RULE, "allow_numeric_indicators") - && numeric_type(&data_type); + let allowed_numeric = config + .rule_option_enabled(BOOLEAN_COLUMN_RULE, "allow_numeric_indicators") + && numeric_type(&data_type); return (data_type != BOOLEAN_TYPE && !allowed_numeric).then(|| { ( BOOLEAN_COLUMN_RULE, @@ -30,11 +30,11 @@ pub(crate) fn finding(column: &Column, config: &RulesConfig) -> Option<(&'static { let allowed_date = column.name.ends_with("_at") && data_type == DATE_TYPE - && rule_option_enabled(config, TIMESTAMP_COLUMN_RULE, "allow_date_for_at"); + && config.rule_option_enabled(TIMESTAMP_COLUMN_RULE, "allow_date_for_at"); let allowed_epoch = numeric_type(&data_type) - && rule_option_enabled(config, TIMESTAMP_COLUMN_RULE, "allow_numeric_epoch"); + && config.rule_option_enabled(TIMESTAMP_COLUMN_RULE, "allow_numeric_epoch"); let allowed_encoded = encoded_temporal_type(&data_type) - && rule_option_enabled(config, TIMESTAMP_COLUMN_RULE, "allow_encoded_values"); + && config.rule_option_enabled(TIMESTAMP_COLUMN_RULE, "allow_encoded_values"); return (!data_type.contains(TIMESTAMP_TYPE) && !allowed_date && !allowed_epoch @@ -51,9 +51,9 @@ pub(crate) fn finding(column: &Column, config: &RulesConfig) -> Option<(&'static } if column.name.ends_with("_date") { let allowed_timestamp = data_type.contains(TIMESTAMP_TYPE) - && rule_option_enabled(config, DATE_COLUMN_RULE, "allow_timestamps"); + && config.rule_option_enabled(DATE_COLUMN_RULE, "allow_timestamps"); let allowed_encoded = encoded_temporal_type(&data_type) - && rule_option_enabled(config, DATE_COLUMN_RULE, "allow_encoded_values"); + && config.rule_option_enabled(DATE_COLUMN_RULE, "allow_encoded_values"); return (data_type != DATE_TYPE && !allowed_timestamp && !allowed_encoded).then(|| { ( DATE_COLUMN_RULE, @@ -67,15 +67,6 @@ pub(crate) fn finding(column: &Column, config: &RulesConfig) -> Option<(&'static None } -fn rule_option_enabled(config: &RulesConfig, code: &str, option: &str) -> bool { - config - .rule_options - .get(code) - .and_then(|options| options.get(option)) - .and_then(serde_json::Value::as_bool) - .unwrap_or(false) -} - fn numeric_type(data_type: &str) -> bool { [ "BIGINT", "DECIMAL", "DOUBLE", "FLOAT", "INT", "NUMBER", "NUMERIC", "REAL", "SMALLINT", diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs index 1636388d13..e997f1fdd4 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs @@ -6,7 +6,7 @@ use crate::models::{Declaration, EvaluateRequest, Fault, Model, RuleMetadata, Ru use crate::rules::_helpers::domain_layout::folder_layer_details; use crate::rules::_helpers::{ contract_name_types, dynamic_contracts, explicit_output_types, model_layers, numeric_decisions, - typed_contract_columns, + sql_test_rules, typed_contract_columns, }; use crate::rules::models::{ FaultCollector, ModelEvaluationRequest, ProjectEvaluationRequest, ResolvedThresholdOverride, @@ -240,7 +240,10 @@ fn requires_parsed_model(selected: &BTreeMap) -> bool { }) } -fn parse_rule_statements(sql: &str, dialect_name: &str) -> Result, ParserError> { +pub(super) fn parse_rule_statements( + sql: &str, + dialect_name: &str, +) -> Result, ParserError> { let dialect = rules_dialect(dialect_name); match parse_with_dialect(sql, dialect.as_ref()) { Ok(statements) => Ok(statements), @@ -700,7 +703,7 @@ pub(super) fn top_ctes(query: &Query) -> &[sqlparser::ast::Cte] { .map_or(&[], |with| with.cte_tables.as_slice()) } -fn group_by_empty(group: &GroupByExpr) -> bool { +pub(super) fn group_by_empty(group: &GroupByExpr) -> bool { matches!(group, GroupByExpr::Expressions(values, _) if values.is_empty()) } @@ -749,7 +752,7 @@ pub(super) fn plain_select(select: &Select, allow_star: bool) -> bool { .all(|item| direct_projection(item, false)) } -fn dependency_name(factor: &TableFactor) -> Option { +pub(super) fn dependency_name(factor: &TableFactor) -> Option { let TableFactor::Table { name, args: Some(_), @@ -1409,18 +1412,15 @@ fn evaluate_test_rules(evaluation: TestRuleEvaluation<'_>) { } if let Some(rule) = evaluation.selected.get("SQBRTEST202") { let minimum = effective_threshold(&evaluation, "min_tests_per_model", 1); - if evaluation.parsed.model.targeting_test_count < minimum { + if let Some(message) = + sql_test_rules::minimum_tests_shortfall(evaluation.parsed.model, minimum) + { evaluation.faults.push(custom_fault!( evaluation.parsed.model, rule, None, - format!( - "model {:?} has {} tests; {} required", - evaluation.parsed.model.name, - evaluation.parsed.model.targeting_test_count, - minimum - ), - None, + message, + None )); } } @@ -1950,7 +1950,7 @@ fn direct_string_literal(expression: &Expr) -> Option { .then(|| expression.to_string()) } -fn unwrap_nested(mut expression: &Expr) -> &Expr { +pub(super) fn unwrap_nested(mut expression: &Expr) -> &Expr { while let Expr::Nested(value) = expression { expression = value; } diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs index b9039419b4..cccdbd2368 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs @@ -1,12 +1,25 @@ +use crate::compiler::main::sql_test_fixture_facts::fixture_facts; use crate::models::{ - DeclarationKind, Fault, ResourceKind, RuleMetadata, ScopeResource, SqlScenarioFact, - SqlTestFact, SqlTestMode, + DeclarationKind, EvaluateRequest, Fault, Model, ResourceKind, RuleMetadata, RulesConfig, + ScopeResource, SqlScenarioFact, SqlTestCteFact, SqlTestFact, SqlTestMode, +}; +use crate::rules::_helpers::evaluation::{ + dependency_name, group_by_empty, normalize_rules_sql, parse_rule_statements, unwrap_nested, }; use crate::rules::models::ProjectEvaluationRequest; +use sqlparser::ast::{ + BinaryOperator, Expr, FunctionArg, FunctionArgExpr, LimitClause, Query, Select, SetExpr, + Statement, TableFactor, Value, Visit, Visitor, +}; +use std::collections::BTreeMap; +use std::ops::ControlFlow; use std::path::Path; const UNIT_ROOT: &str = "tests/unit"; const SCENARIO_ROOT: &str = "tests/scenarios"; +const EMPTY_INPUT_RULE_CODE: &str = "SQBRTEST203"; +const ALLOWED_TESTS_OPTION: &str = "allowed_tests"; +const PROJECT_CONFIG_PATH: &str = "sqlbuild_project.toml"; pub(crate) fn evaluate_project( evaluation: ProjectEvaluationRequest<'_>, @@ -34,6 +47,13 @@ pub(crate) fn evaluate_project( if let Some(rule) = evaluation.selected.get("SQBRTEST105") { faults.extend(scenario_descriptions(rule, &scenarios)); } + if let Some(rule) = evaluation.selected.get(EMPTY_INPUT_RULE_CODE) { + faults.extend(empty_input_only_faults( + rule, + &evaluation.request.config, + &tests, + )); + } Ok(faults) } @@ -545,3 +565,276 @@ fn slug(value: &str) -> String { } output } + +pub(crate) fn annotate_empty_input_only_tests(mut request: EvaluateRequest) -> EvaluateRequest { + for test in &mut request.sql_tests { + test.empty_input_only = is_empty_input_only(test, &request.dialect); + } + let allowed = allowed_tests(&request.config); + let mut excluded: BTreeMap = BTreeMap::new(); + for test in &request.sql_tests { + if !test.empty_input_only || is_allowlisted(test, &allowed) { + continue; + } + for name in &test.target_model_names { + *excluded.entry(name.clone()).or_default() += 1; + } + } + for model in &mut request.models { + model.empty_input_only_test_count = excluded.get(&model.name).copied().unwrap_or(0); + } + request +} + +pub(crate) fn minimum_tests_shortfall(model: &Model, minimum: u32) -> Option { + let excluded = model.empty_input_only_test_count; + let counted = model.targeting_test_count.saturating_sub(excluded); + if counted >= minimum { + return None; + } + let mut message = format!( + "model {:?} has {counted} tests; {minimum} required", + model.name + ); + if excluded > 0 { + let noun = if excluded == 1 { "test" } else { "tests" }; + message.push_str(&format!( + " ({excluded} empty-input-only {noun} not counted; see {EMPTY_INPUT_RULE_CODE})" + )); + } + Some(message) +} + +fn empty_input_only_faults( + rule: &RuleMetadata, + config: &RulesConfig, + tests: &[&SqlTestFact], +) -> Vec { + let allowed = allowed_tests(config); + let mut faults: Vec = Vec::new(); + for test in tests { + if test.empty_input_only && !is_allowlisted(test, &allowed) { + faults.push(path_fault( + rule, + &test.source_path, + format!( + "unit test block {} ({:?}) mocks only empty inputs and asserts only that no rows are produced", + test.block_index, test.name + ), + rule.remediation.clone(), + )); + } + } + for entry in &allowed { + if !tests.iter().any(|test| test_named(test, entry)) { + faults.push(path_fault( + rule, + PROJECT_CONFIG_PATH, + format!("stale {ALLOWED_TESTS_OPTION} entry {entry:?} names no existing SQL test"), + format!( + "Remove {entry:?} from [rules.rule_options.{EMPTY_INPUT_RULE_CODE}] {ALLOWED_TESTS_OPTION}, or correct it to the name of an existing test." + ), + )); + } + } + faults +} + +fn allowed_tests(config: &RulesConfig) -> Vec { + config.rule_option_strings(EMPTY_INPUT_RULE_CODE, ALLOWED_TESTS_OPTION) +} + +fn is_allowlisted(test: &SqlTestFact, allowed: &[String]) -> bool { + allowed.iter().any(|entry| test_named(test, entry)) +} + +fn test_named(test: &SqlTestFact, entry: &str) -> bool { + test.name == entry || (test.case_name.is_some() && test.parent_name.as_deref() == Some(entry)) +} + +fn is_empty_input_only(test: &SqlTestFact, dialect: &str) -> bool { + if !matches!(test.mode, SqlTestMode::Model) + || test.has_macro_mocks + || test.has_model_query_overrides + || (test.expected_ctes.is_empty() && test.assertion_ctes.is_empty()) + { + return false; + } + let mut mocks = test + .authored_ctes + .iter() + .filter(|cte| fixture_facts(&cte.name, &cte.sql).mock) + .peekable(); + mocks.peek().is_some() + && mocks.all(|cte| empty_relation(cte, dialect)) + && test + .expected_ctes + .iter() + .all(|cte| empty_relation(cte, dialect)) + && test + .assertion_ctes + .iter() + .all(|cte| bare_row_existence(&cte.sql, &test.target_model_names, dialect)) +} + +fn empty_relation(cte: &SqlTestCteFact, dialect: &str) -> bool { + if fixture_facts(&cte.name, &cte.sql).empty_fixture_marker { + return true; + } + let Some(query) = parse_fixture_query(&cte.sql, dialect) else { + return false; + }; + let SetExpr::Select(select) = query.body.as_ref() else { + return false; + }; + query.with.is_none() + && (limited_to_zero_rows(&query) + || (filtered_to_zero_rows(select) && !contains_function(&query.order_by))) +} + +fn parse_fixture_query(sql: &str, dialect: &str) -> Option { + let mut statements = match parse_rule_statements(&normalize_rules_sql(dialect, sql), dialect) { + Ok(statements) => statements, + Err(_) => return None, + }; + match (statements.pop(), statements.is_empty()) { + (Some(Statement::Query(query)), true) => Some(*query), + _ => None, + } +} + +fn limited_to_zero_rows(query: &Query) -> bool { + let limit = match &query.limit_clause { + Some(LimitClause::LimitOffset { + limit: Some(limit), .. + }) => limit, + Some(LimitClause::OffsetCommaLimit { limit, .. }) => limit, + _ => return false, + }; + numeric_literal(limit) == Some(0.0) +} + +fn filtered_to_zero_rows(select: &Select) -> bool { + let global_aggregate_possible = contains_function(select); + select.having.is_none() + && group_by_empty(&select.group_by) + && !global_aggregate_possible + && select.selection.as_ref().is_some_and(constant_false) +} + +fn constant_false(predicate: &Expr) -> bool { + match unwrap_nested(predicate) { + Expr::Value(value) => matches!(value.value, Value::Boolean(false)), + Expr::BinaryOp { + left, + op: BinaryOperator::Eq, + right, + } => numeric_literal(left) + .zip(numeric_literal(right)) + .is_some_and(|(left, right)| left != right), + _ => false, + } +} + +fn numeric_literal(expression: &Expr) -> Option { + let Expr::Value(value) = unwrap_nested(expression) else { + return None; + }; + let Value::Number(number, _) = &value.value else { + return None; + }; + let Ok(parsed) = number.parse::() else { + return None; + }; + Some(parsed) +} + +fn bare_row_existence(sql: &str, targets: &[String], dialect: &str) -> bool { + let Some(query) = parse_fixture_query(sql, dialect) else { + return false; + }; + let SetExpr::Select(select) = query.body.as_ref() else { + return false; + }; + let [source] = select.from.as_slice() else { + return false; + }; + let unfiltered = select.selection.is_none() + && select.prewhere.is_none() + && select.having.is_none() + && select.qualify.is_none() + && select.connect_by.is_none() + && select.lateral_views.is_empty() + && group_by_empty(&select.group_by); + query.with.is_none() + && unfiltered + && source.joins.is_empty() + && reference_target(&source.relation).is_some_and(|target| targets.contains(&target)) + && !contains_function(&select.projection) + && !contains_function(&query.order_by) + && query.limit_clause.is_none() + && query.fetch.is_none() + && query_count(&query) == 1 +} + +fn reference_target(factor: &TableFactor) -> Option { + if dependency_name(factor).as_deref() != Some("__ref") { + return None; + } + let TableFactor::Table { + args: Some(arguments), + .. + } = factor + else { + return None; + }; + let [FunctionArg::Unnamed(FunctionArgExpr::Expr(argument))] = arguments.args.as_slice() else { + return None; + }; + match unwrap_nested(argument) { + Expr::Identifier(identifier) => Some(identifier.value.clone()), + Expr::Value(value) => match &value.value { + Value::SingleQuotedString(name) | Value::DoubleQuotedString(name) => Some(name.clone()), + _ => None, + }, + _ => None, + } +} + +fn contains_function(node: &T) -> bool { + struct Functions; + + impl Visitor for Functions { + type Break = (); + + fn pre_visit_expr(&mut self, expression: &Expr) -> ControlFlow { + if matches!(expression, Expr::Function(_)) { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + } + } + + node.visit(&mut Functions).is_break() +} + +fn query_count(query: &Query) -> usize { + #[derive(Default)] + struct Queries { + count: usize, + } + + impl Visitor for Queries { + type Break = (); + + fn pre_visit_query(&mut self, _query: &Query) -> ControlFlow { + self.count += 1; + ControlFlow::Continue(()) + } + } + + let mut queries = Queries::default(); + let _ = query.visit(&mut queries); + queries.count +} diff --git a/crates/sqlbuild-rules-native/src/rules/main/mod.rs b/crates/sqlbuild-rules-native/src/rules/main/mod.rs index fb8034ccda..3ecc50b539 100644 --- a/crates/sqlbuild-rules-native/src/rules/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/rules/main/mod.rs @@ -6,3 +6,4 @@ pub(crate) mod fingerprint; pub(crate) mod resolve_threshold_overrides; pub(crate) mod select; pub(crate) mod selected_codes; +pub(crate) mod sql_test_coverage; diff --git a/crates/sqlbuild-rules-native/src/rules/main/sql_test_coverage.rs b/crates/sqlbuild-rules-native/src/rules/main/sql_test_coverage.rs new file mode 100644 index 0000000000..8f79e2c170 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/rules/main/sql_test_coverage.rs @@ -0,0 +1,5 @@ +use crate::models::EvaluateRequest; + +pub(crate) fn annotate(request: EvaluateRequest) -> EvaluateRequest { + crate::rules::_helpers::sql_test_rules::annotate_empty_input_only_tests(request) +} diff --git a/docs/execution-observability.md b/docs/execution-observability.md index 97cd6217d7..d01dea295c 100644 --- a/docs/execution-observability.md +++ b/docs/execution-observability.md @@ -105,7 +105,8 @@ to restore the terminal index's observed order among records from the same invoc callback without matching terminal evidence emits no record, and final aggregate JSON omits that incomplete resource rather than inventing an outcome. Closing the writer emits nothing. -Each confirmed audit also emits `audit_completed`. The event records the case name, reusable audit +Each confirmed audit also emits `audit_completed` during the run, as soon as its outcome is +confirmed (model audits when their model result completes). The event records the case name, reusable audit definition name, optional authored description, attachment identity, evaluation mode, outcome, severity and run scope, plus applicable measurement, threshold and bounded evidence details. `pass` and `insufficient` are non-failing outcomes; `insufficient` remains distinct for consumers. diff --git a/docs/rules-migration-ledger.md b/docs/rules-migration-ledger.md index 62eb254726..54cd765772 100644 --- a/docs/rules-migration-ledger.md +++ b/docs/rules-migration-ledger.md @@ -46,6 +46,14 @@ for the new rules. | `SQBKX001`–`SQBKX002` | project-aware rules | `SQBRTEST201`–`SQBRTEST202` | Audit and test minima require compiled model evidence. | | `SQBKX201` | project-aware rules | `SQBRTEST301` | Custom-rule coverage requires the configured Rules catalogue and test inventory. | +## Rules added after the migration + +These built-ins have no former `SQBK` counterpart. + +| Code | Owner | Reason | +| --- | --- | --- | +| `SQBRTEST203` | project-aware rules | Rejecting empty-input-only SQL tests requires compiled test fixtures and assertions; see [SQL test coverage rules](sql-test-coverage-rules.md). | + ## Rules families - `SQBRCONTRACT`: contracts and declared column semantics. diff --git a/docs/sinks.md b/docs/sinks.md index c0a7e844ab..86bd83e747 100644 --- a/docs/sinks.md +++ b/docs/sinks.md @@ -135,7 +135,9 @@ Effective kinds are the intersection of declaration, global, and named sets. Eff severity is the strictest of all three. Omitting declaration options means all kinds at `debug`. Unknown lifecycle sink names, keys, kinds, or severities fail before execution. Python declarations should use `LifecycleEventKind`; TOML continues to use the corresponding string -values. The `audit` kind carries `audit_completed` after an audit outcome is confirmed. Its payload +values. The `audit` kind carries `audit_completed` as soon as each audit outcome is confirmed during +the run: standalone, source, and end audits publish per audit, and model audits publish when their +model result completes. Each executed (non-reused) audit publishes exactly once. Its payload includes audit and attachment identity, evaluation mode, outcome and severity, run scope, optional measurement/sample/threshold summary, bounded evidence metadata and rendered audit SQL diagnostics. @@ -165,7 +167,18 @@ injection, constructs providers, and calls each required provider's `setup` at m use. Shared providers are torn down once in reverse setup order. Shutdown stops lifecycle-event acceptance, performs a bounded drain, and only then requests provider -teardown. Each lifecycle sink invocation runs on its own daemon isolation thread. A call has a one-second +teardown. The drain defaults to two seconds and is configurable for all lifecycle sinks: + +```toml +[sinks.lifecycle] +shutdown_timeout = "30s" +``` + +`shutdown_timeout` is a fixed duration string from `"0s"` to `"10m"` such as `"30s"` or `"2m"`; +values outside that range, calendar units, or non-strings fail before execution. `"0s"` drops anything still queued at exit. The setting is global only and is not +accepted under `[sinks.lifecycle.named.]`. It bounds how long the command waits at exit; it does +not change the per-invocation timeout below. A sink that exports audit results should keep each call +well below that one-second invocation bound and use a drain long enough to flush the tail of a run. Each lifecycle sink invocation runs on its own daemon isolation thread. A call has a one-second default timeout; after timeout that sink is blocked from later invocations. Python cannot kill the thread or reverse destination side effects. Provider teardown is deferred until all live invocations return, so a forever-hung daemon retains its provider session until process exit. A @@ -208,6 +221,19 @@ coalesced rather than backlogged. Failure notices use a separate bounded queue a The final summary has a retained write-once slot. Notification callbacks run on one daemon worker; exceptions are swallowed, and shutdown does not wait for callbacks. +When the final summary reports any `dropped` or `failed` lifecycle attempts, the command prints one +warning line to stderr after its own output, for example: + +```text +Warning: command completed successfully, but lifecycle event export was incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). Increase sinks.lifecycle.shutdown_timeout or check sink health. +``` + +The success clause appears only when the command itself succeeded, so the warning never makes a +successful run look failed; otherwise it starts `Warning: lifecycle event export was incomplete`. +Export loss never changes the exit code, and stdout (including `--json`) is unaffected. Nothing is +printed when every accepted attempt was delivered. With several sinks the counts are event +deliveries and only affected sinks are named. + Nonfinal health always has `flush_complete=false`. Final `flush_complete=true` means the dispatcher stopped, no exporter invocation remains live, the queue is empty, and all accepted pairs satisfy the accounting invariant. A timed-out live invocation makes it false even though that attempt is diff --git a/docs/sql-test-coverage-rules.md b/docs/sql-test-coverage-rules.md new file mode 100644 index 0000000000..96996a1cd1 --- /dev/null +++ b/docs/sql-test-coverage-rules.md @@ -0,0 +1,81 @@ +# SQL test coverage rules + +`SQBRTEST202` (`minimum-tests`) requires each non-passthrough model to have at least +`min_tests_per_model` model-mode SQL unit tests. `SQBRTEST203` (`empty-input-only-test`) rejects +tests that cannot fail because they only prove that empty inputs produce no rows. Both rules are +enabled by default when the `SQBRTEST` family is selected. + +## Empty-input-only tests + +Do not write tests like this one: + +```sql +TEST (name "orders__empty_inputs_produce_no_rows"); + +WITH __source__raw_orders AS ( + SELECT NULL AS order_id, NULL AS amount + WHERE FALSE +), __assert__empty_inputs_produce_no_rows AS ( + SELECT 1 AS unexpected_row + FROM __ref("orders") +) +SELECT 1 +``` + +When a model's rows come only from its inputs, this test passes for almost any model logic, so it +adds no coverage. `SQBRTEST203` flags a model-mode test when all of the following hold: + +- Every mock (`__ref__`, `__source__`, `__seed__`, `__dbt_ref__`, `__table_fn__`) is a single + `SELECT` without a set operator that is filtered by `WHERE FALSE` or `WHERE 1 = 0`, limited by + `LIMIT 0`, or is `SELECT * FROM __EMPTY_FIXTURE()`. A `UNION` whose last branch is filtered is + not empty. +- Every `__expected__` CTE is empty in the same way, or there is none. +- Every `__assert__` CTE is a bare row-existence check, `SELECT FROM __ref("")` + on a tested model, with no `WHERE`, `JOIN`, `GROUP BY`, `HAVING`, `QUALIFY`, set operator, + `EXISTS`, or subquery. +- The test has no `__macro__` mocks. + +Macro, UDF, and table-function tests are never flagged, and SQL that the rules parser cannot read +is never flagged. + +Replace a flagged test with one that mocks representative rows and asserts concrete transformed +output, or delete it. + +## Legitimate empty-input tests + +An empty-input test is legitimate when it asserts concrete output. For example, a global aggregate +must still return one zero-valued summary row when it has no input: + +```sql +TEST (name "order_summary__empty_inputs_return_zero_row"); + +WITH __source__raw_orders AS ( + SELECT CAST(NULL AS INTEGER) AS order_id, CAST(NULL AS INTEGER) AS amount + WHERE FALSE +), __expected__order_summary AS ( + SELECT 0 AS order_count, 0 AS total_amount +) +SELECT 1 +``` + +An `__assert__` that compares the model against a summary row with `EXCEPT` in both directions, +or one that filters for invalid rows, is also not flagged. + +## Reviewed exceptions + +List deliberate exceptions by test name in the project file, not in the test: + +```toml +[rules.rule_options.SQBRTEST203] +allowed_tests = ["customers__empty_inputs_produce_no_rows"] +``` + +`allowed_tests` must be a list of strings. For a parameterized test, an entry may name the whole +test or one expanded case. An entry that names no existing test is reported as stale. + +## Interaction with minimum tests + +Tests matched by `SQBRTEST203` do not count toward `min_tests_per_model`, whether or not +`SQBRTEST203` is selected, so filler cannot satisfy the minimum. Tests listed in `allowed_tests` are +reviewed exceptions and do count. A `SQBRTEST202` finding reports how many empty-input-only tests +it did not count. diff --git a/scripts/dupscore/README.md b/scripts/dupscore/README.md new file mode 100644 index 0000000000..70da1588ce --- /dev/null +++ b/scripts/dupscore/README.md @@ -0,0 +1,53 @@ +# dupscore + +Duplication advisory tool for the SQLBuild repository. `make dupscore` runs the default +`clones` mode. + +## `dupscore clones` (default) + +Reports concrete function-level clones: + +- Units are Python functions and methods under `src/sqlbuild` (via `ast`; decorators are outside + the line range) and Rust `fn` items with bodies under `crates/*/src` (free functions, impl and + trait methods; via a small lexer that handles nested block comments, raw and byte strings, and + chars versus lifetimes). Test code is skipped by default: Python `tests/`, Rust `tests/` and + `benches/` directories, `#[cfg(test)]` items and modules, and `#[test]` functions. + `--include-tests` analyzes them too. The dbt integration is included, unlike in `report`. +- Token streams are normalised: identifiers become one placeholder, literals one placeholder per + kind, keywords and operators stay. Comments, Python docstrings, annotations, and `->` are + dropped. +- Candidates come from MOSS-style winnowed fingerprints: 12-token k-grams, minimum of every + window of 6 k-grams, so any shared run of 17 or more normalised tokens yields a shared + fingerprint. Fingerprints present in more than 25 units are boilerplate and ignored. Pairs + sharing fingerprints with a Jaccard index of at least 0.2 are scored with the `difflib` + ratio of their normalised streams. +- Categories: `exact` (identical tokens after dropping comments, docstrings, and annotations), + `renamed` (identical normalised stream), `near-miss` (ratio at or above `--min-similarity`, + default 0.8). Units under `--min-tokens` normalised tokens (default 60) are ignored. +- Similar pairs are grouped into transitive clusters and ranked by estimated duplicated tokens. + +Useful options: + +```bash +uv run dupscore clones --since origin/main # only clusters touching lines changed since a rev +uv run dupscore clones --lang rust --top 10 +uv run dupscore clones --path 'src/sqlbuild/compiler/*' --json +``` + +`--since` diffs the worktree against the revision, so uncommitted edits and untracked files count. +Changed members are marked `[new]` or `[changed]`. + +Intentional mirrors can be suppressed in `dupscore.toml`: + +```toml +[[clone_allowlist]] +paths = ["src/sqlbuild/example/backends/*"] +reason = "Backends intentionally mirror one protocol." +``` + +A pair is suppressed when both sides match the entry's globs (`*` also matches `/`). + +## `dupscore report` and `dupscore pair A B` + +The package-pair ranking fused from call-graph, state fan-in, dataclass overlap, same-name, and +co-change signals, and drill-down evidence for one package pair. diff --git a/scripts/dupscore/_helpers/clones/__init__.py b/scripts/dupscore/_helpers/clones/__init__.py new file mode 100644 index 0000000000..43717e6c56 --- /dev/null +++ b/scripts/dupscore/_helpers/clones/__init__.py @@ -0,0 +1 @@ +"""Dupscore clones helpers.""" diff --git a/scripts/dupscore/_helpers/clones/clustering.py b/scripts/dupscore/_helpers/clones/clustering.py new file mode 100644 index 0000000000..e25f5ae0a2 --- /dev/null +++ b/scripts/dupscore/_helpers/clones/clustering.py @@ -0,0 +1,120 @@ +"""Group clone pairs into transitive clusters and rank them.""" + +from __future__ import annotations + +from collections.abc import Callable + +from scripts.dupscore.constants import CATEGORY_ORDER +from scripts.dupscore.models import ( + CloneCluster, + CloneMember, + ClonePair, + ClonePairLink, + CloneUnit, +) + + +def cluster_clone_pairs( + *, + units: list[CloneUnit], + pairs: list[ClonePair], + change_of: Callable[[CloneUnit], str | None], +) -> list[CloneCluster]: + """Group pairs into connected components ordered by estimated duplicated tokens.""" + + clusters: list[CloneCluster] = [ + _build_cluster(units=units, pairs=component, change_of=change_of) + for component in _connected_pair_groups(pairs) + ] + return sorted(clusters, key=_cluster_sort_key) + + +def _connected_pair_groups(pairs: list[ClonePair]) -> list[list[ClonePair]]: + neighbours: dict[int, list[int]] = {} + for pair in pairs: + neighbours.setdefault(pair.left, []).append(pair.right) + neighbours.setdefault(pair.right, []).append(pair.left) + component_of: dict[int, int] = {} + for root in sorted(neighbours): + if root in component_of: + continue + pending: list[int] = [root] + while pending: + node: int = pending.pop() + if node in component_of: + continue + component_of[node] = root + pending.extend(neighbours[node]) + grouped: dict[int, list[ClonePair]] = {} + for pair in pairs: + grouped.setdefault(component_of[pair.left], []).append(pair) + return list(grouped.values()) + + +def _unit_indexes(pairs: list[ClonePair]) -> set[int]: + indexes: set[int] = set() + for pair in pairs: + indexes.update((pair.left, pair.right)) + return indexes + + +def _build_cluster( + *, + units: list[CloneUnit], + pairs: list[ClonePair], + change_of: Callable[[CloneUnit], str | None], +) -> CloneCluster: + unit_indexes: list[int] = sorted( + _unit_indexes(pairs), + key=lambda index: (units[index].path, units[index].start_line, units[index].name), + ) + position: dict[int, int] = {unit_index: slot for slot, unit_index in enumerate(unit_indexes)} + best_similarity: dict[int, float] = {} + for pair in pairs: + for unit_index in (pair.left, pair.right): + best_similarity[unit_index] = max(best_similarity.get(unit_index, 0.0), pair.similarity) + weights: list[float] = [ + len(units[unit_index].normalized) * best_similarity[unit_index] + for unit_index in unit_indexes + ] + links: list[ClonePairLink] = sorted( + ( + ClonePairLink( + left=min(position[pair.left], position[pair.right]), + right=max(position[pair.left], position[pair.right]), + similarity=pair.similarity, + category=pair.category, + ) + for pair in pairs + ), + key=lambda link: (link.left, link.right), + ) + similarities: list[float] = [pair.similarity for pair in pairs] + category: str = max( + (pair.category for pair in pairs), key=lambda value: CATEGORY_ORDER.index(value) + ) + members: tuple[CloneMember, ...] = tuple( + CloneMember( + language=units[unit_index].language, + path=units[unit_index].path, + name=units[unit_index].name, + start_line=units[unit_index].start_line, + end_line=units[unit_index].end_line, + tokens=len(units[unit_index].normalized), + change=change_of(units[unit_index]), + ) + for unit_index in unit_indexes + ) + return CloneCluster( + category=category, + similarity_min=min(similarities), + similarity_max=max(similarities), + duplicated_tokens=round(sum(weights) - max(weights)), + members=members, + links=tuple(links), + ) + + +def _cluster_sort_key(cluster: CloneCluster) -> tuple[int, float, str, int]: + first: CloneMember = cluster.members[0] + return (-cluster.duplicated_tokens, -cluster.similarity_max, first.path, first.start_line) diff --git a/scripts/dupscore/_helpers/clones/filters.py b/scripts/dupscore/_helpers/clones/filters.py new file mode 100644 index 0000000000..54c14dea16 --- /dev/null +++ b/scripts/dupscore/_helpers/clones/filters.py @@ -0,0 +1,119 @@ +"""Report filters: lines changed since a git revision, allowlist globs, and path globs.""" + +from __future__ import annotations + +import re +from fnmatch import fnmatchcase +from pathlib import Path + +from scripts.dupscore._helpers.inputs.source_provider import run_git +from scripts.dupscore.constants import CHANGE_CHANGED, CHANGE_NEW +from scripts.dupscore.models import CloneAllowlistEntry, CloneUnit, FileChanges + +_HUNK_PATTERN: re.Pattern[str] = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") +_TARGET_PREFIX: str = "+++ " +_TARGET_PATH_PREFIX: str = "b/" +_NULL_PATH: str = "/dev/null" +_GIT_OPTIONS: tuple[str, ...] = ("-c", "core.quotePath=false") + + +def collect_changes_since(*, repo_root: Path, revision: str) -> dict[str, FileChanges]: + """Map paths to worktree lines, including untracked files, changed since ``revision``.""" + + _ = run_git(repo_root=repo_root, arguments=["rev-parse", "--verify", f"{revision}^{{commit}}"]) + diff_text: str = run_git( + repo_root=repo_root, + arguments=[ + *_GIT_OPTIONS, + "diff", + "-U0", + "--no-color", + "--no-ext-diff", + "--find-renames", + "--src-prefix=a/", + "--dst-prefix=b/", + revision, + "--", + ], + ) + changes: dict[str, FileChanges] = parse_unified_diff(diff_text) + untracked: str = run_git( + repo_root=repo_root, + arguments=["ls-files", "--others", "--exclude-standard", "-z"], + ) + for relative in untracked.split("\0"): + if relative: + changes[relative] = FileChanges(whole_file=True) + return changes + + +def parse_unified_diff(diff_text: str) -> dict[str, FileChanges]: + """Parse ``git diff -U0`` output into added line ranges and pure-deletion points.""" + + added: dict[str, list[tuple[int, int]]] = {} + deletions: dict[str, list[int]] = {} + current: str | None = None + for line in diff_text.splitlines(): + if line.startswith(_TARGET_PREFIX): + target: str = line[len(_TARGET_PREFIX) :] + current = None if target == _NULL_PATH else target.removeprefix(_TARGET_PATH_PREFIX) + continue + match: re.Match[str] | None = _HUNK_PATTERN.match(line) + if match is None or current is None: + continue + start: int = int(match.group(1)) + count: int = int(match.group(2)) if match.group(2) is not None else 1 + if count == 0: + deletions.setdefault(current, []).append(start) + else: + added.setdefault(current, []).append((start, start + count - 1)) + return { + path: FileChanges( + added_ranges=tuple(added.get(path, ())), + deletion_points=tuple(deletions.get(path, ())), + ) + for path in sorted(set(added) | set(deletions)) + } + + +def classify_unit_change(*, unit: CloneUnit, changes: FileChanges | None) -> str | None: + """Return ``new`` when every unit line was added, ``changed`` on any overlap.""" + + if changes is None: + return None + if changes.whole_file: + return CHANGE_NEW + covered: int = 0 + for range_start, range_end in changes.added_ranges: + overlap: int = min(range_end, unit.end_line) - max(range_start, unit.start_line) + 1 + covered += max(overlap, 0) + if covered >= unit.end_line - unit.start_line + 1: + return CHANGE_NEW + if covered > 0: + return CHANGE_CHANGED + for point in changes.deletion_points: + if unit.start_line <= point < unit.end_line: + return CHANGE_CHANGED + return None + + +def path_matches_any(*, path: str, globs: tuple[str, ...]) -> bool: + """Match a repo-relative POSIX path against globs where ``*`` also crosses ``/``.""" + + return any(fnmatchcase(path, pattern) for pattern in globs) + + +def allowlist_reason( + *, + left_path: str, + right_path: str, + entries: tuple[CloneAllowlistEntry, ...], +) -> str | None: + """Return the reason of the first entry whose globs cover both sides of a pair.""" + + for entry in entries: + if path_matches_any(path=left_path, globs=entry.paths) and path_matches_any( + path=right_path, globs=entry.paths + ): + return entry.reason + return None diff --git a/scripts/dupscore/_helpers/clones/fingerprints.py b/scripts/dupscore/_helpers/clones/fingerprints.py new file mode 100644 index 0000000000..eb733dfcb9 --- /dev/null +++ b/scripts/dupscore/_helpers/clones/fingerprints.py @@ -0,0 +1,31 @@ +"""Winnowed k-gram fingerprints over normalised token streams (MOSS style).""" + +from __future__ import annotations + +import zlib +from functools import cache + +from scripts.dupscore.constants import CLONE_KGRAM_SIZE, CLONE_WINNOW_WINDOW + + +def fingerprint_tokens(normalized: tuple[str, ...]) -> frozenset[int]: + """Select the minimum k-gram hash of every window of consecutive k-grams.""" + + identifiers: list[int] = [_token_identifier(text) for text in normalized] + gram_count: int = len(identifiers) - CLONE_KGRAM_SIZE + 1 + if gram_count <= 0: + return frozenset() + hashes: list[int] = [ + hash(tuple(identifiers[offset : offset + CLONE_KGRAM_SIZE])) for offset in range(gram_count) + ] + if gram_count <= CLONE_WINNOW_WINDOW: + return frozenset({min(hashes)}) + return frozenset( + min(hashes[offset : offset + CLONE_WINNOW_WINDOW]) + for offset in range(gram_count - CLONE_WINNOW_WINDOW + 1) + ) + + +@cache +def _token_identifier(text: str) -> int: + return zlib.crc32(text.encode("utf-8")) diff --git a/scripts/dupscore/_helpers/clones/matching.py b/scripts/dupscore/_helpers/clones/matching.py new file mode 100644 index 0000000000..307ab53b7a --- /dev/null +++ b/scripts/dupscore/_helpers/clones/matching.py @@ -0,0 +1,100 @@ +"""Find similar unit pairs from normalised identity keys and shared fingerprints.""" + +from __future__ import annotations + +from collections import Counter +from difflib import SequenceMatcher +from itertools import combinations + +from scripts.dupscore.constants import ( + CATEGORY_EXACT, + CATEGORY_NEAR_MISS, + CATEGORY_RENAMED, + CLONE_CANDIDATE_MIN_JACCARD, + CLONE_MAX_FINGERPRINT_UNITS, + CLONE_MIN_FINGERPRINTS, +) +from scripts.dupscore.models import ClonePair, CloneUnit + +_MIN_POSTING_UNITS: int = 2 + + +def find_clone_pairs( + *, + units: list[CloneUnit], + fingerprints: list[frozenset[int]], + min_similarity: float, +) -> list[ClonePair]: + """Return identical-stream pairs plus near-miss pairs above the similarity threshold.""" + + pairs: dict[tuple[int, int], ClonePair] = {} + for left, right in _identical_stream_pairs(units): + category: str = ( + CATEGORY_EXACT + if units[left].concrete_key == units[right].concrete_key + else CATEGORY_RENAMED + ) + pairs[(left, right)] = ClonePair(left=left, right=right, similarity=1.0, category=category) + + postings: dict[int, list[int]] = {} + for unit_index, unit_fingerprints in enumerate(fingerprints): + for fingerprint in unit_fingerprints: + postings.setdefault(fingerprint, []).append(unit_index) + boilerplate: set[int] = { + fingerprint + for fingerprint, members in postings.items() + if len(members) > CLONE_MAX_FINGERPRINT_UNITS + } + distinctive_sizes: list[int] = [ + len(unit_fingerprints - boilerplate) for unit_fingerprints in fingerprints + ] + shared_counts: Counter[tuple[int, int]] = Counter() + for fingerprint, members in postings.items(): + if fingerprint in boilerplate or len(members) < _MIN_POSTING_UNITS: + continue + shared_counts.update(combinations(members, 2)) + + for (left, right), shared in shared_counts.items(): + if (left, right) in pairs or units[left].language != units[right].language: + continue + left_size: int = distinctive_sizes[left] + right_size: int = distinctive_sizes[right] + if min(left_size, right_size) < CLONE_MIN_FINGERPRINTS: + continue + if shared / (left_size + right_size - shared) < CLONE_CANDIDATE_MIN_JACCARD: + continue + similarity: float = _stream_similarity( + left=units[left].normalized, + right=units[right].normalized, + min_similarity=min_similarity, + ) + if similarity >= min_similarity: + pairs[(left, right)] = ClonePair( + left=left, + right=right, + similarity=round(similarity, 4), + category=CATEGORY_NEAR_MISS, + ) + return [pairs[key] for key in sorted(pairs)] + + +def _stream_similarity( + *, + left: tuple[str, ...], + right: tuple[str, ...], + min_similarity: float, +) -> float: + shorter, longer = sorted((len(left), len(right))) + if 2 * shorter / (shorter + longer) < min_similarity: + return 0.0 + return SequenceMatcher(None, left, right, autojunk=False).ratio() + + +def _identical_stream_pairs(units: list[CloneUnit]) -> list[tuple[int, int]]: + groups: dict[tuple[str, str], list[int]] = {} + for unit_index, unit in enumerate(units): + groups.setdefault((unit.language, unit.normalized_key), []).append(unit_index) + identical: list[tuple[int, int]] = [] + for members in groups.values(): + identical.extend(combinations(members, 2)) + return identical diff --git a/scripts/dupscore/_helpers/clones/python_units.py b/scripts/dupscore/_helpers/clones/python_units.py new file mode 100644 index 0000000000..150f7454d0 --- /dev/null +++ b/scripts/dupscore/_helpers/clones/python_units.py @@ -0,0 +1,222 @@ +"""Extract normalised function and method units from Python sources.""" + +from __future__ import annotations + +import ast +import io +import keyword +import token +import tokenize +from bisect import bisect_left +from dataclasses import dataclass + +from scripts.dupscore._helpers.clones.tokens import build_clone_unit +from scripts.dupscore.constants import ( + LANGUAGE_PYTHON, + PLACEHOLDER_FSTRING, + PLACEHOLDER_IDENTIFIER, + PLACEHOLDER_NUMBER, + PLACEHOLDER_STRING, +) +from scripts.dupscore.models import CloneUnit + +_KEYWORDS: frozenset[str] = frozenset(keyword.kwlist) +_DROPPED_TYPES: frozenset[int] = frozenset( + {tokenize.NL, tokenize.COMMENT, tokenize.ENDMARKER, tokenize.ENCODING} +) +_STRUCTURE_MARKERS: dict[int, str] = { + tokenize.NEWLINE: "$nl", + tokenize.INDENT: "$in", + tokenize.DEDENT: "$de", +} +_LITERAL_PLACEHOLDERS: dict[int, str] = { + tokenize.NUMBER: PLACEHOLDER_NUMBER, + tokenize.STRING: PLACEHOLDER_STRING, +} +_FSTRING_START_TYPES: frozenset[int] = frozenset( + getattr(token, name) for name in ("FSTRING_START", "TSTRING_START") if hasattr(token, name) +) +_FSTRING_BODY_TYPES: frozenset[int] = frozenset( + getattr(token, name) + for name in ("FSTRING_MIDDLE", "FSTRING_END", "TSTRING_MIDDLE", "TSTRING_END") + if hasattr(token, name) +) +_RETURN_ARROW: str = "->" + +type _Position = tuple[int, int] +type _Span = tuple[_Position, _Position] +type _FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef + + +@dataclass(frozen=True, slots=True) +class _IgnoredSpan: + start: _Position + end: _Position + statement: bool + + +@dataclass(frozen=True, slots=True) +class _UnitNode: + node: _FunctionNode + qualified_name: str + + +def extract_python_units(*, relative_path: str, source: str) -> tuple[CloneUnit, ...]: + """Return one unit per top-level function or class method, without decorators.""" + + try: + tree: ast.Module = ast.parse(source) + tokens: list[tokenize.TokenInfo] = [ + item + for item in tokenize.generate_tokens(io.StringIO(source).readline) + if item.type not in _DROPPED_TYPES + ] + except (SyntaxError, tokenize.TokenError): + return () + starts: list[_Position] = [item.start for item in tokens] + normalized: list[str | None] = [_normalize(item) for item in tokens] + for span in _ignored_spans(tree.body): + end: int = bisect_left(starts, span.end) + if span.statement and end < len(tokens) and tokens[end].type == tokenize.NEWLINE: + end += 1 + for index in range(bisect_left(starts, span.start), end): + normalized[index] = None + units: list[CloneUnit] = [] + for unit_node in _collect_unit_nodes(body=tree.body, prefix=""): + node: _FunctionNode = unit_node.node + end_line: int = node.end_lineno if node.end_lineno is not None else node.lineno + end_column: int = node.end_col_offset if node.end_col_offset is not None else 0 + first: int = bisect_left(starts, (node.lineno, node.col_offset)) + last: int = bisect_left(starts, (end_line, end_column)) + kept: list[tuple[str, str]] = [ + (text, tokens[index].string) + for index in range(first, last) + if (text := normalized[index]) is not None + ] + units.append( + build_clone_unit( + language=LANGUAGE_PYTHON, + path=relative_path, + name=unit_node.qualified_name, + start_line=node.lineno, + end_line=end_line, + normalized=[text for text, _ in kept], + concrete=[text for _, text in kept], + ) + ) + return tuple(units) + + +def _normalize(item: tokenize.TokenInfo) -> str | None: + if item.type == tokenize.NAME: + return item.string if item.string in _KEYWORDS else PLACEHOLDER_IDENTIFIER + if item.type == tokenize.OP: + return None if item.string == _RETURN_ARROW else item.string + if item.type in _FSTRING_BODY_TYPES: + return None + if item.type in _FSTRING_START_TYPES: + return PLACEHOLDER_FSTRING + marker: str | None = _STRUCTURE_MARKERS.get(item.type) + if marker is not None: + return marker + return _LITERAL_PLACEHOLDERS.get(item.type, item.string) + + +def _collect_unit_nodes(*, body: list[ast.stmt], prefix: str) -> list[_UnitNode]: + collected: list[_UnitNode] = [] + for statement in body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + collected.append(_UnitNode(node=statement, qualified_name=prefix + statement.name)) + elif isinstance(statement, ast.ClassDef): + collected.extend( + _collect_unit_nodes(body=statement.body, prefix=f"{prefix}{statement.name}.") + ) + elif isinstance(statement, (ast.If, ast.Try)): + collected.extend(_collect_unit_nodes(body=statement.body, prefix=prefix)) + collected.extend(_collect_unit_nodes(body=statement.orelse, prefix=prefix)) + return collected + + +def _ignored_spans(body: list[ast.stmt]) -> list[_IgnoredSpan]: + spans: list[_IgnoredSpan] = [] + for statement in body: + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + docstring: ast.Expr | None = _docstring_expression(statement) + if docstring is not None: + docstring_start, docstring_end = _span_of(docstring) + spans.append(_IgnoredSpan(start=docstring_start, end=docstring_end, statement=True)) + annotation_spans: list[_Span] = [] + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + annotation_spans = _signature_annotation_spans(statement) + elif isinstance(statement, ast.AnnAssign): + annotation_spans = [ + _annotation_span(owner=statement.target, annotation=statement.annotation) + ] + spans.extend( + _IgnoredSpan(start=start, end=end, statement=False) for start, end in annotation_spans + ) + for child_body in _child_bodies(statement): + spans.extend(_ignored_spans(child_body)) + return spans + + +def _child_bodies(statement: ast.stmt) -> list[list[ast.stmt]]: + bodies: list[list[ast.stmt]] = [] + for field_name in ("body", "orelse", "finalbody"): + value: object = getattr(statement, field_name, None) + if isinstance(value, list): + bodies.append([item for item in value if isinstance(item, ast.stmt)]) + if isinstance(statement, (ast.Try, ast.TryStar)): + bodies.extend(handler.body for handler in statement.handlers) + elif isinstance(statement, ast.Match): + bodies.extend(case.body for case in statement.cases) + return bodies + + +def _signature_annotation_spans(node: _FunctionNode) -> list[_Span]: + spans: list[_Span] = [_span_of(node.returns)] if node.returns is not None else [] + arguments: ast.arguments = node.args + for argument in ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + *(item for item in (arguments.vararg, arguments.kwarg) if item is not None), + ): + if argument.annotation is not None: + spans.append(_annotation_span(owner=argument, annotation=argument.annotation)) + return spans + + +def _annotation_span(*, owner: ast.expr | ast.arg, annotation: ast.expr) -> _Span: + """Cover the ``:`` separator and the annotation that follow a name.""" + + owner_end: _Position = ( + (owner.lineno, owner.col_offset + len(owner.arg)) + if isinstance(owner, ast.arg) + else ( + owner.end_lineno if owner.end_lineno is not None else owner.lineno, + owner.end_col_offset if owner.end_col_offset is not None else owner.col_offset, + ) + ) + return (owner_end, _span_of(annotation)[1]) + + +def _docstring_expression( + node: ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef, +) -> ast.Expr | None: + if not node.body: + return None + first: ast.stmt = node.body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + return first + return None + + +def _span_of(node: ast.expr | ast.stmt) -> _Span: + end_line: int = node.end_lineno if node.end_lineno is not None else node.lineno + end_column: int = node.end_col_offset if node.end_col_offset is not None else node.col_offset + return ((node.lineno, node.col_offset), (end_line, end_column)) diff --git a/scripts/dupscore/_helpers/clones/rendering.py b/scripts/dupscore/_helpers/clones/rendering.py new file mode 100644 index 0000000000..4ec2f0618b --- /dev/null +++ b/scripts/dupscore/_helpers/clones/rendering.py @@ -0,0 +1,70 @@ +"""Render clone reports as concise text or deterministic JSON.""" + +from __future__ import annotations + +import json +from dataclasses import asdict + +from scripts.dupscore.constants import CLONE_TEXT_MAX_MEMBERS +from scripts.dupscore.models import CloneCluster, CloneMember, CloneReport + +_INDENT: str = " " + + +def render_clone_text(*, report: CloneReport, top: int) -> str: + """Render the ranked clusters with one line per member.""" + + counts: str = ", ".join( + f"{language} {count} units" for language, count in report.unit_counts.items() + ) + scope: str = f" changed since {report.since}" if report.since is not None else "" + lines: list[str] = [ + f"dupscore clones{scope}: {len(report.clusters)} clusters " + f"({counts}; {report.allowlisted_pairs} allowlisted pairs hidden)" + ] + for position, cluster in enumerate(report.clusters[:top], start=1): + lines.append(f"{position:3d}. {_cluster_heading(cluster)}") + shown: tuple[CloneMember, ...] = cluster.members[:CLONE_TEXT_MAX_MEMBERS] + lines.extend(_INDENT + _member_line(member) for member in shown) + hidden: int = len(cluster.members) - len(shown) + if hidden: + lines.append(f"{_INDENT}... {hidden} more members (see --json)") + if not report.clusters: + lines.append("no clones found") + elif len(report.clusters) > top: + lines.append(f"... {len(report.clusters) - top} more clusters (use --top)") + return "\n".join(lines) + + +def render_clone_json(*, report: CloneReport, top: int) -> str: + """Render the ranked clusters as deterministic JSON.""" + + payload: dict[str, object] = { + "since": report.since, + "unit_counts": report.unit_counts, + "allowlisted_pairs": report.allowlisted_pairs, + "total_clusters": len(report.clusters), + "clusters": [ + {"rank": position, **asdict(cluster)} + for position, cluster in enumerate(report.clusters[:top], start=1) + ], + } + return json.dumps(payload, indent=1, sort_keys=True) + + +def _cluster_heading(cluster: CloneCluster) -> str: + lowest: str = f"{cluster.similarity_min:.2f}" + highest: str = f"{cluster.similarity_max:.2f}" + similarity: str = highest if lowest == highest else f"{lowest}-{highest}" + return ( + f"{cluster.category} sim {similarity}, ~{cluster.duplicated_tokens} duplicated tokens, " + f"{len(cluster.members)} members" + ) + + +def _member_line(member: CloneMember) -> str: + change: str = f" [{member.change}]" if member.change is not None else "" + return ( + f"{member.path}:{member.start_line}-{member.end_line} {member.name} " + f"({member.tokens} tokens){change}" + ) diff --git a/scripts/dupscore/_helpers/clones/rust_lexer.py b/scripts/dupscore/_helpers/clones/rust_lexer.py new file mode 100644 index 0000000000..25f0a519ea --- /dev/null +++ b/scripts/dupscore/_helpers/clones/rust_lexer.py @@ -0,0 +1,251 @@ +"""A small Rust lexer that is precise enough for brace matching and clone tokens.""" + +from __future__ import annotations + +from scripts.dupscore.constants import ( + RUST_KIND_BYTE, + RUST_KIND_BYTE_STRING, + RUST_KIND_CHAR, + RUST_KIND_IDENTIFIER, + RUST_KIND_LIFETIME, + RUST_KIND_NUMBER, + RUST_KIND_PUNCTUATION, + RUST_KIND_STRING, +) +from scripts.dupscore.models import RustToken + +_THREE_CHAR_OPERATORS: frozenset[str] = frozenset({"..=", "...", "<<=", ">>="}) +_TWO_CHAR_OPERATORS: frozenset[str] = frozenset( + { + "::", + "->", + "=>", + "==", + "!=", + "<=", + ">=", + "&&", + "||", + "+=", + "-=", + "*=", + "/=", + "%=", + "^=", + "&=", + "|=", + "..", + } +) +_STRING_PREFIX_KINDS: dict[str, str] = { + "b": RUST_KIND_BYTE_STRING, + "br": RUST_KIND_BYTE_STRING, + "c": RUST_KIND_STRING, + "cr": RUST_KIND_STRING, + "r": RUST_KIND_STRING, +} +_RAW_MARKER: str = "r" +_BYTE_PREFIX: str = "b" +_QUOTE: str = '"' +_APOSTROPHE: str = "'" +_BACKSLASH: str = "\\" +_HASH: str = "#" +_NEWLINE: str = "\n" +_UNDERSCORE: str = "_" +_DECIMAL_POINT: str = "." +_EXPONENT_MARKERS: frozenset[str] = frozenset({"e", "E"}) +_EXPONENT_SIGNS: frozenset[str] = frozenset({"+", "-"}) +_HEX_PREFIXES: tuple[str, ...] = ("0x", "0X") +_LINE_COMMENT: str = "//" +_BLOCK_COMMENT_OPEN: str = "/*" +_BLOCK_COMMENT_CLOSE: str = "*/" +_ESCAPE_WIDTH: int = 2 +_EXPONENT_WIDTH: int = 2 +_CHAR_AND_CLOSING_WIDTH: int = 2 +_OPERATOR_WIDTHS: tuple[tuple[int, frozenset[str]], ...] = ( + (3, _THREE_CHAR_OPERATORS), + (2, _TWO_CHAR_OPERATORS), +) + + +def lex_rust(source: str) -> list[RustToken]: + """Tokenise Rust source, dropping whitespace and all comments.""" + + lexer: _Lexer = _Lexer(source=source) + return lexer.run() + + +def _is_word_start(character: str) -> bool: + return character == _UNDERSCORE or character.isalpha() + + +def _is_word_part(character: str) -> bool: + return character == _UNDERSCORE or character.isalnum() + + +class _Lexer: + def __init__(self, *, source: str) -> None: + self._source: str = source + self._index: int = 0 + self._line: int = 1 + self._tokens: list[RustToken] = [] + + def run(self) -> list[RustToken]: + length: int = len(self._source) + while self._index < length: + character: str = self._source[self._index] + if character == _NEWLINE: + self._line += 1 + self._index += 1 + elif character.isspace(): + self._index += 1 + elif self._source.startswith(_LINE_COMMENT, self._index): + self._skip_line_comment() + elif self._source.startswith(_BLOCK_COMMENT_OPEN, self._index): + self._skip_block_comment() + elif character == _QUOTE: + self._lex_quoted(kind=RUST_KIND_STRING, prefix_length=0) + elif character == _APOSTROPHE: + self._lex_apostrophe(prefix_length=0) + elif character.isdigit(): + self._lex_number() + elif _is_word_start(character): + self._lex_word() + else: + self._lex_punctuation() + return self._tokens + + def _emit(self, *, kind: str, start: int, line: int) -> None: + text: str = self._source[start : self._index] + self._tokens.append(RustToken(kind=kind, text=text, line=line)) + + def _advance_to(self, end: int) -> None: + self._line += self._source.count(_NEWLINE, self._index, end) + self._index = end + + def _word_end(self, start: int) -> int: + cursor: int = start + while cursor < len(self._source) and _is_word_part(self._source[cursor]): + cursor += 1 + return cursor + + def _skip_line_comment(self) -> None: + end: int = self._source.find(_NEWLINE, self._index) + self._index = len(self._source) if end < 0 else end + + def _skip_block_comment(self) -> None: + depth: int = 0 + cursor: int = self._index + length: int = len(self._source) + while cursor < length: + if self._source.startswith(_BLOCK_COMMENT_OPEN, cursor): + depth += 1 + cursor += len(_BLOCK_COMMENT_OPEN) + elif self._source.startswith(_BLOCK_COMMENT_CLOSE, cursor): + depth -= 1 + cursor += len(_BLOCK_COMMENT_CLOSE) + if depth == 0: + break + else: + cursor += 1 + self._advance_to(cursor) + + def _lex_word(self) -> None: + start: int = self._index + line: int = self._line + cursor: int = self._word_end(start) + word: str = self._source[start:cursor] + following: str = self._source[cursor : cursor + 1] + prefix_kind: str | None = _STRING_PREFIX_KINDS.get(word) + if prefix_kind is not None and following in (_QUOTE, _HASH): + if word.endswith(_RAW_MARKER): + if self._lex_raw_string(kind=prefix_kind, prefix_end=cursor): + return + elif following == _QUOTE: + self._lex_quoted(kind=prefix_kind, prefix_length=cursor - start) + return + if word == _BYTE_PREFIX and following == _APOSTROPHE: + self._lex_apostrophe(prefix_length=len(_BYTE_PREFIX)) + return + if word == _RAW_MARKER and following == _HASH and self._is_raw_identifier(cursor): + cursor = self._word_end(cursor + 1) + self._index = cursor + self._emit(kind=RUST_KIND_IDENTIFIER, start=start, line=line) + + def _is_raw_identifier(self, hash_index: int) -> bool: + after: int = hash_index + 1 + return after < len(self._source) and _is_word_start(self._source[after]) + + def _lex_raw_string(self, *, kind: str, prefix_end: int) -> bool: + start: int = self._index + line: int = self._line + cursor: int = prefix_end + hashes: int = 0 + while cursor < len(self._source) and self._source[cursor] == _HASH: + hashes += 1 + cursor += 1 + if cursor >= len(self._source) or self._source[cursor] != _QUOTE: + return False + terminator: str = _QUOTE + _HASH * hashes + end: int = self._source.find(terminator, cursor + 1) + end = len(self._source) if end < 0 else end + len(terminator) + self._advance_to(end) + self._emit(kind=kind, start=start, line=line) + return True + + def _lex_quoted(self, *, kind: str, prefix_length: int) -> None: + start: int = self._index + line: int = self._line + cursor: int = start + prefix_length + 1 + length: int = len(self._source) + while cursor < length and self._source[cursor] != _QUOTE: + cursor += _ESCAPE_WIDTH if self._source[cursor] == _BACKSLASH else 1 + self._advance_to(min(cursor + 1, length)) + self._emit(kind=kind, start=start, line=line) + + def _lex_apostrophe(self, *, prefix_length: int) -> None: + start: int = self._index + line: int = self._line + after: int = start + prefix_length + 1 + length: int = len(self._source) + kind: str = RUST_KIND_BYTE if prefix_length else RUST_KIND_CHAR + if after < length and self._source[after] == _BACKSLASH: + closing: int = self._source.find(_APOSTROPHE, after + _ESCAPE_WIDTH) + self._index = length if closing < 0 else closing + 1 + self._emit(kind=kind, start=start, line=line) + return + if after + 1 < length and self._source[after + 1] == _APOSTROPHE: + self._index = after + _CHAR_AND_CLOSING_WIDTH + self._emit(kind=kind, start=start, line=line) + return + self._index = self._word_end(after) + self._emit(kind=RUST_KIND_LIFETIME, start=start, line=line) + + def _lex_number(self) -> None: + start: int = self._index + line: int = self._line + cursor: int = start + length: int = len(self._source) + hexadecimal: bool = self._source.startswith(_HEX_PREFIXES, start) + while cursor < length: + character: str = self._source[cursor] + following: str = self._source[cursor + 1 : cursor + 2] + if not hexadecimal and character in _EXPONENT_MARKERS and following in _EXPONENT_SIGNS: + cursor += _EXPONENT_WIDTH + elif _is_word_part(character) or (character == _DECIMAL_POINT and following.isdigit()): + cursor += 1 + else: + break + self._index = cursor + self._emit(kind=RUST_KIND_NUMBER, start=start, line=line) + + def _lex_punctuation(self) -> None: + start: int = self._index + line: int = self._line + for width, operators in _OPERATOR_WIDTHS: + if self._source[start : start + width] in operators: + self._index = start + width + self._emit(kind=RUST_KIND_PUNCTUATION, start=start, line=line) + return + self._index = start + 1 + self._emit(kind=RUST_KIND_PUNCTUATION, start=start, line=line) diff --git a/scripts/dupscore/_helpers/clones/rust_units.py b/scripts/dupscore/_helpers/clones/rust_units.py new file mode 100644 index 0000000000..5b92341774 --- /dev/null +++ b/scripts/dupscore/_helpers/clones/rust_units.py @@ -0,0 +1,347 @@ +"""Extract normalised `fn` item units from Rust sources.""" + +from __future__ import annotations + +from pathlib import PurePosixPath + +from scripts.dupscore._helpers.clones.rust_lexer import lex_rust +from scripts.dupscore._helpers.clones.tokens import build_clone_unit +from scripts.dupscore.constants import ( + LANGUAGE_RUST, + PLACEHOLDER_BYTE, + PLACEHOLDER_BYTE_STRING, + PLACEHOLDER_CHAR, + PLACEHOLDER_IDENTIFIER, + PLACEHOLDER_LIFETIME, + PLACEHOLDER_NUMBER, + PLACEHOLDER_STRING, + RUST_KIND_BYTE, + RUST_KIND_BYTE_STRING, + RUST_KIND_CHAR, + RUST_KIND_IDENTIFIER, + RUST_KIND_LIFETIME, + RUST_KIND_NUMBER, + RUST_KIND_PUNCTUATION, + RUST_KIND_STRING, +) +from scripts.dupscore.models import CloneUnit, RustFileUnits, RustToken + +_RUST_KEYWORDS: frozenset[str] = frozenset( + { + "as", + "async", + "await", + "break", + "const", + "continue", + "crate", + "dyn", + "else", + "enum", + "extern", + "false", + "fn", + "for", + "if", + "impl", + "in", + "let", + "loop", + "match", + "mod", + "move", + "mut", + "pub", + "ref", + "return", + "self", + "Self", + "static", + "struct", + "super", + "trait", + "true", + "type", + "unsafe", + "use", + "where", + "while", + } +) +_LITERAL_PLACEHOLDERS: dict[str, str] = { + RUST_KIND_STRING: PLACEHOLDER_STRING, + RUST_KIND_BYTE_STRING: PLACEHOLDER_BYTE_STRING, + RUST_KIND_CHAR: PLACEHOLDER_CHAR, + RUST_KIND_BYTE: PLACEHOLDER_BYTE, + RUST_KIND_NUMBER: PLACEHOLDER_NUMBER, + RUST_KIND_LIFETIME: PLACEHOLDER_LIFETIME, +} +_OPEN_BRACE: str = "{" +_SEMICOLON: str = ";" +_ATTRIBUTE_MARKER: str = "#" +_INNER_ATTRIBUTE_MARKER: str = "!" +_ATTRIBUTE_OPEN: str = "[" +_OPENERS: dict[str, str] = {"{": "}", "(": ")", "[": "]"} +_CLOSERS: frozenset[str] = frozenset(_OPENERS.values()) +_SIGNATURE_GROUP_OPENERS: frozenset[str] = frozenset({"(", "["}) +_BODY_OR_END: frozenset[str] = frozenset({_OPEN_BRACE, _SEMICOLON}) +_ANGLE_DEPTH_CHANGES: dict[str, int] = {"<": 1, ">": -1} +_FUNCTION_KEYWORD: str = "fn" +_OWNER_KEYWORDS: frozenset[str] = frozenset({"impl", "trait"}) +_MACRO_RULES: str = "macro_rules" +_MODULE_KEYWORD: str = "mod" +_FOR_KEYWORD: str = "for" +_WHERE_KEYWORD: str = "where" +_MODULE_ROOT_FILES: frozenset[str] = frozenset({"mod.rs", "lib.rs", "main.rs"}) +_TEST_ATTRIBUTE: str = "test" +_CFG_ATTRIBUTE: str = "cfg" +_NEGATION: str = "not" +_PATH_SEPARATOR: str = "::" +_RUST_SUFFIX: str = ".rs" +_DIRECTORY_SUFFIX: str = "/" +_OWNER_SEPARATOR: str = "::" + + +def extract_rust_units( + *, + relative_path: str, + source: str, + include_tests: bool, +) -> RustFileUnits: + """Return every `fn` item with a body, skipping test-only items unless requested.""" + + tokens: list[RustToken] = lex_rust(source) + scanner: _ItemScanner = _ItemScanner( + tokens=tokens, + relative_path=relative_path, + include_tests=include_tests, + ) + scanner.scan(start=0, end=len(tokens), owner=None) + return RustFileUnits( + units=tuple(scanner.units), + test_module_prefixes=tuple(scanner.test_module_prefixes), + ) + + +def _normalize(item: RustToken) -> str: + if item.kind == RUST_KIND_IDENTIFIER: + return item.text if item.text in _RUST_KEYWORDS else PLACEHOLDER_IDENTIFIER + return _LITERAL_PLACEHOLDERS.get(item.kind, item.text) + + +def _match_brackets(tokens: list[RustToken]) -> list[int]: + matches: list[int] = [-1] * len(tokens) + stack: list[int] = [] + for index, item in enumerate(tokens): + if item.kind != RUST_KIND_PUNCTUATION: + continue + if item.text in _OPENERS: + stack.append(index) + elif item.text in _CLOSERS and stack and _OPENERS[tokens[stack[-1]].text] == item.text: + opener: int = stack.pop() + matches[opener] = index + matches[index] = opener + return matches + + +def _is_test_attribute(texts: list[str]) -> bool: + if not texts: + return False + if texts[0] == _CFG_ATTRIBUTE: + return _TEST_ATTRIBUTE in texts and _NEGATION not in texts + return texts[-1] == _TEST_ATTRIBUTE and all( + text == _PATH_SEPARATOR or text.isidentifier() for text in texts + ) + + +def _strip_leading_generics(header: list[RustToken]) -> list[RustToken]: + depth: int = 0 + for position, item in enumerate(header): + depth += _ANGLE_DEPTH_CHANGES.get(item.text, 0) + if depth == 0: + return header if position == 0 else header[position + 1 :] + return [] + + +def _owner_name(header: list[RustToken]) -> str | None: + """Name the self type of an ``impl`` or ``trait`` header (tokens between it and ``{``).""" + + candidates: list[RustToken] = _strip_leading_generics(header) + depth: int = 0 + for position, item in enumerate(candidates): + depth += _ANGLE_DEPTH_CHANGES.get(item.text, 0) + if depth == 0 and item.text == _FOR_KEYWORD: + candidates = candidates[position + 1 :] + break + name: str | None = None + depth = 0 + for item in candidates: + depth += _ANGLE_DEPTH_CHANGES.get(item.text, 0) + if depth == 0 and item.text == _WHERE_KEYWORD: + break + if depth == 0 and item.kind == RUST_KIND_IDENTIFIER and item.text not in _RUST_KEYWORDS: + name = item.text + return name + + +class _ItemScanner: + def __init__( + self, + *, + tokens: list[RustToken], + relative_path: str, + include_tests: bool, + ) -> None: + self._tokens: list[RustToken] = tokens + self._matches: list[int] = _match_brackets(tokens) + self._relative_path: str = relative_path + self._include_tests: bool = include_tests + self.units: list[CloneUnit] = [] + self.test_module_prefixes: list[str] = [] + + def scan(self, *, start: int, end: int, owner: str | None) -> None: + index: int = start + test_item: bool = False + item_start: int | None = None + while index < end: + attribute_end: int | None = self._attribute_end(index) + if attribute_end is not None: + test_item = test_item or self._attribute_is_test(index=index, end=attribute_end) + index = attribute_end + 1 + continue + if self._is_punctuation(index=index, text=_SEMICOLON): + index += 1 + test_item, item_start = False, None + continue + if test_item and not self._include_tests: + self._record_test_module(index) + index = self._item_end(start=index, end=end) + test_item, item_start = False, None + continue + if item_start is None: + item_start = index + next_index: int | None = self._scan_item( + index=index, end=end, owner=owner, item_start=item_start + ) + if next_index is None: + index += 1 + continue + index = next_index + test_item, item_start = False, None + + def _scan_item(self, *, index: int, end: int, owner: str | None, item_start: int) -> int | None: + item: RustToken = self._tokens[index] + is_identifier: bool = item.kind == RUST_KIND_IDENTIFIER + if is_identifier and item.text == _FUNCTION_KEYWORD and self._is_identifier(index + 1): + body: int = self._find_body(start=index + 2, end=end) + if not self._is_punctuation(index=body, text=_OPEN_BRACE) or body >= end: + return body + 1 + close: int = self._closing(opener=body, end=end) + self._emit_unit(start=index, close=close, owner=owner, item_start=item_start) + return close + 1 + if is_identifier and item.text in _OWNER_KEYWORDS: + body = self._find_body(start=index + 1, end=end) + if not self._is_punctuation(index=body, text=_OPEN_BRACE) or body >= end: + return body + 1 + close = self._closing(opener=body, end=end) + self.scan( + start=body + 1, + end=close, + owner=_owner_name(self._tokens[index + 1 : body]), + ) + return close + 1 + if is_identifier and item.text == _MACRO_RULES: + body = index + 1 + while body < end and self._tokens[body].text not in _OPENERS: + body += 1 + return self._closing(opener=body, end=end) + 1 if body < end else end + if self._is_punctuation(index=index, text=_OPEN_BRACE): + close = self._closing(opener=index, end=end) + self.scan(start=index + 1, end=close, owner=owner) + return close + 1 + return None + + def _is_punctuation(self, *, index: int, text: str) -> bool: + return ( + index < len(self._tokens) + and self._tokens[index].kind == RUST_KIND_PUNCTUATION + and self._tokens[index].text == text + ) + + def _is_identifier(self, index: int) -> bool: + return index < len(self._tokens) and self._tokens[index].kind == RUST_KIND_IDENTIFIER + + def _attribute_open(self, index: int) -> int | None: + if not self._is_punctuation(index=index, text=_ATTRIBUTE_MARKER): + return None + opener: int = index + 1 + if self._is_punctuation(index=opener, text=_INNER_ATTRIBUTE_MARKER): + opener += 1 + return opener if self._is_punctuation(index=opener, text=_ATTRIBUTE_OPEN) else None + + def _attribute_end(self, index: int) -> int | None: + opener: int | None = self._attribute_open(index) + if opener is None: + return None + close: int = self._matches[opener] + return close if close > opener else None + + def _attribute_is_test(self, *, index: int, end: int) -> bool: + opener: int | None = self._attribute_open(index) + if opener is None: + return False + return _is_test_attribute([item.text for item in self._tokens[opener + 1 : end]]) + + def _closing(self, *, opener: int, end: int) -> int: + close: int = self._matches[opener] + return close if opener < close < end else end - 1 + + def _find_body(self, *, start: int, end: int) -> int: + index: int = start + while index < end: + item: RustToken = self._tokens[index] + if item.kind == RUST_KIND_PUNCTUATION and item.text in _BODY_OR_END: + return index + if item.kind == RUST_KIND_PUNCTUATION and item.text in _SIGNATURE_GROUP_OPENERS: + index = self._closing(opener=index, end=end) + 1 + continue + index += 1 + return end + + def _item_end(self, *, start: int, end: int) -> int: + body: int = self._find_body(start=start, end=end) + if body >= end: + return end + if self._is_punctuation(index=body, text=_OPEN_BRACE): + return self._closing(opener=body, end=end) + 1 + return body + 1 + + def _record_test_module(self, index: int) -> None: + if not ( + self._tokens[index].text == _MODULE_KEYWORD + and self._is_identifier(index + 1) + and self._is_punctuation(index=index + 2, text=_SEMICOLON) + ): + return + path: PurePosixPath = PurePosixPath(self._relative_path) + base: PurePosixPath = ( + path.parent if path.name in _MODULE_ROOT_FILES else path.parent / path.stem + ) + module_path: str = (base / self._tokens[index + 1].text).as_posix() + self.test_module_prefixes.append(module_path + _RUST_SUFFIX) + self.test_module_prefixes.append(module_path + _DIRECTORY_SUFFIX) + + def _emit_unit(self, *, start: int, close: int, owner: str | None, item_start: int) -> None: + function_name: str = self._tokens[start + 1].text + body: list[RustToken] = self._tokens[start : close + 1] + self.units.append( + build_clone_unit( + language=LANGUAGE_RUST, + path=self._relative_path, + name=f"{owner}{_OWNER_SEPARATOR}{function_name}" if owner else function_name, + start_line=self._tokens[item_start].line, + end_line=self._tokens[close].line, + normalized=[_normalize(item) for item in body], + concrete=[item.text for item in body], + ) + ) diff --git a/scripts/dupscore/_helpers/clones/sources.py b/scripts/dupscore/_helpers/clones/sources.py new file mode 100644 index 0000000000..98c8cc7784 --- /dev/null +++ b/scripts/dupscore/_helpers/clones/sources.py @@ -0,0 +1,86 @@ +"""Collect clone units from the Python package and Rust crates in the worktree.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.dupscore._helpers.clones.python_units import extract_python_units +from scripts.dupscore._helpers.clones.rust_units import extract_rust_units +from scripts.dupscore.constants import ( + LANGUAGE_PYTHON, + LANGUAGE_RUST, + PROJECT_PACKAGE, + PYTHON_TEST_ROOT, + RUST_CRATES_ROOT, + RUST_SOURCE_DIRECTORY, + RUST_TEST_DIRECTORIES, + SOURCE_ROOT, +) +from scripts.dupscore.models import CloneUnit, RustFileUnits + +_PYTHON_PATTERN: str = "*.py" +_RUST_PATTERN: str = "*.rs" +_CRATE_DIRECTORY_INDEX: int = 1 + + +def collect_clone_units( + *, + repo_root: Path, + languages: tuple[str, ...], + include_tests: bool, +) -> list[CloneUnit]: + """Extract units for the requested languages in deterministic path order.""" + + units: list[CloneUnit] = [] + if LANGUAGE_PYTHON in languages: + units.extend(_python_units(repo_root=repo_root, include_tests=include_tests)) + if LANGUAGE_RUST in languages: + units.extend(_rust_units(repo_root=repo_root, include_tests=include_tests)) + return units + + +def _python_units(*, repo_root: Path, include_tests: bool) -> list[CloneUnit]: + files: list[Path] = sorted((repo_root / SOURCE_ROOT / PROJECT_PACKAGE).rglob(_PYTHON_PATTERN)) + if include_tests: + files.extend(sorted((repo_root / PYTHON_TEST_ROOT).rglob(_PYTHON_PATTERN))) + units: list[CloneUnit] = [] + for path in files: + units.extend( + extract_python_units( + relative_path=path.relative_to(repo_root).as_posix(), + source=path.read_text(encoding="utf-8", errors="replace"), + ) + ) + return units + + +def _rust_units(*, repo_root: Path, include_tests: bool) -> list[CloneUnit]: + crates_root: Path = repo_root / RUST_CRATES_ROOT + files: list[Path] = sorted( + path + for path in crates_root.rglob(_RUST_PATTERN) + if _is_rust_candidate(path=path.relative_to(crates_root), include_tests=include_tests) + ) + units: list[CloneUnit] = [] + test_prefixes: list[str] = [] + for path in files: + extracted: RustFileUnits = extract_rust_units( + relative_path=path.relative_to(repo_root).as_posix(), + source=path.read_text(encoding="utf-8", errors="replace"), + include_tests=include_tests, + ) + units.extend(extracted.units) + test_prefixes.extend(extracted.test_module_prefixes) + prefixes: tuple[str, ...] = tuple(test_prefixes) + return [unit for unit in units if not unit.path.startswith(prefixes)] + + +def _is_rust_candidate(*, path: Path, include_tests: bool) -> bool: + parts: tuple[str, ...] = path.parts + if len(parts) <= _CRATE_DIRECTORY_INDEX: + return False + section: str = parts[_CRATE_DIRECTORY_INDEX] + if include_tests: + return section == RUST_SOURCE_DIRECTORY or section in RUST_TEST_DIRECTORIES + nested: tuple[str, ...] = parts[_CRATE_DIRECTORY_INDEX + 1 :] + return section == RUST_SOURCE_DIRECTORY and not RUST_TEST_DIRECTORIES.intersection(nested) diff --git a/scripts/dupscore/_helpers/clones/tokens.py b/scripts/dupscore/_helpers/clones/tokens.py new file mode 100644 index 0000000000..7acfeca6bf --- /dev/null +++ b/scripts/dupscore/_helpers/clones/tokens.py @@ -0,0 +1,40 @@ +"""Build clone units from lexed token streams.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Iterable + +from scripts.dupscore.models import CloneUnit + +_KEY_SEPARATOR: str = "\x1f" +_DIGEST_BYTES: int = 16 + + +def build_clone_unit( + *, + language: str, + path: str, + name: str, + start_line: int, + end_line: int, + normalized: list[str], + concrete: list[str], +) -> CloneUnit: + """Assemble a unit with identity keys for its concrete and normalised streams.""" + + return CloneUnit( + language=language, + path=path, + name=name, + start_line=start_line, + end_line=end_line, + normalized=tuple(normalized), + concrete_key=_digest(concrete), + normalized_key=_digest(normalized), + ) + + +def _digest(parts: Iterable[str]) -> str: + joined: str = _KEY_SEPARATOR.join(parts) + return hashlib.blake2b(joined.encode("utf-8"), digest_size=_DIGEST_BYTES).hexdigest() diff --git a/scripts/dupscore/_helpers/inputs/__init__.py b/scripts/dupscore/_helpers/inputs/__init__.py new file mode 100644 index 0000000000..6385831d4f --- /dev/null +++ b/scripts/dupscore/_helpers/inputs/__init__.py @@ -0,0 +1 @@ +"""Dupscore inputs helpers.""" diff --git a/scripts/dupscore/_helpers/inputs/cli.py b/scripts/dupscore/_helpers/inputs/cli.py new file mode 100644 index 0000000000..e779c6a56b --- /dev/null +++ b/scripts/dupscore/_helpers/inputs/cli.py @@ -0,0 +1,131 @@ +"""Parse and validate dupscore command-line arguments.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from scripts.dupscore.constants import ( + CLONES_MODE, + DEFAULT_CLONE_MIN_SIMILARITY, + DEFAULT_CLONE_MIN_TOKENS, + PAIR_MODE, + REPORT_MODE, + SUPPORTED_LANGUAGES, +) +from scripts.dupscore.models import CloneOptions + +_CLONE_ONLY_OPTIONS: dict[str, str] = { + "min_similarity": "--min-similarity", + "min_tokens": "--min-tokens", + "language": "--lang", + "path_globs": "--path", + "include_tests": "--include-tests", +} +_MAX_SIMILARITY: float = 1.0 +_MIN_POSITIVE_COUNT: int = 1 + + +def parse_arguments(argv: list[str] | None) -> argparse.Namespace: + """Parse CLI arguments, exiting with a usage error for invalid combinations.""" + + parser: argparse.ArgumentParser = _build_parser() + args: argparse.Namespace = parser.parse_args(argv) + problem: str | None = _argument_problem(args) + if problem is not None: + parser.error(problem) + return args + + +def clone_options_from(args: argparse.Namespace) -> CloneOptions: + """Build clone-mode options from parsed arguments and defaults.""" + + return CloneOptions( + languages=(args.language,) if args.language is not None else SUPPORTED_LANGUAGES, + include_tests=bool(args.include_tests), + min_tokens=args.min_tokens if args.min_tokens is not None else DEFAULT_CLONE_MIN_TOKENS, + min_similarity=( + args.min_similarity if args.min_similarity is not None else DEFAULT_CLONE_MIN_SIMILARITY + ), + path_globs=tuple(args.path_globs or ()), + since=args.since, + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + prog="dupscore", + description=( + "Duplication advisory tool. 'clones' (default) reports concrete function-level " + "clones in Python and Rust; 'report' ranks package pairs; 'pair' drills into one." + ), + ) + parser.add_argument( + "mode", + nargs="?", + choices=[CLONES_MODE, REPORT_MODE, PAIR_MODE], + default=CLONES_MODE, + ) + parser.add_argument("packages", nargs="*", default=[]) + parser.add_argument("--top", type=int, default=None, help="Number of entries to print.") + parser.add_argument("--domain", type=str, default=None, help="report: package prefix.") + parser.add_argument( + "--since", + type=str, + default=None, + help="clones: only clusters touching lines changed since REV; report: delta vs REV.", + ) + parser.add_argument("--json", action="store_true", dest="as_json") + parser.add_argument("--repo-root", type=Path, default=None, help="Repository to analyze.") + parser.add_argument( + "--min-similarity", + type=float, + default=None, + help=f"clones: minimum token similarity (default {DEFAULT_CLONE_MIN_SIMILARITY}).", + ) + parser.add_argument( + "--min-tokens", + type=int, + default=None, + help=f"clones: minimum normalised tokens per unit (default {DEFAULT_CLONE_MIN_TOKENS}).", + ) + parser.add_argument( + "--lang", + choices=SUPPORTED_LANGUAGES, + default=None, + dest="language", + help="clones: analyze one language only.", + ) + parser.add_argument( + "--path", + action="append", + default=None, + dest="path_globs", + help="clones: only clusters with a member matching this glob (repeatable).", + ) + parser.add_argument( + "--include-tests", + action="store_true", + default=None, + help="clones: also analyze Python tests/ and Rust test modules.", + ) + return parser + + +def _argument_problem(args: argparse.Namespace) -> str | None: + if args.mode != CLONES_MODE: + misused: list[str] = [ + flag for attribute, flag in _CLONE_ONLY_OPTIONS.items() if getattr(args, attribute) + ] + return f"{', '.join(misused)} only apply to the {CLONES_MODE} mode" if misused else None + if args.packages: + return f"the {CLONES_MODE} mode takes no positional package names" + if args.domain is not None: + return f"--domain only applies to the {REPORT_MODE} mode" + if args.top is not None and args.top < _MIN_POSITIVE_COUNT: + return "--top must be at least 1" + if args.min_tokens is not None and args.min_tokens < _MIN_POSITIVE_COUNT: + return "--min-tokens must be at least 1" + if args.min_similarity is not None and not 0.0 < args.min_similarity <= _MAX_SIMILARITY: + return "--min-similarity must be in (0, 1]" + return None diff --git a/scripts/dupscore/_helpers/config.py b/scripts/dupscore/_helpers/inputs/config.py similarity index 54% rename from scripts/dupscore/_helpers/config.py rename to scripts/dupscore/_helpers/inputs/config.py index 735470d2f3..25bce159c2 100644 --- a/scripts/dupscore/_helpers/config.py +++ b/scripts/dupscore/_helpers/inputs/config.py @@ -6,13 +6,16 @@ from pathlib import Path from scripts.dupscore.exceptions import DupscoreConfigError -from scripts.dupscore.models import DupscoreConfig +from scripts.dupscore.models import CloneAllowlistEntry, DupscoreConfig _ALLOWLIST_KEY: str = "allowlist" _SURFACES_KEY: str = "persisted_state_surfaces" _PAIR_KEY: str = "pair" _REASON_KEY: str = "reason" _PAIR_LENGTH: int = 2 +_CLONE_ALLOWLIST_KEY: str = "clone_allowlist" +_PATHS_KEY: str = "paths" +_CLONE_ALLOWLIST_KEYS: frozenset[str] = frozenset({_PATHS_KEY, _REASON_KEY}) def load_config(config_path: Path) -> DupscoreConfig: @@ -51,4 +54,38 @@ def load_config(config_path: Path) -> DupscoreConfig: return DupscoreConfig( persisted_state_surfaces=tuple(sorted(surfaces)), allowlisted_pairs=allowlisted_pairs, + clone_allowlist=_load_clone_allowlist(raw.get(_CLONE_ALLOWLIST_KEY, [])), ) + + +def _load_clone_allowlist(raw: object) -> tuple[CloneAllowlistEntry, ...]: + if not isinstance(raw, list): + raise DupscoreConfigError(f"{_CLONE_ALLOWLIST_KEY} must be an array of tables") + entries: list[CloneAllowlistEntry] = [] + for position, raw_item in enumerate(raw, start=1): + label: str = f"{_CLONE_ALLOWLIST_KEY} entry {position}" + if not isinstance(raw_item, dict): + raise DupscoreConfigError(f"{label} must be a table") + item: dict[str, object] = {str(key): value for key, value in raw_item.items()} + unknown: set[str] = set(item) - _CLONE_ALLOWLIST_KEYS + if unknown: + raise DupscoreConfigError(f"{label} has unknown keys: {', '.join(sorted(unknown))}") + paths_raw: object = item.get(_PATHS_KEY) + if ( + not isinstance(paths_raw, list) + or not paths_raw + or not all(isinstance(path, str) and path.strip() for path in paths_raw) + ): + raise DupscoreConfigError( + f"{label} needs a non-empty {_PATHS_KEY} list of non-empty glob strings" + ) + reason_raw: object = item.get(_REASON_KEY) + if not isinstance(reason_raw, str) or not reason_raw.strip(): + raise DupscoreConfigError(f"{label} needs a non-empty {_REASON_KEY}") + entries.append( + CloneAllowlistEntry( + paths=tuple(str(path) for path in paths_raw), + reason=reason_raw.strip(), + ) + ) + return tuple(entries) diff --git a/scripts/dupscore/_helpers/source_provider.py b/scripts/dupscore/_helpers/inputs/source_provider.py similarity index 90% rename from scripts/dupscore/_helpers/source_provider.py rename to scripts/dupscore/_helpers/inputs/source_provider.py index 56f7316c02..7bb5575800 100644 --- a/scripts/dupscore/_helpers/source_provider.py +++ b/scripts/dupscore/_helpers/inputs/source_provider.py @@ -57,7 +57,7 @@ def read_worktree_sources(repo_root: Path) -> dict[str, str]: def read_revision_sources(*, repo_root: Path, revision: str) -> dict[str, str]: """Read all analyzable module sources as stored at one git revision.""" - listing: str = _run_git( + listing: str = run_git( repo_root=repo_root, arguments=["ls-tree", "-r", "--name-only", revision] ) sources: dict[str, str] = {} @@ -66,17 +66,19 @@ def read_revision_sources(*, repo_root: Path, revision: str) -> dict[str, str]: if module is None: continue blob_reference: str = f"{revision}:{relative}" - sources[relative] = _run_git(repo_root=repo_root, arguments=["show", blob_reference]) + sources[relative] = run_git(repo_root=repo_root, arguments=["show", blob_reference]) return sources def resolve_default_base_revision(repo_root: Path) -> str: """Resolve the default comparison base as the merge-base with the main branch.""" - return _run_git(repo_root=repo_root, arguments=["merge-base", "main", "HEAD"]).strip() + return run_git(repo_root=repo_root, arguments=["merge-base", "main", "HEAD"]).strip() -def _run_git(*, repo_root: Path, arguments: list[str]) -> str: +def run_git(*, repo_root: Path, arguments: list[str]) -> str: + """Run one git command in the repository and return its stdout.""" + completed: subprocess.CompletedProcess[str] = subprocess.run( ["git", *arguments], cwd=repo_root, diff --git a/scripts/dupscore/_helpers/package_pairs/__init__.py b/scripts/dupscore/_helpers/package_pairs/__init__.py new file mode 100644 index 0000000000..fb4b2f7c7f --- /dev/null +++ b/scripts/dupscore/_helpers/package_pairs/__init__.py @@ -0,0 +1 @@ +"""Dupscore package pairs helpers.""" diff --git a/scripts/dupscore/_helpers/facts.py b/scripts/dupscore/_helpers/package_pairs/facts.py similarity index 99% rename from scripts/dupscore/_helpers/facts.py rename to scripts/dupscore/_helpers/package_pairs/facts.py index 5af03314cb..0ec1e87714 100644 --- a/scripts/dupscore/_helpers/facts.py +++ b/scripts/dupscore/_helpers/package_pairs/facts.py @@ -5,7 +5,7 @@ import ast import re -from scripts.dupscore._helpers.source_provider import module_name_for +from scripts.dupscore._helpers.inputs.source_provider import module_name_for from scripts.dupscore.models import ClassFact, FunctionFact, ModuleFacts, ProjectFacts _IDENTIFIER_PATTERN: re.Pattern[str] = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") diff --git a/scripts/dupscore/_helpers/fusion.py b/scripts/dupscore/_helpers/package_pairs/fusion.py similarity index 100% rename from scripts/dupscore/_helpers/fusion.py rename to scripts/dupscore/_helpers/package_pairs/fusion.py diff --git a/scripts/dupscore/_helpers/rendering.py b/scripts/dupscore/_helpers/package_pairs/rendering.py similarity index 100% rename from scripts/dupscore/_helpers/rendering.py rename to scripts/dupscore/_helpers/package_pairs/rendering.py diff --git a/scripts/dupscore/_helpers/signal_callgraph.py b/scripts/dupscore/_helpers/package_pairs/signal_callgraph.py similarity index 98% rename from scripts/dupscore/_helpers/signal_callgraph.py rename to scripts/dupscore/_helpers/package_pairs/signal_callgraph.py index b4b3f53d5f..da87a4f98e 100644 --- a/scripts/dupscore/_helpers/signal_callgraph.py +++ b/scripts/dupscore/_helpers/package_pairs/signal_callgraph.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from itertools import combinations -from scripts.dupscore._helpers.source_provider import package_of, sorted_pair +from scripts.dupscore._helpers.inputs.source_provider import package_of, sorted_pair from scripts.dupscore.constants import ( MAX_LEAF_USERS_FOR_PAIRING, MIN_REACHABLE_FUNCTIONS, diff --git a/scripts/dupscore/_helpers/signal_cochange.py b/scripts/dupscore/_helpers/package_pairs/signal_cochange.py similarity index 98% rename from scripts/dupscore/_helpers/signal_cochange.py rename to scripts/dupscore/_helpers/package_pairs/signal_cochange.py index b155745cba..55684d24c1 100644 --- a/scripts/dupscore/_helpers/signal_cochange.py +++ b/scripts/dupscore/_helpers/package_pairs/signal_cochange.py @@ -6,7 +6,7 @@ from itertools import combinations from pathlib import Path -from scripts.dupscore._helpers.source_provider import module_name_for, package_of +from scripts.dupscore._helpers.inputs.source_provider import module_name_for, package_of from scripts.dupscore.constants import ( MAX_FILES_PER_COMMIT, MIN_COCHANGES, diff --git a/scripts/dupscore/_helpers/signal_dataclasses.py b/scripts/dupscore/_helpers/package_pairs/signal_dataclasses.py similarity index 96% rename from scripts/dupscore/_helpers/signal_dataclasses.py rename to scripts/dupscore/_helpers/package_pairs/signal_dataclasses.py index 29b21fe6b3..08745f0496 100644 --- a/scripts/dupscore/_helpers/signal_dataclasses.py +++ b/scripts/dupscore/_helpers/package_pairs/signal_dataclasses.py @@ -4,7 +4,7 @@ from itertools import combinations -from scripts.dupscore._helpers.source_provider import package_of, sorted_pair +from scripts.dupscore._helpers.inputs.source_provider import package_of, sorted_pair from scripts.dupscore.constants import MIN_SHARED_DATACLASS_FIELDS, SIGNAL_NAME_DATACLASS_OVERLAP from scripts.dupscore.models import ProjectFacts, SignalPairScore, SignalRanking diff --git a/scripts/dupscore/_helpers/signal_same_names.py b/scripts/dupscore/_helpers/package_pairs/signal_same_names.py similarity index 97% rename from scripts/dupscore/_helpers/signal_same_names.py rename to scripts/dupscore/_helpers/package_pairs/signal_same_names.py index 85d9628c01..6336db2356 100644 --- a/scripts/dupscore/_helpers/signal_same_names.py +++ b/scripts/dupscore/_helpers/package_pairs/signal_same_names.py @@ -4,7 +4,7 @@ from itertools import combinations -from scripts.dupscore._helpers.source_provider import package_of, sorted_pair +from scripts.dupscore._helpers.inputs.source_provider import package_of, sorted_pair from scripts.dupscore.constants import ( GENERIC_FUNCTION_NAMES, HELPERS_ROLE_SEGMENT, diff --git a/scripts/dupscore/_helpers/signal_state_fanin.py b/scripts/dupscore/_helpers/package_pairs/signal_state_fanin.py similarity index 98% rename from scripts/dupscore/_helpers/signal_state_fanin.py rename to scripts/dupscore/_helpers/package_pairs/signal_state_fanin.py index 533fdff3bc..0ee78476c0 100644 --- a/scripts/dupscore/_helpers/signal_state_fanin.py +++ b/scripts/dupscore/_helpers/package_pairs/signal_state_fanin.py @@ -4,7 +4,7 @@ from itertools import combinations -from scripts.dupscore._helpers.source_provider import package_of +from scripts.dupscore._helpers.inputs.source_provider import package_of from scripts.dupscore.constants import ( MIN_STATE_FANIN_PACKAGES, SIGNAL_NAME_STATE_FANIN, diff --git a/scripts/dupscore/constants.py b/scripts/dupscore/constants.py index aa20c8d2d2..9c6f345e55 100644 --- a/scripts/dupscore/constants.py +++ b/scripts/dupscore/constants.py @@ -44,6 +44,7 @@ DEFAULT_TOP_RESULTS: int = 20 CONFIG_FILENAME: str = "dupscore.toml" +CLONES_MODE: str = "clones" REPORT_MODE: str = "report" PAIR_MODE: str = "pair" PAIR_ARGUMENT_COUNT: int = 2 @@ -53,3 +54,47 @@ HELPERS_ROLE_SEGMENT: str = "_helpers" MAIN_ROLE_SEGMENT: str = "main" + +LANGUAGE_PYTHON: str = "python" +LANGUAGE_RUST: str = "rust" +SUPPORTED_LANGUAGES: tuple[str, ...] = (LANGUAGE_PYTHON, LANGUAGE_RUST) +PYTHON_TEST_ROOT: str = "tests" +RUST_CRATES_ROOT: str = "crates" +RUST_SOURCE_DIRECTORY: str = "src" +RUST_TEST_DIRECTORIES: frozenset[str] = frozenset({"tests", "benches"}) + +CLONE_KGRAM_SIZE: int = 12 +CLONE_WINNOW_WINDOW: int = 6 +CLONE_MIN_FINGERPRINTS: int = 6 +CLONE_CANDIDATE_MIN_JACCARD: float = 0.2 +CLONE_MAX_FINGERPRINT_UNITS: int = 25 +DEFAULT_CLONE_MIN_TOKENS: int = 60 +DEFAULT_CLONE_MIN_SIMILARITY: float = 0.8 +DEFAULT_CLONE_TOP_RESULTS: int = 30 +CLONE_TEXT_MAX_MEMBERS: int = 12 + +CATEGORY_EXACT: str = "exact" +CATEGORY_RENAMED: str = "renamed" +CATEGORY_NEAR_MISS: str = "near-miss" +CATEGORY_ORDER: tuple[str, ...] = (CATEGORY_EXACT, CATEGORY_RENAMED, CATEGORY_NEAR_MISS) + +CHANGE_NEW: str = "new" +CHANGE_CHANGED: str = "changed" + +PLACEHOLDER_IDENTIFIER: str = "$id" +PLACEHOLDER_NUMBER: str = "$num" +PLACEHOLDER_STRING: str = "$str" +PLACEHOLDER_FSTRING: str = "$fstr" +PLACEHOLDER_BYTE_STRING: str = "$bstr" +PLACEHOLDER_CHAR: str = "$char" +PLACEHOLDER_BYTE: str = "$byte" +PLACEHOLDER_LIFETIME: str = "$life" + +RUST_KIND_IDENTIFIER: str = "identifier" +RUST_KIND_LIFETIME: str = "lifetime" +RUST_KIND_STRING: str = "string" +RUST_KIND_BYTE_STRING: str = "byte_string" +RUST_KIND_CHAR: str = "char" +RUST_KIND_BYTE: str = "byte" +RUST_KIND_NUMBER: str = "number" +RUST_KIND_PUNCTUATION: str = "punctuation" diff --git a/scripts/dupscore/dupscore.toml b/scripts/dupscore/dupscore.toml index 24cc38ec31..39c2df498e 100644 --- a/scripts/dupscore/dupscore.toml +++ b/scripts/dupscore/dupscore.toml @@ -40,3 +40,10 @@ reason = "Adapter backends intentionally conform to the shared adapter contract. [[allowlist]] pair = ["sqlbuild.adapter.contract", "sqlbuild.adapters.sqlserver"] reason = "Adapter backends intentionally conform to the shared adapter contract." + +# Clone mode: suppress clone pairs whose two sides both match one entry's globs. +# `*` also crosses directory separators. Every entry needs a non-empty reason. +# +# [[clone_allowlist]] +# paths = ["src/sqlbuild/example/backends/*"] +# reason = "Backends intentionally mirror one protocol." diff --git a/scripts/dupscore/main/build_clone_report.py b/scripts/dupscore/main/build_clone_report.py new file mode 100644 index 0000000000..34d5ab7447 --- /dev/null +++ b/scripts/dupscore/main/build_clone_report.py @@ -0,0 +1,94 @@ +"""Build the function-level clone report for the worktree.""" + +from __future__ import annotations + +from collections import Counter +from pathlib import Path + +from scripts.dupscore._helpers.clones.clustering import cluster_clone_pairs +from scripts.dupscore._helpers.clones.filters import ( + allowlist_reason, + classify_unit_change, + collect_changes_since, + path_matches_any, +) +from scripts.dupscore._helpers.clones.fingerprints import fingerprint_tokens +from scripts.dupscore._helpers.clones.matching import find_clone_pairs +from scripts.dupscore._helpers.clones.sources import collect_clone_units +from scripts.dupscore.models import ( + CloneCluster, + CloneOptions, + ClonePair, + CloneReport, + CloneUnit, + DupscoreConfig, + FileChanges, +) + + +def build_clone_report( + *, + repo_root: Path, + options: CloneOptions, + config: DupscoreConfig, +) -> CloneReport: + """Detect exact, renamed, and near-miss function clones and group them into clusters.""" + + units: list[CloneUnit] = [ + unit + for unit in collect_clone_units( + repo_root=repo_root, + languages=options.languages, + include_tests=options.include_tests, + ) + if len(unit.normalized) >= options.min_tokens + ] + fingerprints: list[frozenset[int]] = [fingerprint_tokens(unit.normalized) for unit in units] + candidate_pairs: list[ClonePair] = find_clone_pairs( + units=units, + fingerprints=fingerprints, + min_similarity=options.min_similarity, + ) + pairs: list[ClonePair] = [ + pair + for pair in candidate_pairs + if allowlist_reason( + left_path=units[pair.left].path, + right_path=units[pair.right].path, + entries=config.clone_allowlist, + ) + is None + ] + changes: dict[str, FileChanges] | None = ( + collect_changes_since(repo_root=repo_root, revision=options.since) + if options.since is not None + else None + ) + + def change_of(unit: CloneUnit) -> str | None: + if changes is None: + return None + return classify_unit_change(unit=unit, changes=changes.get(unit.path)) + + clusters: list[CloneCluster] = [ + cluster + for cluster in cluster_clone_pairs(units=units, pairs=pairs, change_of=change_of) + if _cluster_selected(cluster=cluster, options=options) + ] + unit_counts: Counter[str] = Counter(unit.language for unit in units) + return CloneReport( + since=options.since, + unit_counts={language: unit_counts[language] for language in options.languages}, + allowlisted_pairs=len(candidate_pairs) - len(pairs), + clusters=tuple(clusters), + ) + + +def _cluster_selected(*, cluster: CloneCluster, options: CloneOptions) -> bool: + if options.since is not None and all(member.change is None for member in cluster.members): + return False + if options.path_globs and not any( + path_matches_any(path=member.path, globs=options.path_globs) for member in cluster.members + ): + return False + return True diff --git a/scripts/dupscore/main/build_report.py b/scripts/dupscore/main/build_report.py index c685c18a37..7232ba4fe5 100644 --- a/scripts/dupscore/main/build_report.py +++ b/scripts/dupscore/main/build_report.py @@ -4,14 +4,17 @@ from pathlib import Path -from scripts.dupscore._helpers.facts import extract_project_facts -from scripts.dupscore._helpers.fusion import fuse_rankings -from scripts.dupscore._helpers.signal_callgraph import score_callgraph_shape -from scripts.dupscore._helpers.signal_cochange import score_cochange -from scripts.dupscore._helpers.signal_dataclasses import score_dataclass_overlap -from scripts.dupscore._helpers.signal_same_names import score_same_name_symbols -from scripts.dupscore._helpers.signal_state_fanin import score_state_fanin -from scripts.dupscore._helpers.source_provider import read_revision_sources, read_worktree_sources +from scripts.dupscore._helpers.inputs.source_provider import ( + read_revision_sources, + read_worktree_sources, +) +from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts +from scripts.dupscore._helpers.package_pairs.fusion import fuse_rankings +from scripts.dupscore._helpers.package_pairs.signal_callgraph import score_callgraph_shape +from scripts.dupscore._helpers.package_pairs.signal_cochange import score_cochange +from scripts.dupscore._helpers.package_pairs.signal_dataclasses import score_dataclass_overlap +from scripts.dupscore._helpers.package_pairs.signal_same_names import score_same_name_symbols +from scripts.dupscore._helpers.package_pairs.signal_state_fanin import score_state_fanin from scripts.dupscore.constants import HEAD_REVISION, WORKTREE_LABEL from scripts.dupscore.models import DupscoreConfig, DupscoreReport, ProjectFacts, SignalRanking diff --git a/scripts/dupscore/main/dupscore.py b/scripts/dupscore/main/dupscore.py index a9e86a34f8..956c943c7a 100644 --- a/scripts/dupscore/main/dupscore.py +++ b/scripts/dupscore/main/dupscore.py @@ -1,13 +1,17 @@ -"""CLI entry for the dupscore duplication-risk advisory report.""" +"""CLI entry for the dupscore duplication advisory tool (see scripts/dupscore/README.md).""" from __future__ import annotations import argparse +import sys +import time from pathlib import Path -from scripts.dupscore._helpers.config import load_config -from scripts.dupscore._helpers.fusion import filter_report_to_domain -from scripts.dupscore._helpers.rendering import ( +from scripts.dupscore._helpers.clones.rendering import render_clone_json, render_clone_text +from scripts.dupscore._helpers.inputs.cli import clone_options_from, parse_arguments +from scripts.dupscore._helpers.inputs.config import load_config +from scripts.dupscore._helpers.package_pairs.fusion import filter_report_to_domain +from scripts.dupscore._helpers.package_pairs.rendering import ( render_delta_json, render_delta_text, render_pair_json, @@ -16,17 +20,20 @@ render_report_text, ) from scripts.dupscore.constants import ( + CLONES_MODE, CONFIG_FILENAME, + DEFAULT_CLONE_TOP_RESULTS, DEFAULT_TOP_RESULTS, PAIR_ARGUMENT_COUNT, PAIR_MODE, - REPORT_MODE, ) from scripts.dupscore.exceptions import DupscoreUsageError +from scripts.dupscore.main.build_clone_report import build_clone_report from scripts.dupscore.main.build_pair_evidence import build_pair_evidence from scripts.dupscore.main.build_report import build_report from scripts.dupscore.main.build_report_delta import build_report_delta from scripts.dupscore.models import ( + CloneReport, DupscoreConfig, DupscoreReport, PairEvidenceReport, @@ -37,28 +44,25 @@ def dupscore(argv: list[str] | None = None) -> int: """Run the dupscore CLI and print the requested advisory output.""" - parser: argparse.ArgumentParser = _build_parser() - args: argparse.Namespace = parser.parse_args(argv) + args: argparse.Namespace = parse_arguments(argv) config_path: Path = Path(__file__).resolve().parents[1] / CONFIG_FILENAME config: DupscoreConfig = load_config(config_path) - output: str = _execute(args=args, config=config) + repo_root: Path = ( + args.repo_root.resolve() + if args.repo_root is not None + else Path(__file__).resolve().parents[3] + ) + output: str = ( + _execute_clones(args=args, config=config, repo_root=repo_root) + if args.mode == CLONES_MODE + else _execute(args=args, config=config, repo_root=repo_root) + ) print(output) return 0 -def _build_parser() -> argparse.ArgumentParser: - parser: argparse.ArgumentParser = argparse.ArgumentParser(prog="dupscore") - parser.add_argument("mode", nargs="?", choices=[REPORT_MODE, PAIR_MODE], default=REPORT_MODE) - parser.add_argument("packages", nargs="*", default=[]) - parser.add_argument("--top", type=int, default=DEFAULT_TOP_RESULTS) - parser.add_argument("--domain", type=str, default=None) - parser.add_argument("--since", type=str, default=None) - parser.add_argument("--json", action="store_true", dest="as_json") - return parser - - -def _execute(*, args: argparse.Namespace, config: DupscoreConfig) -> str: - repo_root: Path = Path(__file__).resolve().parents[3] +def _execute(*, args: argparse.Namespace, config: DupscoreConfig, repo_root: Path) -> str: + top: int = args.top if args.top is not None else DEFAULT_TOP_RESULTS current: DupscoreReport = build_report(repo_root=repo_root, revision=None, config=config) if args.mode == PAIR_MODE: if len(args.packages) != PAIR_ARGUMENT_COUNT: @@ -71,11 +75,33 @@ def _execute(*, args: argparse.Namespace, config: DupscoreConfig) -> str: return render_pair_json(evidence) if args.as_json else render_pair_text(evidence) if args.since is not None: base: DupscoreReport = build_report(repo_root=repo_root, revision=args.since, config=config) - delta: ReportDelta = build_report_delta(base=base, current=current, top=args.top) + delta: ReportDelta = build_report_delta(base=base, current=current, top=top) return render_delta_json(delta) if args.as_json else render_delta_text(delta) filtered: DupscoreReport = filter_report_to_domain(report=current, domain=args.domain) return ( - render_report_json(report=filtered, top=args.top) + render_report_json(report=filtered, top=top) if args.as_json - else render_report_text(report=filtered, top=args.top) + else render_report_text(report=filtered, top=top) + ) + + +def _execute_clones(*, args: argparse.Namespace, config: DupscoreConfig, repo_root: Path) -> str: + top: int = args.top if args.top is not None else DEFAULT_CLONE_TOP_RESULTS + print(f"dupscore: detecting function clones in {repo_root} ...", file=sys.stderr) + started: float = time.perf_counter() + try: + report: CloneReport = build_clone_report( + repo_root=repo_root, options=clone_options_from(args), config=config + ) + except Exception: + print("dupscore: clone detection failed", file=sys.stderr) + raise + unit_total: int = sum(report.unit_counts.values()) + print( + f"dupscore: found {len(report.clusters)} clone clusters across {unit_total} units " + f"in {time.perf_counter() - started:.1f}s", + file=sys.stderr, ) + if args.as_json: + return render_clone_json(report=report, top=top) + return render_clone_text(report=report, top=top) diff --git a/scripts/dupscore/models.py b/scripts/dupscore/models.py index 505fbc7e66..a086b39c6a 100644 --- a/scripts/dupscore/models.py +++ b/scripts/dupscore/models.py @@ -119,9 +119,125 @@ class ReportDelta: new_same_name_evidence: tuple[str, ...] +@dataclass(frozen=True, slots=True) +class CloneAllowlistEntry: + """Path globs whose mutual clones are intentional, with the recorded reason.""" + + paths: tuple[str, ...] + reason: str + + @dataclass(frozen=True, slots=True) class DupscoreConfig: """User configuration loaded from dupscore.toml.""" persisted_state_surfaces: tuple[str, ...] = () allowlisted_pairs: dict[tuple[str, str], str] = field(default_factory=dict) + clone_allowlist: tuple[CloneAllowlistEntry, ...] = () + + +@dataclass(frozen=True, slots=True) +class RustToken: + """One lexed Rust token with its kind, spelling, and 1-based line.""" + + kind: str + text: str + line: int + + +@dataclass(frozen=True, slots=True) +class CloneUnit: + """One function-level unit with its normalised token stream.""" + + language: str + path: str + name: str + start_line: int + end_line: int + normalized: tuple[str, ...] + concrete_key: str + normalized_key: str + + +@dataclass(frozen=True, slots=True) +class ClonePair: + """Two units whose fingerprints overlap above the similarity threshold.""" + + left: int + right: int + similarity: float + category: str + + +@dataclass(frozen=True, slots=True) +class CloneMember: + """One unit participating in a reported clone cluster.""" + + language: str + path: str + name: str + start_line: int + end_line: int + tokens: int + change: str | None + + +@dataclass(frozen=True, slots=True) +class ClonePairLink: + """One similarity edge between two members of a cluster, by member index.""" + + left: int + right: int + similarity: float + category: str + + +@dataclass(frozen=True, slots=True) +class CloneCluster: + """A transitive group of mutually similar units.""" + + category: str + similarity_min: float + similarity_max: float + duplicated_tokens: int + members: tuple[CloneMember, ...] + links: tuple[ClonePairLink, ...] + + +@dataclass(frozen=True, slots=True) +class CloneOptions: + """Detection and reporting options for the clones mode.""" + + languages: tuple[str, ...] + include_tests: bool + min_tokens: int + min_similarity: float + path_globs: tuple[str, ...] + since: str | None + + +@dataclass(frozen=True, slots=True) +class CloneReport: + """Every reported clone cluster for the worktree.""" + + since: str | None + unit_counts: dict[str, int] + allowlisted_pairs: int + clusters: tuple[CloneCluster, ...] + + +@dataclass(frozen=True, slots=True) +class FileChanges: + """Lines of one file added or changed since a base revision.""" + + added_ranges: tuple[tuple[int, int], ...] = () + deletion_points: tuple[int, ...] = () + whole_file: bool = False + + +@dataclass(frozen=True, slots=True) +class RustFileUnits: + """Units found in one Rust file plus the test-only module files it declares.""" + + units: tuple[CloneUnit, ...] + test_module_prefixes: tuple[str, ...] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index dcc7473a26..55033028ee 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -118,7 +118,9 @@ window into batches, each with its own audits. Always `sqb plan` an incremental `__expected__` (only the listed columns are compared, matched by name; unlisted columns are ignored), adds zero-row checks with `__assert__`, and ends with `SELECT 1`. One test can span many models: mock the sources, assert on the final model, and every intermediate -model runs from its real SQL. Read [references/testing.md](references/testing.md) for fixtures, +model runs from its real SQL. Never write a test that only proves empty inputs produce no rows; it +cannot fail, Rules reject it, and it does not count toward `min_tests_per_model`. Mock real rows +and assert concrete output. Read [references/testing.md](references/testing.md) for fixtures, parameterised cases, scenarios, audits and Rules. **Audits** validate built data and block bad data: `error` audits run on staging data before the diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md index 644c78e5c5..dc322397d2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/incremental-models.md @@ -58,7 +58,10 @@ Behaviour to rely on: cannot run ahead of a slow one. - `__cursor_start()` (inclusive) and `__cursor_end()` (exclusive) expose the effective bounds when SQL needs them explicitly, for example to filter an unlisted input or compute a derived range. - They are valid only in cursor incremental model SQL, not in tests, hooks, audits or functions. + They are valid only in cursor incremental model SQL, not in test, hook, audit or function SQL. + SQL unit tests of such a model render them with a wide default window, or the window declared + with `cursor_start`/`cursor_end` in the `TEST(...)` header; see + [testing.md](testing.md#cursor-windows-in-tests). - For `delete_insert` and `merge`, only target rows whose cursor falls inside the window are rewritten. A change in an unlisted input does not rewrite older rows. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/testing.md index 5a33b9856d..af77e18f97 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/testing.md @@ -10,6 +10,8 @@ ## Contents - Writing a unit test +- Empty-input tests +- Cursor windows in tests - Multi-model tests - Fixtures that fail early - Repeated cases @@ -50,17 +52,69 @@ SELECT 1 | `__macro__` | Replace every `@(...)` call | | `__expected__` | Expected rows of the model's real SQL, compared both ways on the listed columns only | | `__assert__` | Passes when the query returns zero rows | -| anything else | Helper CTE visible to mocks and model SQL | +| anything else | Helper CTE visible to mocks, `__expected__` and `__assert__` SQL | An `__expected__` CTE compares only the columns it lists, matched by name, so column order does not matter and unlisted model columns are ignored. Row counts are always compared. Listing a column the model does not output is a clear test error. When the expected CTE's columns are not explicit (for example `SELECT *` from another CTE), SQLBuild compares every column by position. +Helper CTEs can read other helpers and mocks by CTE name (for example `__ref__stg_orders`), so +shared expected rows can live in one helper that both an `__expected__` CTE and an assertion use. + Macros work inside tests, so reusable mock generators such as `@mock_orders(count=5)` are normal. Tests can also target a macro, UDF or table function directly; see [docs/concepts/testing.md](docs/concepts/testing.md) for those modes. +## Empty-input tests + +Do not write a test whose mocks are all empty (`WHERE FALSE`, `WHERE 1 = 0`, `LIMIT 0` or +`__EMPTY_FIXTURE()`) and whose only checks are an empty `__expected__` or a bare +`SELECT ... FROM __ref("")` assertion. It cannot fail for a model whose rows come from its +inputs. Rule `SQBRTEST203` rejects it, and `SQBRTEST202` does not count it toward +`min_tests_per_model`, so it never satisfies the minimum. Mock representative rows and assert +concrete output instead. + +An empty-input test is legitimate only when it asserts concrete output, for example a global +aggregate that must return one zero-valued summary row: + +```sql +TEST (name "order_summary__empty_inputs_return_zero_row"); + +WITH +__source__raw__orders AS ( + SELECT CAST(NULL AS INTEGER) AS id, CAST(NULL AS INTEGER) AS amount WHERE FALSE +), +__expected__order_summary AS ( + SELECT 0 AS order_count, 0 AS total_amount +) +SELECT 1 +``` + +Reviewed exceptions go in `sqlbuild_project.toml` under `[rules.rule_options.SQBRTEST203]` as +`allowed_tests = [""]`; they count toward the minimum, and stale entries are reported. +Do not add entries to get past the rule. + +## Cursor windows in tests + +Models that call `__cursor_start()` / `__cursor_end()` run in a test over one wide window, never +the model's build-time `cursor_start`/`cursor_end`: `1900-01-01 00:00:00` to `2999-12-31 00:00:00` +for timestamp cursors and `-1000000000000000` to `1000000000000000` for integer cursors. The +bounds render as the same typed literals as a real run, so date arithmetic such as +`DATEADD('day', -7, __cursor_start()::DATE)` keeps working and mocked rows are not filtered out. +Microbatch models run once over the whole window, not per batch. + +To test windowing itself, declare the window in the header. Either key is optional; as in model +config, `cursor_start` is inclusive and `cursor_end` is exclusive: + +```sql +TEST (name "daily_orders_skips_rows_outside_window", cursor_start "2026-02-01", cursor_end "2026-02-03"); +``` + +The window applies to every model the test evaluates that uses the intrinsics. Values are checked +against each model's `cursor_type` and `cursor_grain` at compile time; invalid, misaligned or +inverted bounds, or a window on a test whose models do not use the intrinsics, are compile errors. + ## Multi-model tests Mock only the sources, give `__expected__` for the model you care about, and every intermediate diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index e8268fb2bf..18556cfe0e 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -17,7 +17,10 @@ elapsed_ms, start_compile_phase, ) -from sqlbuild.cli.commands._helpers.compile.target_writer import write_static_compile_target +from sqlbuild.cli.commands._helpers.compile.target_writer import ( + static_sql_test_planning_diagnostics, + write_static_compile_target, +) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.types import CompileLineageMode @@ -29,6 +32,7 @@ from sqlbuild.cli.output.models import ( WrittenTarget, ) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, CompiledObjectKey, @@ -44,6 +48,7 @@ build_project_graph_with_analysis_selection, ) from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.rule_engine.classes.early_sql_lint import EarlySqlLint @@ -313,6 +318,36 @@ def write_compile_dag_artifact( ) +def compile_sql_test_planning_diagnostics( + *, + project_dir: Path, + analysis: CompileAnalysis, + prepared_artifacts: PreparedCompileArtifacts | None, +) -> tuple[CompilerDiagnostic, ...]: + """Report SQL test planning errors when artifacts are withheld, reusing staged planning.""" + + try: + staged: tuple[CompilerDiagnostic, ...] | None = ( + prepared_artifacts.planning_diagnostics() if prepared_artifacts is not None else None + ) + if staged is not None: + return staged + return static_sql_test_planning_diagnostics( + target_dir=project_dir / "target", + adapter=analysis.adapter, + project=analysis.graph.project, + ) + except (CompileInputError, PlannerInputError) as error: + return ( + CompilerDiagnostic( + phase=DiagnosticPhase.TEST, + severity=DiagnosticSeverity.ERROR, + code=error.code, + message=str(error), + ), + ) + + def write_compile_artifacts( *, profile_skip_write: bool, diff --git a/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py b/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py index c531648972..c34c588311 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py @@ -206,6 +206,10 @@ def _model_identity(*, model: CompiledModel) -> str: "name": model.name, "query_sql": model.query_sql, "deps": sorted((str(dep.resource_type), dep.name) for dep in model.deps), + "cursor": ( + model.config.values.get("cursor_type"), + model.config.values.get("cursor_grain"), + ), } ) @@ -224,6 +228,7 @@ def _test_payload(*, test: CompiledSqlTest) -> dict[str, object]: "model_query_overrides": sorted(test.payload.model_query_overrides.items()), "expected_model_names": test.payload.expected_model_names, "assertion_ctes": _cte_payload(test.payload.assertion_ctes), + "cursor_window": (test.test_block.cursor_start, test.test_block.cursor_end), } elif isinstance(test.payload, CompiledDirectLogicSqlTestPayload): payload["direct"] = { diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index 1fd649e2f3..bf74b5c7f2 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -361,18 +361,34 @@ def _write_tests( return managed_paths -def _write_static_tests( - *, - target_dir: Path, - adapter: BaseAdapter, - project: CompiledProject, - check_existing: bool, -) -> tuple[set[Path], tuple[CompilerDiagnostic, ...]]: - """Write offline SQL-native test SQL and report uncached planning errors.""" +def static_sql_test_planning_diagnostics( + *, target_dir: Path, adapter: BaseAdapter, project: CompiledProject +) -> tuple[CompilerDiagnostic, ...]: + """Plan uncached SQL tests and report their errors without writing artifacts or cache.""" - managed_paths: set[Path] = set() + pending, _, _ = _partition_cached_static_tests( + tests_root=target_dir / _COMPILED_DIR / _TESTS_DIR, adapter=adapter, project=project + ) + tests: tuple[CompiledSqlTest, ...] = tuple(test for test, _, _ in pending) + artifacts: tuple[NativeSqlTestArtifact, ...] = _plan_static_test_artifacts( + adapter=adapter, project=project, tests=tests + ) diagnostics: list[CompilerDiagnostic] = [] - tests_root: Path = target_dir / _COMPILED_DIR / _TESTS_DIR + for test, artifact in zip(tests, artifacts, strict=True): + diagnostics.extend(_sql_test_artifact_diagnostics(test=test, artifact=artifact)) + return tuple(diagnostics) + + +def _partition_cached_static_tests( + *, tests_root: Path, adapter: BaseAdapter, project: CompiledProject +) -> tuple[ + list[tuple[CompiledSqlTest, str | None, str | None]], + set[Path], + dict[str, SqlTestArtifactCacheRecord], +]: + """Split SQL tests into pending work and reusable cached artifacts.""" + + managed_paths: set[Path] = set() cached_records: dict[str, SqlTestArtifactCacheRecord] = read_sql_test_artifact_cache( cache_dir=project.compile_cache_dir ) @@ -411,19 +427,44 @@ def _write_static_tests( current_records[record_key] = cached_record continue pending.append((test, record_key, artifact_identity)) + return pending, managed_paths, current_records + + +def _plan_static_test_artifacts( + *, adapter: BaseAdapter, project: CompiledProject, tests: tuple[CompiledSqlTest, ...] +) -> tuple[NativeSqlTestArtifact, ...]: + if not tests: + return () + with OperationLifecycle( + operation_kind="project", operation_name="sql_test_planning" + ) as lifecycle: + native_artifacts: tuple[NativeSqlTestArtifact, ...] = plan_and_render_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + ) + lifecycle.completed(metadata={"item_count": len(native_artifacts)}) + return native_artifacts - native_artifacts: tuple[NativeSqlTestArtifact, ...] = () - if pending: - with OperationLifecycle( - operation_kind="project", operation_name="sql_test_planning" - ) as lifecycle: - native_artifacts = plan_and_render_sql_test_artifacts( - project=project, - tests=tuple(test for test, _, _ in pending), - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - ) - lifecycle.completed(metadata={"item_count": len(native_artifacts)}) + +def _write_static_tests( + *, + target_dir: Path, + adapter: BaseAdapter, + project: CompiledProject, + check_existing: bool, +) -> tuple[set[Path], tuple[CompilerDiagnostic, ...]]: + """Write offline SQL-native test SQL and report uncached planning errors.""" + + diagnostics: list[CompilerDiagnostic] = [] + tests_root: Path = target_dir / _COMPILED_DIR / _TESTS_DIR + pending, managed_paths, current_records = _partition_cached_static_tests( + tests_root=tests_root, adapter=adapter, project=project + ) + native_artifacts: tuple[NativeSqlTestArtifact, ...] = _plan_static_test_artifacts( + adapter=adapter, project=project, tests=tuple(test for test, _, _ in pending) + ) for (test, record_key, artifact_identity), artifact in zip( pending, native_artifacts, strict=True ): @@ -433,11 +474,11 @@ def _write_static_tests( ) _write_sql(path=test_path, sql=artifact.sql, check_existing=check_existing) managed_paths.add(test_path) - if artifact.error_messages: - diagnostics.extend( - _sql_test_error_diagnostic(test=test, message=message) - for message in artifact.error_messages - ) + artifact_diagnostics: tuple[CompilerDiagnostic, ...] = _sql_test_artifact_diagnostics( + test=test, artifact=artifact + ) + if artifact_diagnostics: + diagnostics.extend(artifact_diagnostics) continue if record_key is not None and artifact_identity is not None: record: SqlTestArtifactCacheRecord | None = build_sql_test_artifact_cache_record( @@ -455,15 +496,20 @@ def _write_static_tests( return managed_paths, tuple(diagnostics) -def _sql_test_error_diagnostic(*, test: CompiledSqlTest, message: str) -> CompilerDiagnostic: - return CompilerDiagnostic( - phase=DiagnosticPhase.TEST, - severity=DiagnosticSeverity.ERROR, - code=_SQL_TEST_PLANNING_ERROR_CODE, - message=message, - resource_type=CompiledResourceType.SQL_TEST, - resource_name=test.name, - path=test.source_path, +def _sql_test_artifact_diagnostics( + *, test: CompiledSqlTest, artifact: NativeSqlTestArtifact +) -> tuple[CompilerDiagnostic, ...]: + return tuple( + CompilerDiagnostic( + phase=DiagnosticPhase.TEST, + severity=DiagnosticSeverity.ERROR, + code=_SQL_TEST_PLANNING_ERROR_CODE, + message=message, + resource_type=CompiledResourceType.SQL_TEST, + resource_name=test.name, + path=test.source_path, + ) + for message in artifact.error_messages ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/observability.py b/src/sqlbuild/cli/commands/_helpers/entry/observability.py index 2fdfb1081e..8e6c05f867 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/observability.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/observability.py @@ -7,6 +7,7 @@ from collections.abc import Callable, Iterator from contextlib import ExitStack, contextmanager from contextvars import Token +from functools import partial from pathlib import Path from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace @@ -30,15 +31,21 @@ DiscoveredCommandOutputSink, DiscoveredEventExporter, DiscoveredProvider, + DiscoveredRuntimeExtensions, ) from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope from sqlbuild.runtime.event_exporting.classes.dispatcher import EventExporterDispatcher +from sqlbuild.runtime.event_exporting.constants import ( + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, + INVOCATION_TERMINAL_EVENT_TYPES, +) from sqlbuild.runtime.event_exporting.main.event_exporter_command_scope import ( event_exporter_command_scope, ) from sqlbuild.runtime.event_exporting.models import ( + EventExporterAccounting, EventExporterCounts, EventExporterFailure, EventExportSummary, @@ -48,10 +55,16 @@ current_execution_identity, ) from sqlbuild.runtime.observability.main.dispatcher_scope import dispatcher_scope -from sqlbuild.runtime.observability.models import DispatchFailure, ExecutionIdentity +from sqlbuild.runtime.observability.models import ( + DispatchFailure, + ExecutionIdentity, + LifecycleEvent, +) from sqlbuild.runtime.observability.types import Unsubscribe _LOGGER: logging.Logger = logging.getLogger("sqlbuild.cli.observability") +_INVOCATION_COMPLETED_EVENT_TYPE: str = "invocation_completed" +_SHUTDOWN_TIMEOUT_SETTING: str = "sinks.lifecycle.shutdown_timeout" @contextmanager @@ -85,18 +98,27 @@ def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterato projector_token: Token[NativeProgressProjector | None] = projector.install() exporter_scope: EventExporterCommandScope | None = None unsubscribe_exporters: Unsubscribe | None = None + unsubscribe_invocation_outcome: Unsubscribe | None = None + invocation_outcome: _InvocationOutcome = _InvocationOutcome() + final_export_summary: EventExportSummary | None = None try: - providers: tuple[DiscoveredProvider, ...] - event_exporters: tuple[DiscoveredEventExporter, ...] - command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] - if args.command == LINEAGE_COMMAND: - providers, event_exporters, command_output_sinks = (), (), () - else: - providers, event_exporters, command_output_sinks = discover_runtime_extensions( - project_dir=project_dir - ) + extensions: DiscoveredRuntimeExtensions = ( + DiscoveredRuntimeExtensions() + if args.command == LINEAGE_COMMAND + else discover_runtime_extensions(project_dir=project_dir) + ) + providers: tuple[DiscoveredProvider, ...] = extensions.providers + event_exporters: tuple[DiscoveredEventExporter, ...] = extensions.event_exporters + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = ( + extensions.command_output_sinks + ) if event_exporters or command_output_sinks: exporter_delivery: EventExporterDispatcher = EventExporterDispatcher( + shutdown_timeout_seconds=( + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS + if extensions.lifecycle_shutdown_timeout_seconds is None + else float(extensions.lifecycle_shutdown_timeout_seconds) + ), failure_callback=_log_exporter_failure, summary_callback=_log_exporter_summary, ) @@ -111,6 +133,10 @@ def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterato subscriber=exporter_delivery.enqueue, accepts_opaque=False, ) + unsubscribe_invocation_outcome = dispatcher.subscribe_lifecycle( + subscriber=invocation_outcome.consume, + accepts_opaque=False, + ) with ExitStack() as stack: _ = stack.enter_context(dispatcher_scope(dispatcher)) _ = stack.enter_context(terminal_event_index_scope(terminal_index)) @@ -129,8 +155,12 @@ def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterato finally: if unsubscribe_exporters is not None: _run_cleanup(action=unsubscribe_exporters, phase="event_exporter_unsubscribe") + if unsubscribe_invocation_outcome is not None: + _run_cleanup( + action=unsubscribe_invocation_outcome, phase="invocation_outcome_unsubscribe" + ) if exporter_scope is not None: - _run_cleanup(action=exporter_scope.close, phase="event_exporter_shutdown") + final_export_summary = _close_exporter_scope(exporter_scope) _run_cleanup(action=unsubscribe_terminal_index, phase="terminal_index_unsubscribe") _run_cleanup(action=unsubscribe_progress, phase="progress_unsubscribe") _run_cleanup(action=projector.close, phase="progress_close") @@ -138,6 +168,69 @@ def cli_observability_scope(*, args: CliNamespace, project_dir: Path) -> Iterato action=lambda: projector.restore(projector_token), phase="progress_context_restore", ) + if final_export_summary is not None: + _run_cleanup( + action=partial( + _warn_incomplete_export, + summary=final_export_summary, + command_succeeded=invocation_outcome.succeeded, + ), + phase="event_exporter_warning", + ) + + +def format_event_export_warning( + *, summary: EventExportSummary, command_succeeded: bool +) -> str | None: + """Return one stderr warning line when lifecycle events were dropped or failed.""" + + if summary.dropped == 0 and summary.failed == 0: + return None + affected: tuple[EventExporterAccounting, ...] = tuple( + exporter + for exporter in summary.per_exporter + if exporter.counts.dropped > 0 or exporter.counts.failed > 0 + ) + names: str = ", ".join(f"'{exporter.exporter_name}'" for exporter in affected) + sink_label: str = "sink" if len(affected) == 1 else "sinks" + unit: str = "events" if len(summary.per_exporter) == 1 else "event deliveries" + prefix: str = ( + "Warning: command completed successfully, but lifecycle event export was incomplete" + if command_succeeded + else "Warning: lifecycle event export was incomplete" + ) + return ( + f"{prefix}: {summary.dropped} of {summary.accepted} {unit} dropped, " + f"{summary.failed} failed ({sink_label} {names}). " + f"Increase {_SHUTDOWN_TIMEOUT_SETTING} or check sink health." + ) + + +class _InvocationOutcome: + """Record whether the invocation published a successful terminal fact.""" + + def __init__(self) -> None: + self.succeeded: bool = False + + def consume(self, event: LifecycleEvent) -> None: + if event.event_type in INVOCATION_TERMINAL_EVENT_TYPES: + self.succeeded = event.event_type == _INVOCATION_COMPLETED_EVENT_TYPE + + +def _warn_incomplete_export(*, summary: EventExportSummary, command_succeeded: bool) -> None: + message: str | None = format_event_export_warning( + summary=summary, command_succeeded=command_succeeded + ) + if message is not None: + print(message, file=sys.stderr, flush=True) + + +def _close_exporter_scope(scope: EventExporterCommandScope) -> EventExportSummary | None: + try: + return scope.close() + except BaseException as error: + _report_cleanup_failure(error=error, phase="event_exporter_shutdown") + return None def _run_cleanup(*, action: Callable[[], object], phase: str) -> None: diff --git a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py index 9e53343942..0dd6f0f036 100644 --- a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py +++ b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py @@ -14,7 +14,7 @@ write_static_compile_target, ) from sqlbuild.cli.output.models import WrittenTarget -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic _MIN_PREPARED_ARTIFACT_MODELS: int = 128 _MAX_PREPARED_ARTIFACT_MODELS: int = 5000 @@ -83,3 +83,13 @@ def publish( return publish_static_compile_target( prepared=prepared, target_dir=target_dir, manifest=manifest ) + + def planning_diagnostics(self) -> tuple[CompilerDiagnostic, ...] | None: + """Return staged SQL test planning diagnostics without publishing any artifact.""" + + if self._future is None: + return None + try: + return self._future.result().diagnostics + except OSError: + return None diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index ac87980508..049201aa3f 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -13,6 +13,7 @@ from sqlbuild.cli.commands._helpers.compile.pipeline import ( analyze_compile_project, build_compile_manifest_payload, + compile_sql_test_planning_diagnostics, write_compile_artifacts, write_compile_dag_artifact, ) @@ -126,9 +127,17 @@ def _run_compile_with_status( **detailed_timings.as_milliseconds(), "total_ms": elapsed_ms(total_start), } + withheld_test_diagnostics: tuple[CompilerDiagnostic, ...] = ( + compile_sql_test_planning_diagnostics( + project_dir=project_dir, analysis=analysis, prepared_artifacts=prepared_artifacts + ) + if rules_failed and not request.profile_flags.skip_write + else () + ) diagnostics: tuple[CompilerDiagnostic, ...] = ( *analysis.diagnostics, *write_result.written.diagnostics, + *withheld_test_diagnostics, ) exit_code: int = 1 if any(diagnostic.is_error for diagnostic in diagnostics) else 0 diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py index 8f37fff5f4..a99deb0368 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py +++ b/src/sqlbuild/compiler/discovery/_helpers/sql/tests.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from datetime import date, datetime from inspect import cleandoc from pathlib import Path from typing import cast @@ -36,6 +37,8 @@ _TEST_MODE_HEADER_KEY: str = "mode" _TEST_PARAMETERS_HEADER_KEY: str = "parameters" _TEST_CASES_HEADER_KEY: str = "cases" +_TEST_CURSOR_START_HEADER_KEY: str = "cursor_start" +_TEST_CURSOR_END_HEADER_KEY: str = "cursor_end" _PARAMETER_TYPES: tuple[SqlValueKind, ...] = ( SqlValueKind.STRING, SqlValueKind.INTEGER, @@ -135,6 +138,17 @@ def _parse_single_sql_test_block( header_values=header_values, file_path=file_path, ) + cursor_start: str | None = _parse_test_cursor_bound( + header_values=header_values, key=_TEST_CURSOR_START_HEADER_KEY, file_path=file_path + ) + cursor_end: str | None = _parse_test_cursor_bound( + header_values=header_values, key=_TEST_CURSOR_END_HEADER_KEY, file_path=file_path + ) + if (cursor_start is not None or cursor_end is not None) and test_mode != SqlTestMode.MODEL: + raise SqlTestParseError( + f"TEST() in '{file_path}' declares `cursor_start` or `cursor_end`, which are only " + f"supported for model tests, not mode '{test_mode.value}'" + ) return DiscoveredSqlTestBlock( test_index=test_index, header_values=header_values, @@ -143,6 +157,25 @@ def _parse_single_sql_test_block( mode=test_mode, parameters=parameters, cases=cases, + cursor_start=cursor_start, + cursor_end=cursor_end, + ) + + +def _parse_test_cursor_bound( + *, header_values: dict[str, object], key: str, file_path: Path +) -> str | None: + value: object | None = header_values.get(key) + if value is None: + return None + if isinstance(value, datetime | date): + return value.isoformat() + if isinstance(value, int) and not isinstance(value, bool): + return str(value) + if isinstance(value, str) and value.strip(): + return value.strip() + raise SqlTestParseError( + f"TEST() {key} in '{file_path}' must be a non-empty string or an integer" ) @@ -160,6 +193,8 @@ def _parse_test_header(*, header: str, file_path: Path) -> dict[str, object]: _TEST_MODE_HEADER_KEY, _TEST_PARAMETERS_HEADER_KEY, _TEST_CASES_HEADER_KEY, + _TEST_CURSOR_START_HEADER_KEY, + _TEST_CURSOR_END_HEADER_KEY, } ) unsupported_keys: tuple[str, ...] = tuple( @@ -167,8 +202,8 @@ def _parse_test_header(*, header: str, file_path: Path) -> dict[str, object]: ) if unsupported_keys: raise SqlTestParseError( - f"TEST() in '{file_path}' only supports `name`, `mode`, `parameters`, and " - "`cases`; unsupported keys: " + f"TEST() in '{file_path}' only supports `name`, `mode`, `parameters`, `cases`, " + "`cursor_start`, and `cursor_end`; unsupported keys: " f"{', '.join(unsupported_keys)}" ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index 01a4e9b1c1..06869d331a 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -100,7 +100,12 @@ _BATCH_CONCURRENCY_CONFIG_KEY: str = "batch_concurrency" _MAX_BATCHES_CONFIG_KEY: str = "max_batches" _EVENT_EXPORTER_FILTER_KEYS: frozenset[str] = frozenset({"event_kinds", "min_severity"}) -_EVENT_EXPORTER_KEYS: frozenset[str] = _EVENT_EXPORTER_FILTER_KEYS | frozenset({"named"}) +_LIFECYCLE_SHUTDOWN_TIMEOUT_CONFIG_KEY: str = "shutdown_timeout" +_LIFECYCLE_SHUTDOWN_TIMEOUT_MAX_SECONDS: int = 600 +_ZERO_FIXED_DURATION: str = "0s" +_EVENT_EXPORTER_KEYS: frozenset[str] = _EVENT_EXPORTER_FILTER_KEYS | frozenset( + {"named", _LIFECYCLE_SHUTDOWN_TIMEOUT_CONFIG_KEY} +) _LEGACY_EVENT_EXPORTERS_CONFIG_KEY: str = "event_exporters" @@ -194,8 +199,7 @@ def _load_diff(*, payload: object, file_path: Path) -> DiffConfig: file_path=file_path, ) value: str = _optional_str(payload=mapping, key=key) or "24h" - duration: Duration | None = Duration.parse(value) - if duration is None or duration.has_calendar_component: + if _fixed_duration_seconds(value=value) is None: raise ProjectConfigError( f"{file_path} diff.{key} must be a positive fixed duration such as '24h'" ) @@ -288,10 +292,44 @@ def load_filter(*, value: object, label: str) -> LifecycleEventSinkFilterConfig: defaults_mapping: dict[str, object] = { key: value for key, value in config_mapping.items() if key in _EVENT_EXPORTER_FILTER_KEYS } + shutdown_timeout, shutdown_timeout_seconds = _load_lifecycle_shutdown_timeout( + value=config_mapping.get(_LIFECYCLE_SHUTDOWN_TIMEOUT_CONFIG_KEY), + file_path=file_path, + ) return LifecycleEventSinksConfig( defaults=load_filter(value=defaults_mapping, label="sinks.lifecycle"), named=named, + shutdown_timeout=shutdown_timeout, + shutdown_timeout_seconds=shutdown_timeout_seconds, + ) + + +def _load_lifecycle_shutdown_timeout( + *, value: object, file_path: Path +) -> tuple[str | None, int | None]: + if value is None: + return None, None + message: str = ( + f"{file_path} sinks.lifecycle.{_LIFECYCLE_SHUTDOWN_TIMEOUT_CONFIG_KEY} must be a fixed " + f"duration from '0s' to '10m' such as '30s'" ) + if not isinstance(value, str): + raise ProjectConfigError(message) + seconds: int | None = _fixed_duration_seconds(value=value, allow_zero=True) + if seconds is None or seconds > _LIFECYCLE_SHUTDOWN_TIMEOUT_MAX_SECONDS: + raise ProjectConfigError(message) + return value, seconds + + +def _fixed_duration_seconds(*, value: str, allow_zero: bool = False) -> int | None: + """Return a fixed duration's seconds, or None when it is invalid or calendar-based.""" + + if allow_zero and value == _ZERO_FIXED_DURATION: + return 0 + duration: Duration | None = Duration.parse(value) + if duration is None or duration.has_calendar_component: + return None + return duration.fixed_seconds def load_local_config(*, project_dir: Path) -> LocalConfig: @@ -1347,12 +1385,11 @@ def _load_execution_limits( max_duration: str | None = _optional_str(payload=mapping, key="max_duration") max_duration_seconds: int | None = None if max_duration is not None: - duration: Duration | None = Duration.parse(max_duration) - if duration is None or duration.has_calendar_component or duration.fixed_seconds <= 0: + max_duration_seconds = _fixed_duration_seconds(value=max_duration) + if max_duration_seconds is None: raise ProjectConfigError( f"{file_path} {label}.max_duration must be a positive fixed duration such as '30m'" ) - max_duration_seconds = duration.fixed_seconds return ExecutionLimitsConfig( max_models=max_models, max_duration=max_duration, diff --git a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py index bdc180e17e..a7cd51954d 100644 --- a/src/sqlbuild/compiler/discovery/main/runtime_extensions.py +++ b/src/sqlbuild/compiler/discovery/main/runtime_extensions.py @@ -20,6 +20,7 @@ DiscoveredEventExporter, DiscoveredEventExporterDeclaration, DiscoveredProvider, + DiscoveredRuntimeExtensions, ) from sqlbuild.runtime.event_exporting.main.stricter_severity import stricter_severity from sqlbuild.spec.contracts.models import ( @@ -29,14 +30,8 @@ from sqlbuild.spec.contracts.types import EventExportSeverity -def discover_runtime_extensions( - *, project_dir: Path -) -> tuple[ - tuple[DiscoveredProvider, ...], - tuple[DiscoveredEventExporter, ...], - tuple[DiscoveredCommandOutputSink, ...], -]: - """Discover providers and typed sinks at command startup.""" +def discover_runtime_extensions(*, project_dir: Path) -> DiscoveredRuntimeExtensions: + """Discover providers, typed sinks, and lifecycle delivery settings at command startup.""" declarations: tuple[DiscoveredEventExporterDeclaration, ...] = ( discover_event_exporter_declarations(project_dir=project_dir) @@ -49,7 +44,7 @@ def discover_runtime_extensions( and not output_declarations and not (project_dir / "sqlbuild_project.toml").exists() ): - return (), (), () + return DiscoveredRuntimeExtensions() config: LifecycleEventSinksConfig = load_project_config(project_dir=project_dir).sinks.lifecycle declaration_names: frozenset[str] = frozenset(item.name for item in declarations) unknown_names: frozenset[str] = frozenset(config.named) - declaration_names @@ -58,7 +53,7 @@ def discover_runtime_extensions( "sinks.lifecycle.named contains unknown sink(s): " + ", ".join(sorted(unknown_names)) ) if not declarations and not output_declarations: - return (), (), () + return DiscoveredRuntimeExtensions() providers: tuple[DiscoveredProvider, ...] = discover_provider_classes(project_dir=project_dir) exporters: tuple[DiscoveredEventExporter, ...] = bind_event_exporter_declarations( declarations=declarations, @@ -70,10 +65,13 @@ def discover_runtime_extensions( project_dir=project_dir, providers=providers, ) - return ( - providers, - tuple(_apply_runtime_filter(exporter=exporter, config=config) for exporter in exporters), - output_sinks, + return DiscoveredRuntimeExtensions( + providers=providers, + event_exporters=tuple( + _apply_runtime_filter(exporter=exporter, config=config) for exporter in exporters + ), + command_output_sinks=output_sinks, + lifecycle_shutdown_timeout_seconds=config.shutdown_timeout_seconds, ) diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index fc9023e217..530fd3a6be 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -303,6 +303,8 @@ class DiscoveredSqlTestBlock: mode: SqlTestMode = DEFAULT_SQL_TEST_MODE parameters: tuple[SqlTestParameterDeclaration, ...] = field(default_factory=tuple) cases: tuple[DiscoveredSqlTestCase, ...] = field(default_factory=tuple) + cursor_start: str | None = None + cursor_end: str | None = None @dataclass(frozen=True) @@ -583,6 +585,16 @@ class DiscoveredProjectInputs: adapter_file: DiscoveredAdapterFile | None = None +@dataclass(frozen=True) +class DiscoveredRuntimeExtensions: + """Command-startup providers, typed sinks, and lifecycle delivery settings.""" + + providers: tuple[DiscoveredProvider, ...] = field(default_factory=tuple) + event_exporters: tuple[DiscoveredEventExporter, ...] = field(default_factory=tuple) + command_output_sinks: tuple[DiscoveredCommandOutputSink, ...] = field(default_factory=tuple) + lifecycle_shutdown_timeout_seconds: int | None = None + + @dataclass(frozen=True) class DiscoveryFileFault: """One project-relative authored file fault captured during tolerant discovery.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py new file mode 100644 index 0000000000..1479a470b4 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/cursor_intrinsics.py @@ -0,0 +1,25 @@ +"""Render cursor intrinsics with one interval's adapter-typed bound literals.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics +from sqlbuild.compiler.planner.models import CursorBounds +from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token + + +def render_cursor_intrinsic_bounds( + *, sql: str, bounds: CursorBounds, cursor_type: str | None, adapter: BaseAdapter +) -> str: + """Replace `__cursor_start()` / `__cursor_end()` with the adapter's typed bound literals.""" + + rendered, _ = resolve_cursor_intrinsics( + sql=sql, + start_sql=adapter.render_cursor_bound_literal( + value=sentinel_to_token(sentinel=bounds.start), cursor_type=cursor_type + ), + end_sql=adapter.render_cursor_bound_literal( + value=sentinel_to_token(sentinel=bounds.end), cursor_type=cursor_type + ), + ) + return rendered diff --git a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py index ce01950cec..c11d3b0ef8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py +++ b/src/sqlbuild/compiler/planner/_helpers/resolve/resolve.py @@ -24,6 +24,9 @@ compute_cursor_bounds, without_destination_cursor, ) +from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( + render_cursor_intrinsic_bounds, +) from sqlbuild.compiler.planner._helpers.resolve.cursor_policies import ( resolve_future_cursor_config, resolve_start_cursor_config, @@ -53,7 +56,6 @@ assert_no_unresolved_sql_markers, ) from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token from sqlbuild.cursor_algebra.types import BoundSentinel from sqlbuild.spec.contracts.main.get_config_cursor_bound import get_config_cursor_bound from sqlbuild.spec.contracts.main.get_config_str import get_config_str @@ -149,16 +151,8 @@ def resolve_model_sql( f"Model '{model.name}' uses cursor intrinsics, but cursor bounds could not be " "resolved" ) - query_sql, _ = resolve_cursor_intrinsics( - sql=query_sql, - start_sql=adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=cursor_bounds.start), - cursor_type=cursor_type, - ), - end_sql=adapter.render_cursor_bound_literal( - value=sentinel_to_token(sentinel=cursor_bounds.end), - cursor_type=cursor_type, - ), + query_sql = render_cursor_intrinsic_bounds( + sql=query_sql, bounds=cursor_bounds, cursor_type=cursor_type, adapter=adapter ) assert_no_unresolved_sql_markers(sql=query_sql, context=f"Model '{model.name}' planned SQL") diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py new file mode 100644 index 0000000000..59cc840666 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/cursor_window.py @@ -0,0 +1,166 @@ +"""Cursor windows rendered for models that SQL tests evaluate from their real SQL.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.compiler.compile.main.cursor_intrinsics import resolve_cursor_intrinsics +from sqlbuild.compiler.compile.models import CompiledModel, CompiledSqlTest +from sqlbuild.compiler.planner._helpers.resolve.cursor_intrinsics import ( + render_cursor_intrinsic_bounds, +) +from sqlbuild.compiler.planner.models import CursorBounds +from sqlbuild.compiler.planner.types import CursorGrain, CursorType +from sqlbuild.cursor_algebra.exceptions import CursorAlgebraError +from sqlbuild.cursor_algebra.main.compare import compare +from sqlbuild.cursor_algebra.main.floor_to_grain import floor_to_grain +from sqlbuild.cursor_algebra.main.parse import parse +from sqlbuild.cursor_algebra.main.render import render +from sqlbuild.cursor_algebra.types import CursorScalar +from sqlbuild.spec.contracts.main.get_config_str import get_config_str + +_DEFAULT_TIMESTAMP_START: str = "1900-01-01 00:00:00" +_DEFAULT_TIMESTAMP_END: str = "2999-12-31 00:00:00" +_DEFAULT_INTEGER_START: str = "-1000000000000000" +_DEFAULT_INTEGER_END: str = "1000000000000000" +_CURSOR_START_KEY: str = "cursor_start" +_CURSOR_END_KEY: str = "cursor_end" + + +@dataclass(frozen=True) +class _ModelCursor: + name: str + cursor_type: CursorType + cursor_grain: CursorGrain | None + + +def uses_cursor_intrinsics(*, sql: str) -> bool: + """Return whether SQL calls `__cursor_start()` or `__cursor_end()`.""" + + _, found = resolve_cursor_intrinsics(sql=sql) + return found + + +def declares_cursor_window(*, test: CompiledSqlTest) -> bool: + """Return whether a SQL test header declares its own cursor window.""" + + return test.test_block.cursor_start is not None or test.test_block.cursor_end is not None + + +def render_test_cursor_intrinsics( + *, + sql: str, + model: CompiledModel, + adapter: BaseAdapter, + test: CompiledSqlTest | None, +) -> str: + """Render cursor intrinsics with the test's declared window, or the wide default window.""" + + if not uses_cursor_intrinsics(sql=sql): + return sql + cursor: _ModelCursor = _model_cursor(model=model) + start, end = _test_window(cursor=cursor, test=test) + return render_cursor_intrinsic_bounds( + sql=sql, + bounds=CursorBounds(start=start, end=end), + cursor_type=cursor.cursor_type, + adapter=adapter, + ) + + +def declared_window_cursor_models( + *, test: CompiledSqlTest, chain_models: tuple[CompiledModel, ...] +) -> tuple[CompiledModel, ...]: + """Validate a declared window against every chain model using intrinsics and return them.""" + + cursor_models: tuple[CompiledModel, ...] = tuple( + model for model in chain_models if uses_cursor_intrinsics(sql=model.query_sql) + ) + if not cursor_models: + raise CompileInputError( + f"{_test_label(test=test)} declares cursor_start or cursor_end, but no model it " + "evaluates uses __cursor_start() or __cursor_end()" + ) + for model in cursor_models: + _test_window(cursor=_model_cursor(model=model), test=test) + return cursor_models + + +def _test_window( + *, cursor: _ModelCursor, test: CompiledSqlTest | None +) -> tuple[CursorScalar, CursorScalar]: + declared_start: str | None = test.test_block.cursor_start if test is not None else None + declared_end: str | None = test.test_block.cursor_end if test is not None else None + integer: bool = cursor.cursor_type == CursorType.INTEGER + start: CursorScalar = _bound( + raw=declared_start, + default=_DEFAULT_INTEGER_START if integer else _DEFAULT_TIMESTAMP_START, + key=_CURSOR_START_KEY, + cursor=cursor, + test=test, + ) + end: CursorScalar = _bound( + raw=declared_end, + default=_DEFAULT_INTEGER_END if integer else _DEFAULT_TIMESTAMP_END, + key=_CURSOR_END_KEY, + cursor=cursor, + test=test, + ) + if test is not None and compare(left=start, right=end) >= 0: + raise CompileInputError( + f"{_test_label(test=test)}: cursor_start '{render(value=start)}' must be before " + f"the exclusive cursor_end '{render(value=end)}' for model '{cursor.name}'" + ) + return start, end + + +def _bound( + *, + raw: str | None, + default: str, + key: str, + cursor: _ModelCursor, + test: CompiledSqlTest | None, +) -> CursorScalar: + if raw is None or test is None: + return parse(raw=default, cursor_type=cursor.cursor_type) + try: + value: CursorScalar = parse(raw=raw, cursor_type=cursor.cursor_type) + except CursorAlgebraError: + expected: str = ( + "an integer" if cursor.cursor_type == CursorType.INTEGER else "an ISO timestamp or date" + ) + raise CompileInputError( + f"{_test_label(test=test)}: {key} '{raw}' must be {expected} because model " + f"'{cursor.name}' uses cursor_type {cursor.cursor_type.value}" + ) from None + if cursor.cursor_grain is not None and compare( + left=floor_to_grain(value=value, grain=cursor.cursor_grain), right=value + ): + raise CompileInputError( + f"{_test_label(test=test)}: {key} '{raw}' is not aligned to cursor_grain " + f"{cursor.cursor_grain.value} of model '{cursor.name}'" + ) + return value + + +def _model_cursor(*, model: CompiledModel) -> _ModelCursor: + cursor_type: CursorType = CursorType( + get_config_str(values=model.config.values, key="cursor_type") or CursorType.TIMESTAMP + ) + grain: str | None = get_config_str(values=model.config.values, key="cursor_grain") + return _ModelCursor( + name=model.name, + cursor_type=cursor_type, + cursor_grain=( + CursorGrain(grain) + if grain is not None and cursor_type == CursorType.TIMESTAMP + else None + ), + ) + + +def _test_label(*, test: CompiledSqlTest) -> str: + return f"SQL test '{test.name}' in '{test.test_file.relative_path}'" diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py index f8a279b15e..93742a0b40 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py @@ -12,12 +12,18 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, + CompiledModel, CompiledModelSqlTestPayload, CompiledProject, CompiledSqlTest, CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( + declared_window_cursor_models, + declares_cursor_window, + render_test_cursor_intrinsics, +) from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError, PlannerInputError from sqlbuild.compiler.planner.main.execution.sql_test_dialect import ( restore_sql_test_dialect_function_names, @@ -93,9 +99,9 @@ def plan_sql_tests_natively( return () start_ns: int = time.perf_counter_ns() request: dict[str, object] = { - "models": _model_requests(project=project), + "models": _model_requests(project=project, adapter=adapter), "functions": _function_requests(project=project, adapter=adapter), - "tests": [_test_request(test=test) for test in tests], + "tests": _planning_test_requests(project=project, tests=tests, adapter=adapter), "sqlAnalysisEnabled": sql_analysis_enabled, "sqlAnalysisDialect": adapter.sql_analysis_dialect(), "setDifferenceOperator": adapter.render_set_difference_operator(), @@ -223,6 +229,7 @@ def _chain_step(*, value: object) -> ChainStep: expected_columns=_optional_string_tuple( value=payload.get("expectedColumns"), context="chain step expected columns" ), + expected_lifted_ctes=_cte_pairs(value=payload.get("expectedLiftedCtes", [])), ) @@ -317,7 +324,79 @@ def _function_requests( return functions -def _test_request(*, test: CompiledSqlTest) -> dict[str, object]: +def _planning_test_requests( + *, project: CompiledProject, tests: tuple[CompiledSqlTest, ...], adapter: BaseAdapter +) -> list[dict[str, object]]: + """Build test requests whose model SQL has cursor intrinsics rendered for the test window.""" + + models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + windowed_indexes: tuple[int, ...] = tuple( + index + for index, test in enumerate(tests) + if isinstance(test.payload, CompiledModelSqlTestPayload) + and declares_cursor_window(test=test) + ) + chains: tuple[tuple[str, ...], ...] = ( + resolve_sql_test_model_chains( + project=project, tests=tuple(tests[index] for index in windowed_indexes) + ) + if windowed_indexes + else () + ) + chains_by_index: dict[int, tuple[str, ...]] = dict(zip(windowed_indexes, chains, strict=True)) + return [ + _test_request( + test=test, + model_query_overrides=_windowed_model_query_overrides( + test=test, + models_by_name=models_by_name, + adapter=adapter, + chain=chains_by_index.get(index), + ), + ) + for index, test in enumerate(tests) + ] + + +def _windowed_model_query_overrides( + *, + test: CompiledSqlTest, + models_by_name: dict[str, CompiledModel], + adapter: BaseAdapter, + chain: tuple[str, ...] | None, +) -> dict[str, str] | None: + payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload + if not isinstance(payload, CompiledModelSqlTestPayload): + return None + chain_names: frozenset[str] = frozenset(chain or ()) + overrides: dict[str, str] = {} + for model_name, sql in payload.model_query_overrides.items(): + model: CompiledModel | None = models_by_name.get(model_name) + overrides[model_name] = ( + sql + if model is None + else render_test_cursor_intrinsics( + sql=sql, + model=model, + adapter=adapter, + test=test if model_name in chain_names else None, + ) + ) + if chain is not None: + chain_models: tuple[CompiledModel, ...] = tuple( + models_by_name[name] for name in chain if name in models_by_name + ) + for model in declared_window_cursor_models(test=test, chain_models=chain_models): + if model.name not in overrides: + overrides[model.name] = render_test_cursor_intrinsics( + sql=model.query_sql, model=model, adapter=adapter, test=test + ) + return overrides + + +def _test_request( + *, test: CompiledSqlTest, model_query_overrides: dict[str, str] | None = None +) -> dict[str, object]: payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload = test.payload if isinstance(payload, CompiledDirectLogicSqlTestPayload): payload_request: dict[str, object] = { @@ -331,7 +410,11 @@ def _test_request(*, test: CompiledSqlTest) -> dict[str, object]: payload_request = { "kind": "model", "authoredCtes": [_cte_request(cte=cte) for cte in payload.authored_ctes], - "modelQueryOverrides": payload.model_query_overrides, + "modelQueryOverrides": ( + payload.model_query_overrides + if model_query_overrides is None + else model_query_overrides + ), "expectedCtes": [_cte_request(cte=cte) for cte in payload.expected_ctes], "expectedModelNames": list(payload.expected_model_names), "assertionCtes": [_cte_request(cte=cte) for cte in payload.assertion_ctes], @@ -343,17 +426,26 @@ def _test_request(*, test: CompiledSqlTest) -> dict[str, object]: } -def _model_requests(*, project: CompiledProject) -> list[dict[str, object]]: +def _model_requests( + *, project: CompiledProject, adapter: BaseAdapter | None = None +) -> list[dict[str, object]]: requests: list[dict[str, object]] = [] for model in project.models: dependencies: list[str] = [] for dependency in model.deps: if dependency.resource_type == CompiledResourceType.MODEL: dependencies.append(dependency.name) + query_sql: str = ( + model.query_sql + if adapter is None + else render_test_cursor_intrinsics( + sql=model.query_sql, model=model, adapter=adapter, test=None + ) + ) requests.append( { "name": model.name, - "querySql": model.query_sql, + "querySql": query_sql, "modelDependencies": dependencies, } ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 7c87d97531..58ff77e55e 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -1169,6 +1169,7 @@ class ChainStep: lifted_ctes: tuple[tuple[str, str], ...] = field(default_factory=tuple) comparison_body_sql: str | None = None expected_columns: tuple[str, ...] | None = None + expected_lifted_ctes: tuple[tuple[str, str], ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/src/sqlbuild/executor/auditing/_helpers/result_projection.py b/src/sqlbuild/executor/auditing/_helpers/result_projection.py index 86a5c2e280..3f5c4438c1 100644 --- a/src/sqlbuild/executor/auditing/_helpers/result_projection.py +++ b/src/sqlbuild/executor/auditing/_helpers/result_projection.py @@ -5,8 +5,11 @@ import hashlib import json import logging +import threading from collections import defaultdict -from contextvars import ContextVar +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import Context, ContextVar, Token, copy_context from datetime import UTC, datetime from typing import Any @@ -34,6 +37,141 @@ ) +class _AuditResultPublisher: + """Publish each confirmed audit once, in the captured run context, and keep its record.""" + + def __init__( + self, + *, + plan: PlanOutput, + storage_database: str | None, + storage_schema: str | None, + ) -> None: + self.plan: PlanOutput = plan + self.identity: ExecutionIdentity | None = current_execution_identity() + self.database, self.schema = _storage_location( + plan=plan, database=storage_database, schema=storage_schema + ) + self._context: Context = copy_context() + self._lock: threading.Lock = threading.Lock() + self._occurrence: defaultdict[tuple[str, str], int] = defaultdict(int) + self._records: dict[int, tuple[AuditExecutionResult, AuditResultRecord | None]] = {} + self.build_error: Exception | None = None + + @property + def available(self) -> bool: + return ( + self.identity is not None + and self.identity.run_id is not None + and self.schema is not None + ) + + def publish(self, results: tuple[AuditExecutionResult, ...]) -> None: + """Build and publish records for newly confirmed, non-reused results.""" + + if not self.available: + return + with self._lock: + for result in results: + if result.reused or id(result) in self._records: + continue + record: AuditResultRecord | None = self._build(result) + self._records[id(result)] = (result, record) + if record is not None: + self._context.run(_publish_safely, record) + + def records_for( + self, results: tuple[AuditExecutionResult, ...] + ) -> tuple[AuditResultRecord, ...]: + """Return batch-ordered records, publishing any result not yet confirmed.""" + + self.publish(results) + if self.build_error is not None: + raise self.build_error + records: list[AuditResultRecord] = [] + with self._lock: + for result in results: + record: AuditResultRecord | None = ( + None if result.reused else self._records[id(result)][1] + ) + if record is not None: + records.append(record) + return tuple(records) + + def _build(self, result: AuditExecutionResult) -> AuditResultRecord | None: + if self.build_error is not None or self.identity is None or self.schema is None: + return None + try: + entry: AuditPlanEntry | None = next( + ( + candidate + for candidate in self.plan.audit_entries + if _matches(entry=candidate, result=result) + ), + None, + ) + if entry is None: + _LOGGER.warning( + "Audit result projection skipped unknown audit result '%s'", result.audit_name + ) + return None + audit_id: AuditIdentity = build_audit_gate_identity(audits=(entry,)).audits[0] + occurrence_key: tuple[str, str] = (audit_id.binding_key, result.run_scope_phase.value) + ordinal: int = self._occurrence[occurrence_key] + self._occurrence[occurrence_key] = ordinal + 1 + return _build_record( + result=result, + entry=entry, + audit_id=audit_id, + ordinal=ordinal, + plan=self.plan, + identity=self.identity, + storage_database=self.database, + storage_schema=self.schema, + ) + except Exception as error: + self.build_error = error + return None + + +_ACTIVE_AUDIT_RESULT_PUBLISHER: ContextVar[_AuditResultPublisher | None] = ContextVar( + "sqlbuild_active_audit_result_publisher", default=None +) + + +@contextmanager +def audit_result_publication_scope_impl( + *, + plan: PlanOutput, + storage_database: str | None = None, + storage_schema: str | None = None, +) -> Iterator[None]: + """Publish audit completions incrementally until the batch projection finalizes them.""" + + publisher: _AuditResultPublisher = _AuditResultPublisher( + plan=plan, + storage_database=storage_database, + storage_schema=storage_schema, + ) + token: Token[_AuditResultPublisher | None] = _ACTIVE_AUDIT_RESULT_PUBLISHER.set(publisher) + try: + yield + finally: + _ACTIVE_AUDIT_RESULT_PUBLISHER.reset(token) + + +def publish_completed_audit_results_impl(results: tuple[AuditExecutionResult, ...]) -> None: + """Publish confirmed audit outcomes through the active scope without raising.""" + + publisher: _AuditResultPublisher | None = _ACTIVE_AUDIT_RESULT_PUBLISHER.get() + if publisher is None: + return + try: + publisher.publish(results) + except Exception as error: + _LOGGER.warning("Audit result lifecycle publication degraded (%s)", error) + + def current_audit_result_projection_impl() -> AuditResultProjection | None: """Return the most recent audit projection report in this execution context.""" @@ -58,7 +196,15 @@ def project_audit_result_batch_impl( projection: AuditResultProjection = AuditResultProjection() _LAST_AUDIT_RESULT_PROJECTION.set(projection) return projection - identity: ExecutionIdentity | None = current_execution_identity() + active: _AuditResultPublisher | None = _ACTIVE_AUDIT_RESULT_PUBLISHER.get() + publisher: _AuditResultPublisher = ( + active + if active is not None and active.plan is plan + else _AuditResultPublisher( + plan=plan, storage_database=storage_database, storage_schema=storage_schema + ) + ) + identity: ExecutionIdentity | None = publisher.identity if identity is None or identity.run_id is None: _LOGGER.warning("Audit result projection skipped because runtime identity is unavailable") projection = AuditResultProjection( @@ -66,11 +212,8 @@ def project_audit_result_batch_impl( ) _LAST_AUDIT_RESULT_PROJECTION.set(projection) return projection - database, schema = _storage_location( - plan=plan, - database=storage_database, - schema=storage_schema, - ) + database: str | None = publisher.database + schema: str | None = publisher.schema if schema is None: _LOGGER.warning("Audit result projection skipped because target schema is unavailable") projection = AuditResultProjection( @@ -80,13 +223,7 @@ def project_audit_result_batch_impl( return projection try: - records: tuple[AuditResultRecord, ...] = _build_records( - plan=plan, - results=executed, - identity=identity, - storage_database=database, - storage_schema=schema, - ) + records: tuple[AuditResultRecord, ...] = publisher.records_for(executed) except Exception as error: _LOGGER.warning( "Audit result projection degraded: attempted=%d written=0 failed=%d (%s)", @@ -100,15 +237,6 @@ def project_audit_result_batch_impl( _LAST_AUDIT_RESULT_PROJECTION.set(projection) return projection unmatched_count: int = len(executed) - len(records) - for record in records: - try: - _publish_audit_completed(record) - except Exception as error: - _LOGGER.warning( - "Audit result lifecycle publication degraded for '%s' (%s)", - record.audit_name, - error, - ) if not records: projection = AuditResultProjection( attempted_count=len(executed), failed_count=len(executed) @@ -148,103 +276,93 @@ def project_audit_result_batch_impl( return projection -def _build_records( +def _build_record( # noqa: PLR0913 *, + result: AuditExecutionResult, + entry: AuditPlanEntry, + audit_id: AuditIdentity, + ordinal: int, plan: PlanOutput, - results: tuple[AuditExecutionResult, ...], identity: ExecutionIdentity, storage_database: str | None, storage_schema: str, -) -> tuple[AuditResultRecord, ...]: - entries: list[AuditPlanEntry] = list(plan.audit_entries) - occurrence: defaultdict[tuple[str, str], int] = defaultdict(int) - records: list[AuditResultRecord] = [] - for result in results: - entry: AuditPlanEntry | None = next( - (candidate for candidate in entries if _matches(entry=candidate, result=result)), - None, - ) - if entry is None: - _LOGGER.warning( - "Audit result projection skipped unknown audit result '%s'", result.audit_name - ) - continue - audit_id: AuditIdentity = build_audit_gate_identity(audits=(entry,)).audits[0] - occurrence_key: tuple[str, str] = (audit_id.binding_key, result.run_scope_phase.value) - ordinal: int = occurrence[occurrence_key] - occurrence[occurrence_key] += 1 - attempt_key: str = f"{result.run_scope_phase.value}:{ordinal}" - result_id: str = build_audit_result_id( - invocation_id=identity.invocation_id, - run_id=identity.run_id or "", - binding_key=audit_id.binding_key, - execution_fingerprint=audit_id.execution_fingerprint, - run_scope_phase=result.run_scope_phase.value, - attempt_key=attempt_key, - ) - target: CompiledRelationLocation | None = ( - plan.model_locations.get(result.attached_target_name) - if result.attached_target_name is not None +) -> AuditResultRecord: + attempt_key: str = f"{result.run_scope_phase.value}:{ordinal}" + result_id: str = build_audit_result_id( + invocation_id=identity.invocation_id, + run_id=identity.run_id or "", + binding_key=audit_id.binding_key, + execution_fingerprint=audit_id.execution_fingerprint, + run_scope_phase=result.run_scope_phase.value, + attempt_key=attempt_key, + ) + target: CompiledRelationLocation | None = ( + plan.model_locations.get(result.attached_target_name) + if result.attached_target_name is not None + else None + ) + thresholds: dict[str, object] | None = _render_thresholds(entry) + evidence: list[dict[str, object]] = [dict(row) for row in result.evidence_rows] + return AuditResultRecord( + result_id=result_id, + schema_version=AUDIT_RESULT_SCHEMA_VERSION, + occurred_at=datetime.now(UTC), + invocation_id=identity.invocation_id, + run_id=identity.run_id or "", + audit_name=result.audit_name, + audit_definition_name=result.audit_definition_name, + audit_description=result.audit_description, + binding_key=audit_id.binding_key, + definition_fingerprint=audit_id.definition_fingerprint, + execution_fingerprint=audit_id.execution_fingerprint, + evaluation_mode=result.evaluation_mode.value, + run_scope_phase=result.run_scope_phase.value, + attachment_kind=result.attachment_kind.value, + attached_target_kind=( + None if result.attached_target_kind is None else result.attached_target_kind.value + ), + attached_target_name=result.attached_target_name, + attached_column_name=result.attached_column_name, + target_database=(target.database if target is not None else storage_database), + target_schema=(target.schema if target is not None else storage_schema), + target_name=(target.name if target is not None else result.attached_target_name), + severity=result.severity.value, + outcome=result.outcome.value, + execution_error=result.execution_error, + violation_count=( + result.row_count if result.evaluation_mode == AuditEvaluationMode.VIOLATIONS else None + ), + measured_value=result.measured_value, + sample_count=result.sample_count, + sample_unit=result.sample_unit, + minimum_samples=result.minimum_samples, + thresholds_json=_json_or_none(thresholds), + evidence_json=_json_or_none(evidence) if evidence else None, + evidence_count=len(evidence), + evidence_truncated=result.evidence_truncated, + evidence_error=result.evidence_error, + measurement_sql=( + result.executed_sql + if result.evaluation_mode == AuditEvaluationMode.MEASUREMENT else None + ), + evidence_sql=result.evidence_sql, + executed_sql=result.executed_sql, + sql_digest=hashlib.sha256(result.executed_sql.encode("utf-8")).hexdigest(), + metadata_json=None, + reused=False, + ) + + +def _publish_safely(record: AuditResultRecord) -> None: + try: + _publish_audit_completed(record) + except Exception as error: + _LOGGER.warning( + "Audit result lifecycle publication degraded for '%s' (%s)", + record.audit_name, + error, ) - thresholds: dict[str, object] | None = _render_thresholds(entry) - evidence: list[dict[str, object]] = [dict(row) for row in result.evidence_rows] - records.append( - AuditResultRecord( - result_id=result_id, - schema_version=AUDIT_RESULT_SCHEMA_VERSION, - occurred_at=datetime.now(UTC), - invocation_id=identity.invocation_id, - run_id=identity.run_id or "", - audit_name=result.audit_name, - audit_definition_name=result.audit_definition_name, - audit_description=result.audit_description, - binding_key=audit_id.binding_key, - definition_fingerprint=audit_id.definition_fingerprint, - execution_fingerprint=audit_id.execution_fingerprint, - evaluation_mode=result.evaluation_mode.value, - run_scope_phase=result.run_scope_phase.value, - attachment_kind=result.attachment_kind.value, - attached_target_kind=( - None - if result.attached_target_kind is None - else result.attached_target_kind.value - ), - attached_target_name=result.attached_target_name, - attached_column_name=result.attached_column_name, - target_database=(target.database if target is not None else storage_database), - target_schema=(target.schema if target is not None else storage_schema), - target_name=(target.name if target is not None else result.attached_target_name), - severity=result.severity.value, - outcome=result.outcome.value, - execution_error=result.execution_error, - violation_count=( - result.row_count - if result.evaluation_mode == AuditEvaluationMode.VIOLATIONS - else None - ), - measured_value=result.measured_value, - sample_count=result.sample_count, - sample_unit=result.sample_unit, - minimum_samples=result.minimum_samples, - thresholds_json=_json_or_none(thresholds), - evidence_json=_json_or_none(evidence) if evidence else None, - evidence_count=len(evidence), - evidence_truncated=result.evidence_truncated, - evidence_error=result.evidence_error, - measurement_sql=( - result.executed_sql - if result.evaluation_mode == AuditEvaluationMode.MEASUREMENT - else None - ), - evidence_sql=result.evidence_sql, - executed_sql=result.executed_sql, - sql_digest=hashlib.sha256(result.executed_sql.encode("utf-8")).hexdigest(), - metadata_json=None, - reused=False, - ) - ) - return tuple(records) def _publish_audit_completed(record: AuditResultRecord) -> None: diff --git a/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py b/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py new file mode 100644 index 0000000000..af77daa140 --- /dev/null +++ b/src/sqlbuild/executor/auditing/main/audit_result_publication_scope.py @@ -0,0 +1,28 @@ +"""Incremental audit completion publication scope entrypoint.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.executor.auditing._helpers.result_projection import ( + audit_result_publication_scope_impl, +) + + +@contextmanager +def audit_result_publication_scope( + *, + plan: PlanOutput, + storage_database: str | None = None, + storage_schema: str | None = None, +) -> Iterator[None]: + """Publish audit completions as they are confirmed until the batch projection runs.""" + + with audit_result_publication_scope_impl( + plan=plan, + storage_database=storage_database, + storage_schema=storage_schema, + ): + yield diff --git a/src/sqlbuild/executor/auditing/main/publish_completed_audit_results.py b/src/sqlbuild/executor/auditing/main/publish_completed_audit_results.py new file mode 100644 index 0000000000..11854114c7 --- /dev/null +++ b/src/sqlbuild/executor/auditing/main/publish_completed_audit_results.py @@ -0,0 +1,14 @@ +"""Incremental audit completion publication entrypoint.""" + +from __future__ import annotations + +from sqlbuild.executor.auditing._helpers.result_projection import ( + publish_completed_audit_results_impl, +) +from sqlbuild.executor.auditing.models import AuditExecutionResult + + +def publish_completed_audit_results(results: tuple[AuditExecutionResult, ...]) -> None: + """Publish confirmed, non-reused audit outcomes once; no-op outside a publication scope.""" + + return publish_completed_audit_results_impl(results) diff --git a/src/sqlbuild/executor/build/_helpers/end_audits.py b/src/sqlbuild/executor/build/_helpers/end_audits.py index fbc4bfb919..4564788050 100644 --- a/src/sqlbuild/executor/build/_helpers/end_audits.py +++ b/src/sqlbuild/executor/build/_helpers/end_audits.py @@ -9,6 +9,9 @@ from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.compiler.planner.models import AuditPlanEntry from sqlbuild.executor.auditing.main._execute import execute_audit +from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( + publish_completed_audit_results, +) from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( @@ -58,4 +61,5 @@ def run_end_audits( if result.outcome == AuditOutcome.ERROR: lifecycle.failed() results.append(result) + publish_completed_audit_results((result,)) return tuple(results) diff --git a/src/sqlbuild/executor/build/_helpers/source_audits.py b/src/sqlbuild/executor/build/_helpers/source_audits.py index df227a90f4..8de5d5448f 100644 --- a/src/sqlbuild/executor/build/_helpers/source_audits.py +++ b/src/sqlbuild/executor/build/_helpers/source_audits.py @@ -10,6 +10,9 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import AuditPlanEntry, PlanOutput from sqlbuild.executor.auditing.main._execute import execute_audit +from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( + publish_completed_audit_results, +) from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build._helpers.blocking import downstream_blocked_keys @@ -79,6 +82,7 @@ def run_pending_source_audits( if result.outcome == AuditOutcome.ERROR: lifecycle.failed() audit_results.append(result) + publish_completed_audit_results((result,)) if result.outcome == AuditOutcome.ERROR: failed_names.append(source_name) newly_blocked.update( diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index fd0f9e1d57..f566f4adc2 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -42,6 +42,9 @@ from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( + publish_completed_audit_results, +) from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build._helpers.blocking import downstream_blocked_keys from sqlbuild.executor.build._helpers.end_audits import run_end_audits @@ -1200,6 +1203,7 @@ def _handle_completion( elif isinstance(result, ModelExecutionResult): self._model_results.append(result) + publish_completed_audit_results(result.audit_results) if self._on_node_complete is not None: self._on_node_complete(result) failed = result.status in {ExecutionStatus.FAILED, ExecutionStatus.SKIPPED} @@ -1295,6 +1299,7 @@ def _record_skipped_result( ) if isinstance(result, ModelExecutionResult): self._model_results.append(result) + publish_completed_audit_results(result.audit_results) elif isinstance(result, SeedExecutionResult): self._seed_results.append(result) elif isinstance(result, FunctionExecutionResult): diff --git a/src/sqlbuild/executor/build/main/_execute.py b/src/sqlbuild/executor/build/main/_execute.py index 4aed32c523..6ad0eb9052 100644 --- a/src/sqlbuild/executor/build/main/_execute.py +++ b/src/sqlbuild/executor/build/main/_execute.py @@ -12,6 +12,9 @@ AuditResultProjection, project_audit_result_batch, ) +from sqlbuild.executor.auditing.main.audit_result_publication_scope import ( + audit_result_publication_scope, +) from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build._helpers.retention import ( apply_retention_phase, @@ -83,15 +86,27 @@ def execute_build_plan( test_results: tuple[SqlTestExecutionResult, ...] source_audit_results: tuple[AuditExecutionResult, ...] end_audit_results: tuple[AuditExecutionResult, ...] - ( - model_results, - seed_results, - function_results, - load_results, - test_results, - source_audit_results, - end_audit_results, - ) = scheduler.run() + with audit_result_publication_scope(plan=plan): + ( + model_results, + seed_results, + function_results, + load_results, + test_results, + source_audit_results, + end_audit_results, + ) = scheduler.run() + all_audit_results: tuple[AuditExecutionResult, ...] = _all_audit_results( + model_results=model_results, + source_audit_results=source_audit_results, + end_audit_results=end_audit_results, + ) + projection: AuditResultProjection = project_audit_result_batch( + plan=plan, + results=all_audit_results, + adapter=adapter, + connection=scheduler_connection, + ) result: BuildExecutionResult = aggregate_build_result( model_results=model_results, @@ -102,17 +117,6 @@ def execute_build_plan( source_audit_results=source_audit_results, end_audit_results=end_audit_results, ) - all_audit_results: tuple[AuditExecutionResult, ...] = _all_audit_results( - model_results=model_results, - source_audit_results=source_audit_results, - end_audit_results=end_audit_results, - ) - projection: AuditResultProjection = project_audit_result_batch( - plan=plan, - results=all_audit_results, - adapter=adapter, - connection=scheduler_connection, - ) result = replace( result, audit_result_projection_attempted_count=projection.attempted_count, diff --git a/src/sqlbuild/executor/pipeline/_helpers/auditing.py b/src/sqlbuild/executor/pipeline/_helpers/auditing.py index 7a175946d1..f645564581 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/auditing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/auditing.py @@ -23,6 +23,12 @@ from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.executor.auditing.main._execute import execute_audit from sqlbuild.executor.auditing.main._project_results import project_audit_result_batch +from sqlbuild.executor.auditing.main.audit_result_publication_scope import ( + audit_result_publication_scope, +) +from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( + publish_completed_audit_results, +) from sqlbuild.executor.auditing.main.resource_id import audit_resource_id from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.pipeline._helpers.connections import ( @@ -76,6 +82,7 @@ def consume( callbacks.on_audit_error(entries[completion.index]) elif completion.result is not None: self.results[completion.index] = completion.result + publish_completed_audit_results((completion.result,)) if callbacks.on_audit_physical_complete is not None: callbacks.on_audit_physical_complete(completion.result) except BaseException as error: @@ -299,33 +306,38 @@ def _run_audits( callbacks.on_connection_complete( worker_count, elapsed_seconds=time.monotonic() - connection_started ) - if worker_count == 1: - results: tuple[AuditExecutionResult, ...] = _run_serial_audits( - entries=entries, + with audit_result_publication_scope( + plan=plan, + storage_database=storage_database, + storage_schema=storage_schema, + ): + if worker_count == 1: + results: tuple[AuditExecutionResult, ...] = _run_serial_audits( + entries=entries, + plan=plan, + adapter=adapter, + connection=connections[0], + callbacks=callbacks, + run_id=run_id, + ) + else: + results = _run_concurrent_audits( + entries=entries, + plan=plan, + adapter=adapter, + connections=connections, + worker_count=worker_count, + callbacks=callbacks, + run_id=run_id, + ) + project_audit_result_batch( plan=plan, + results=results, adapter=adapter, connection=connections[0], - callbacks=callbacks, - run_id=run_id, - ) - else: - results = _run_concurrent_audits( - entries=entries, - plan=plan, - adapter=adapter, - connections=connections, - worker_count=worker_count, - callbacks=callbacks, - run_id=run_id, + storage_database=storage_database, + storage_schema=storage_schema, ) - project_audit_result_batch( - plan=plan, - results=results, - adapter=adapter, - connection=connections[0], - storage_database=storage_database, - storage_schema=storage_schema, - ) return results except BaseException as error: active_error = error diff --git a/src/sqlbuild/executor/testing/_helpers/native_requests.py b/src/sqlbuild/executor/testing/_helpers/native_requests.py index b78603822f..37e3566c0d 100644 --- a/src/sqlbuild/executor/testing/_helpers/native_requests.py +++ b/src/sqlbuild/executor/testing/_helpers/native_requests.py @@ -34,6 +34,7 @@ def chain_step_request(*, step: ChainStep) -> dict[str, object]: "liftedCtes": step.lifted_ctes, "comparisonBodySql": step.comparison_body_sql, "expectedColumns": step.expected_columns, + "expectedLiftedCtes": step.expected_lifted_ctes, } diff --git a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py index 9fc23e3aa6..626acd3757 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/catalogue.py @@ -20,6 +20,7 @@ CONTRACT_BOOLEAN_NAME_CODE, CONTRACT_DATE_NAME_CODE, CONTRACT_TIMESTAMP_NAME_CODE, + EMPTY_INPUT_TEST_CODE, INIT_MODULE_NAME, RULE_DECORATOR_TOKEN, ) @@ -139,6 +140,20 @@ def _builtin_options(code: str) -> tuple[RuleOption[object], ...]: ), ), ) + if code == EMPTY_INPUT_TEST_CODE: + return ( + cast( + RuleOption[object], + RuleOption.string_list( + name="allowed_tests", + default=(), + description=( + "Reviewed SQL test names that may assert only that empty inputs produce " + "no rows; listed tests count toward min_tests_per_model." + ), + ), + ), + ) return () diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 5fd3a7e7b6..51ec3a3b6a 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -23,9 +23,11 @@ CompactLineageFacts, CompiledLineageColumnFact, CompiledModel, + CompiledModelSqlTestPayload, CompiledProject, CompiledSqlScenario, CompiledSqlTest, + CompileSqlTestCte, DynamicColumnContractProof, InferredColumn, ) @@ -340,6 +342,16 @@ def _sql_test_payload(test: CompiledSqlTest) -> dict[str, object]: for resource in test.tested_resources ], } + if isinstance(test.payload, CompiledModelSqlTestPayload): + payload.update( + { + "authored_ctes": _sql_test_cte_payloads(test.payload.authored_ctes), + "expected_ctes": _sql_test_cte_payloads(test.payload.expected_ctes), + "assertion_ctes": _sql_test_cte_payloads(test.payload.assertion_ctes), + "has_macro_mocks": bool(test.payload.macro_mocks), + "has_model_query_overrides": bool(test.payload.model_query_overrides), + } + ) if test.case_name is not None: parameter_types: dict[str, str] = { parameter.name: parameter.value_type.value for parameter in test.parameter_schema @@ -371,6 +383,10 @@ def _sql_test_payload(test: CompiledSqlTest) -> dict[str, object]: return payload +def _sql_test_cte_payloads(ctes: tuple[CompileSqlTestCte, ...]) -> list[dict[str, str]]: + return [{"name": cte.name, "sql": cte.sql_body} for cte in ctes] + + def _sql_scenario_payloads(project: CompiledProject) -> list[dict[str, object]]: return [_sql_scenario_payload(scenario) for scenario in project.sql_scenarios] diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index 7442e15643..e435ae04b8 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -29,6 +29,7 @@ CONTRACT_BOOLEAN_NAME_CODE: str = "SQBRCONTRACT102" CONTRACT_TIMESTAMP_NAME_CODE: str = "SQBRCONTRACT103" CONTRACT_DATE_NAME_CODE: str = "SQBRCONTRACT104" +EMPTY_INPUT_TEST_CODE: str = "SQBRTEST203" MATERIALIZED_VIEW: str = "view" REFERENCE_KIND_REF: str = "ref" LAYER_INT_CLEAN: str = "int_clean" diff --git a/src/sqlbuild/runtime/event_exporting/README.md b/src/sqlbuild/runtime/event_exporting/README.md index 62a0c21254..76b6c2fc03 100644 --- a/src/sqlbuild/runtime/event_exporting/README.md +++ b/src/sqlbuild/runtime/event_exporting/README.md @@ -11,7 +11,8 @@ the invocation timeout; after a timeout that exporter is not invoked again and i are dropped. This bounds concurrently hung threads to the number of exporters, but Python cannot terminate those daemon threads or undo destination side effects if they eventually return. -Shutdown stops acceptance, drains only until the shared deadline, accounts queued and unattempted +Shutdown stops acceptance, drains only until the shared deadline (CLI commands use +`sinks.lifecycle.shutdown_timeout`, default two seconds), accounts queued and unattempted deliveries as dropped, terminates the event dispatcher, and returns without waiting for timed-out invocation threads. Concurrent shutdown callers receive the same cached final accounting snapshot. The shared provider session is retained while any invocation remains live. Teardown runs exactly diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index 9052288303..ae9e9d557d 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -379,10 +379,12 @@ class LifecycleEventSinkFilterConfig: @dataclass(frozen=True) class LifecycleEventSinksConfig: - """Global and named lifecycle-event sink runtime filters.""" + """Global and named lifecycle-event sink runtime filters and delivery settings.""" defaults: LifecycleEventSinkFilterConfig = field(default_factory=LifecycleEventSinkFilterConfig) named: dict[str, LifecycleEventSinkFilterConfig] = field(default_factory=dict) + shutdown_timeout: str | None = None + shutdown_timeout_seconds: int | None = None @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py index a3e6481059..3aa603d3f1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py @@ -14,3 +14,40 @@ class AuditE2ETestCase: expected_stdout_fragment: str expected_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) expected_ordered_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class IncrementalAuditEventE2ETestCase: + """One command whose audit_completed events must arrive during execution.""" + + description: str + setup_command: tuple[str, ...] + command: tuple[str, ...] + expected_audit_count: int + + +@dataclass(frozen=True) +class SlowSinkDropE2ETestCase: + """A short lifecycle drain against a deliberately slow sink.""" + + description: str + shutdown_timeout: str + expected_accepted: int + + +@dataclass(frozen=True) +class SlowSinkFullDeliveryE2ETestCase: + """A generous lifecycle drain against a deliberately slow sink.""" + + description: str + shutdown_timeout: str + expected_delivered: int + + +@dataclass(frozen=True) +class InvalidDrainTimeoutE2ETestCase: + """One invalid lifecycle drain timeout value.""" + + description: str + shutdown_timeout_toml: str + expected_error: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py new file mode 100644 index 0000000000..9741150667 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/helpers.py @@ -0,0 +1,96 @@ +"""Local helpers for audit command E2E tests.""" + +from __future__ import annotations + +import json +from pathlib import Path +from textwrap import dedent + +ORDER_COLUMNS: tuple[str, ...] = tuple(f"order_attribute_{index:02d}" for index in range(12)) + + +def delivery_project_toml(*, lifecycle_config: str) -> str: + """Return a DuckDB project config with optional lifecycle sink settings.""" + + return ( + dedent( + """ + name = "audit_event_delivery" + adapter = "duckdb" + + [connection] + database = "audit_event_delivery.duckdb" + """ + ).lstrip() + + lifecycle_config + ) + + +def delivery_model_files() -> dict[str, str]: + """Return a customers -> orders chain carrying one plus twelve column audits.""" + + columns: str = "\n".join(f" {name} (audits [not_null])," for name in ORDER_COLUMNS) + selected: str = ",\n ".join(f"1 AS {name}" for name in ORDER_COLUMNS) + return { + "models/customers.sql": dedent( + """ + MODEL ( + materialized table, + columns ( + customer_id (audits [not_null]), + ), + ); + + SELECT 1 AS customer_id + """ + ).lstrip(), + "models/orders.sql": ( + "MODEL (\n materialized table,\n columns (\n" + f"{columns}\n" + " ),\n);\n\n" + f"SELECT\n customer_id,\n {selected}\n" + 'FROM __ref("customers")\n' + ), + } + + +def recording_sink_source(*, event_kinds: str, delay_seconds: float) -> str: + """Return a lifecycle sink that appends each event to ORDERS_EVENT_PATH after a delay.""" + + return dedent( + f""" + import os + import time + from pathlib import Path + + from sqlbuild.sinks import ( + LifecycleEvent, + LifecycleEventKind, + lifecycle_event_sink, + lifecycle_event_to_json, + ) + + + @lifecycle_event_sink(name="slow_orders_events", event_kinds={event_kinds}) + def record_event(event: LifecycleEvent) -> None: + time.sleep({delay_seconds}) + path = Path(os.environ["ORDERS_EVENT_PATH"]) + with path.open("a", encoding="utf-8") as stream: + stream.write(lifecycle_event_to_json(event) + "\\n") + """ + ).lstrip() + + +def read_sequenced_events(path: Path) -> list[dict[str, object]]: + """Return recorded events ordered by canonical invocation sequence.""" + + events: list[dict[str, object]] = [ + json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() + ] + return sorted(events, key=lambda event: int(str(event["invocation_sequence"]))) + + +def events_of_type(events: list[dict[str, object]], event_type: str) -> list[dict[str, object]]: + """Return events with one lifecycle type in their existing order.""" + + return list(filter(lambda event: event["event_type"] == event_type, events)) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py new file mode 100644 index 0000000000..7c213b7fe7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py @@ -0,0 +1,263 @@ +"""E2E coverage for incremental audit lifecycle delivery and sink drain accounting.""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.audit._test_types import ( + IncrementalAuditEventE2ETestCase, + InvalidDrainTimeoutE2ETestCase, + SlowSinkDropE2ETestCase, + SlowSinkFullDeliveryE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.audit.helpers import ( + delivery_model_files, + delivery_project_toml, + events_of_type, + read_sequenced_events, + recording_sink_source, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_WARNING_PATTERN: re.Pattern[str] = re.compile( + r"Warning: command completed successfully, but lifecycle event export was incomplete: " + r"(?P\d+) of (?P\d+) events dropped, (?P\d+) failed " + r"\(sink 'slow_orders_events'\)\. " + r"Increase sinks\.lifecycle\.shutdown_timeout or check sink health\." +) + + +@pytest.mark.parametrize( + "test_case", + ( + IncrementalAuditEventE2ETestCase( + "serial audit command", + ("build", "--no-audits"), + ("audit", "--concurrency", "1"), + 13, + ), + IncrementalAuditEventE2ETestCase( + "concurrent audit command", + ("build", "--no-audits"), + ("audit", "--concurrency", "3"), + 13, + ), + IncrementalAuditEventE2ETestCase("build with model audits", ("compile",), ("build",), 13), + ), + ids=lambda case: case.description, +) +def test_given_many_audits_when_running_then_audit_completed_events_arrive_during_execution( + test_case: IncrementalAuditEventE2ETestCase, + tmp_path: Path, +) -> None: + event_path: Path = tmp_path / "orders-events.jsonl" + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="audit_event_delivery", + repo_files={ + "sqlbuild_project.toml": delivery_project_toml(lifecycle_config=""), + **delivery_model_files(), + "sinks/orders_events.py": recording_sink_source( + event_kinds="{LifecycleEventKind.AUDIT, LifecycleEventKind.RESOURCE}", + delay_seconds=0, + ), + }, + ) + setup: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.setup_command), + project_dir=project_dir, + env={"ORDERS_EVENT_PATH": str(tmp_path / "setup-events.jsonl")}, + ) + assert setup.returncode == 0, setup.stdout + setup.stderr + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), + project_dir=project_dir, + env={"ORDERS_EVENT_PATH": str(event_path)}, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "event export was incomplete" not in result.stderr + events: list[dict[str, object]] = read_sequenced_events(event_path) + audit_events: list[dict[str, object]] = events_of_type(events, "audit_completed") + resource_starts: list[dict[str, object]] = events_of_type(events, "resource_attempt_started") + result_ids: set[str] = { + str(json.loads(json.dumps(event["payload"]))["result_id"]) for event in audit_events + } + persisted: list[tuple[object, ...]] = query_duckdb( + db_path=project_dir / "audit_event_delivery.duckdb", + sql=( + "SELECT result_id FROM main._sqlbuild_audit_results " + f"WHERE invocation_id = '{audit_events[0]['invocation_id']}'" + ), + ) + assert len(audit_events) == test_case.expected_audit_count + assert len(result_ids) == test_case.expected_audit_count + assert {str(row[0]) for row in persisted} == result_ids + assert int(str(audit_events[0]["invocation_sequence"])) < int( + str(resource_starts[-1]["invocation_sequence"]) + ) + assert {event["resource_id"] for event in audit_events} == {None} + + +@pytest.mark.parametrize( + "test_case", + (SlowSinkDropE2ETestCase("zero drain drops queued events", "0s", 13),), + ids=lambda case: case.description, +) +def test_given_slow_sink_and_zero_drain_when_auditing_then_warns_with_counts_and_succeeds( + test_case: SlowSinkDropE2ETestCase, + tmp_path: Path, +) -> None: + event_path: Path = tmp_path / "orders-events.jsonl" + event_path.touch() + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="audit_event_delivery", + repo_files={ + "sqlbuild_project.toml": delivery_project_toml( + lifecycle_config=( + f'\n[sinks.lifecycle]\nshutdown_timeout = "{test_case.shutdown_timeout}"\n' + ) + ), + **delivery_model_files(), + "sinks/orders_events.py": recording_sink_source( + event_kinds="{LifecycleEventKind.AUDIT}", delay_seconds=0.4 + ), + }, + ) + setup: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--no-audits"), + project_dir=project_dir, + env={"ORDERS_EVENT_PATH": str(tmp_path / "setup-events.jsonl")}, + ) + assert setup.returncode == 0, setup.stdout + setup.stderr + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "audit", "--concurrency", "1", "--json"), + project_dir=project_dir, + env={"ORDERS_EVENT_PATH": str(event_path)}, + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, object] = json.loads(result.stdout) + match: re.Match[str] | None = _WARNING_PATTERN.search(result.stderr) + assert match is not None, result.stderr + dropped: int = int(match.group("dropped")) + failed: int = int(match.group("failed")) + delivered_lines: int = len(read_sequenced_events(event_path)) + assert payload["summary"] is not None + assert int(match.group("accepted")) == test_case.expected_accepted + assert dropped + failed > 0 + assert test_case.expected_accepted - dropped - failed <= delivered_lines + assert delivered_lines <= test_case.expected_accepted - dropped + assert result.stderr.count("event export was incomplete") == 1 + + +@pytest.mark.parametrize( + "test_case", + (SlowSinkFullDeliveryE2ETestCase("raised drain delivers every event", "60s", 13),), + ids=lambda case: case.description, +) +def test_given_slow_sink_and_raised_drain_when_auditing_then_delivers_all_without_warning( + test_case: SlowSinkFullDeliveryE2ETestCase, + tmp_path: Path, +) -> None: + event_path: Path = tmp_path / "orders-events.jsonl" + event_path.touch() + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="audit_event_delivery", + repo_files={ + "sqlbuild_project.toml": delivery_project_toml( + lifecycle_config=( + f'\n[sinks.lifecycle]\nshutdown_timeout = "{test_case.shutdown_timeout}"\n' + ) + ), + **delivery_model_files(), + "sinks/orders_events.py": recording_sink_source( + event_kinds="{LifecycleEventKind.AUDIT}", delay_seconds=0.4 + ), + }, + ) + setup: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--no-audits"), + project_dir=project_dir, + env={"ORDERS_EVENT_PATH": str(tmp_path / "setup-events.jsonl")}, + ) + assert setup.returncode == 0, setup.stdout + setup.stderr + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "audit", "--concurrency", "1", "--json"), + project_dir=project_dir, + env={"ORDERS_EVENT_PATH": str(event_path)}, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert json.loads(result.stdout)["summary"] is not None + assert "event export was incomplete" not in result.stderr + assert len(read_sequenced_events(event_path)) == test_case.expected_delivered + + +@pytest.mark.parametrize( + "test_case", + ( + InvalidDrainTimeoutE2ETestCase( + "negative duration", + '"-5s"', + "sinks.lifecycle.shutdown_timeout must be a fixed duration from '0s' to '10m'", + ), + InvalidDrainTimeoutE2ETestCase( + "bare number", + "5", + "sinks.lifecycle.shutdown_timeout must be a fixed duration from '0s' to '10m'", + ), + InvalidDrainTimeoutE2ETestCase( + "above upper bound", + '"1h"', + "sinks.lifecycle.shutdown_timeout must be a fixed duration from '0s' to '10m'", + ), + ), + ids=lambda case: case.description, +) +def test_given_invalid_drain_timeout_when_auditing_then_fails_with_config_error( + test_case: InvalidDrainTimeoutE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="audit_event_delivery", + repo_files={ + "sqlbuild_project.toml": delivery_project_toml( + lifecycle_config=( + f"\n[sinks.lifecycle]\nshutdown_timeout = {test_case.shutdown_timeout_toml}\n" + ) + ), + **delivery_model_files(), + "sinks/orders_events.py": recording_sink_source( + event_kinds="{LifecycleEventKind.AUDIT}", delay_seconds=0 + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "audit"), + project_dir=project_dir, + env={"ORDERS_EVENT_PATH": str(tmp_path / "orders-events.jsonl")}, + ) + + assert result.returncode != 0 + assert test_case.expected_error in result.stdout + result.stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 168c186a49..4fc2e65bff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -196,3 +196,34 @@ class DenseCompileGuardTestCase: expected_max_wall_seconds: float expected_max_rss_bytes: int expected_fingerprint: str + + +@dataclass(frozen=True) +class RuleGatedTestDiagnosticsTestCase: + description: str + filler_model_count: int + extra_args: tuple[str, ...] + expected_diagnostics: tuple[tuple[str, str, str], ...] + expected_text_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class EmptyInputTestRuleCompileTestCase: + description: str + select: tuple[str, ...] + expected_diagnostics: frozenset[tuple[str, str, str]] + + +@dataclass(frozen=True) +class EmptyInputTestRuleOptionErrorTestCase: + description: str + allowed_tests_toml: str + expected_error: str + + +@dataclass(frozen=True) +class EmptyInputTestRuleCacheTestCase: + description: str + replacement_mock: str + expected_first_codes: tuple[str, ...] + expected_second_codes: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index f0cb744140..2f79a13c91 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -1799,3 +1799,113 @@ def _layered_pad_model_sql(*, sql: str, target_bytes: int, index: int) -> str: for line in range(line_count) ) return f"{sql.rstrip()}\n{comments}" + + +def build_rule_gated_test_project_files(*, filler_model_count: int) -> dict[str, str]: + """Build a project with one rule error and one SQL test that lacks a source mock.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "rule_gated_orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRMODEL102"]\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n schema: main\n table: raw_orders\n" + " - name: raw_refunds\n schema: main\n table: raw_refunds\n" + ), + "models/orders.sql": ( + "MODEL (materialized view);\n\n" + 'SELECT o.order_id, r.refund_id FROM __source("raw_orders") AS o\n' + 'LEFT JOIN __source("raw_refunds") AS r ON r.order_id = o.order_id\n' + ), + "models/bad_star.sql": ( + 'MODEL (materialized view);\n\nSELECT * FROM __source("raw_orders")\n' + ), + "tests/unit/orders_case.sql": ( + 'TEST (name "orders_case");\n' + "WITH\n" + "__source__raw_orders AS (SELECT 1 AS order_id),\n" + "__expected__orders AS (SELECT 1 AS order_id)\n" + "SELECT 1\n" + ), + } + for index in range(filler_model_count): + files[f"models/filler/filler_{index:03d}.sql"] = ( + f"MODEL (materialized view);\n\nSELECT {index} AS filler_id\n" + ) + return files + + +def build_empty_input_test_project_files( + *, select: tuple[str, ...], allowed_tests_toml: str +) -> dict[str, str]: + """Build an orders project mixing empty-input-only filler with real SQL tests.""" + + selected: str = ", ".join(f'"{code}"' for code in select) + empty_orders: str = ( + "__source__raw_orders AS (\n" + " SELECT CAST(NULL AS INTEGER) AS order_id, CAST(NULL AS INTEGER) AS amount\n" + " WHERE FALSE\n" + ")" + ) + return { + "sqlbuild_project.toml": ( + 'name = "empty_input_orders"\nadapter = "duckdb"\n\n' + f"[rules]\nselect = [{selected}]\n\n" + "[rules.thresholds]\nmin_tests_per_model = 1\n\n" + f"[rules.rule_options.SQBRTEST203]\nallowed_tests = {allowed_tests_toml}\n" + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + ), + "models/orders.sql": ( + "MODEL (materialized view);\n\n" + "SELECT order_id, amount * 2 AS doubled_amount\n" + 'FROM __source("raw_orders")\nWHERE amount > 0\n' + ), + "models/large_orders.sql": ( + "MODEL (materialized view);\n\n" + 'SELECT order_id FROM __source("raw_orders") WHERE amount > 100\n' + ), + "models/customers.sql": ( + "MODEL (materialized view);\n\n" + 'SELECT order_id AS customer_id FROM __source("raw_orders") WHERE amount > 10\n' + ), + "models/order_summary.sql": ( + "MODEL (materialized view);\n\n" + "SELECT COUNT(*) AS order_count, COALESCE(SUM(amount), 0) AS total_amount\n" + 'FROM __source("raw_orders")\n' + ), + "tests/unit/test_orders__empty_inputs_produce_no_rows.sql": ( + 'TEST (name "orders__empty_inputs_produce_no_rows");\n\n' + f"WITH {empty_orders}, __assert__empty_inputs_produce_no_rows AS (\n" + ' SELECT 1 AS unexpected_row FROM __ref("orders")\n' + ")\nSELECT 1\n" + ), + "tests/unit/test_large_orders__empty_inputs_produce_no_rows.sql": ( + 'TEST (name "large_orders__empty_inputs_produce_no_rows");\n\n' + f"WITH {empty_orders}, __expected__large_orders AS (\n" + " SELECT * FROM __EMPTY_FIXTURE()\n" + ")\nSELECT 1\n" + ), + "tests/unit/test_large_orders__keeps_orders_over_threshold.sql": ( + 'TEST (name "large_orders__keeps_orders_over_threshold");\n\n' + "WITH __source__raw_orders AS (\n" + " SELECT 1 AS order_id, 150 AS amount UNION ALL SELECT 2 AS order_id, 20 AS amount\n" + "), __expected__large_orders AS (\n" + " SELECT 1 AS order_id\n" + ")\nSELECT 1\n" + ), + "tests/unit/test_customers__reviewed_empty_input.sql": ( + 'TEST (name "customers__reviewed_empty_input");\n\n' + f"WITH {empty_orders}, __assert__no_customers AS (\n" + ' SELECT customer_id FROM __ref("customers")\n' + ")\nSELECT 1\n" + ), + "tests/unit/test_order_summary__empty_inputs_return_zero_row.sql": ( + 'TEST (name "order_summary__empty_inputs_return_zero_row");\n\n' + f"WITH {empty_orders}, __expected__order_summary AS (\n" + " SELECT 0 AS order_count, 0 AS total_amount\n" + ")\nSELECT 1\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_rule_empty_input_tests.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_rule_empty_input_tests.py new file mode 100644 index 0000000000..4412cbd468 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_rule_empty_input_tests.py @@ -0,0 +1,191 @@ +"""E2E tests for the empty-input-only SQL test rule and its minimum-test interaction.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + EmptyInputTestRuleCacheTestCase, + EmptyInputTestRuleCompileTestCase, + EmptyInputTestRuleOptionErrorTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + build_empty_input_test_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_ALLOWED_TESTS: str = '["customers__reviewed_empty_input", "orders__retired_filler"]' +_FLAGGED_MESSAGE: str = ( + 'unit test block 1 ("{name}") mocks only empty inputs and asserts only that no rows ' + "are produced" +) +_ORDERS_FILLER: tuple[str, str, str] = ( + "SQBRTEST203", + "tests/unit/test_orders__empty_inputs_produce_no_rows.sql", + _FLAGGED_MESSAGE.format(name="orders__empty_inputs_produce_no_rows"), +) +_LARGE_ORDERS_FILLER: tuple[str, str, str] = ( + "SQBRTEST203", + "tests/unit/test_large_orders__empty_inputs_produce_no_rows.sql", + _FLAGGED_MESSAGE.format(name="large_orders__empty_inputs_produce_no_rows"), +) +_STALE_ENTRY: tuple[str, str, str] = ( + "SQBRTEST203", + "sqlbuild_project.toml", + 'stale allowed_tests entry "orders__retired_filler" names no existing SQL test', +) +_ORDERS_MINIMUM: tuple[str, str, str] = ( + "SQBRTEST202", + "models/orders.sql", + 'model "orders" has 0 tests; 1 required (1 empty-input-only test not counted; see SQBRTEST203)', +) + + +@pytest.mark.parametrize( + "test_case", + ( + EmptyInputTestRuleCompileTestCase( + description="empty-input rule and minimum tests selected", + select=("SQBRTEST202", "SQBRTEST203"), + expected_diagnostics=frozenset( + {_ORDERS_FILLER, _LARGE_ORDERS_FILLER, _STALE_ENTRY, _ORDERS_MINIMUM} + ), + ), + EmptyInputTestRuleCompileTestCase( + description="minimum tests selected alone still excludes filler", + select=("SQBRTEST202",), + expected_diagnostics=frozenset({_ORDERS_MINIMUM}), + ), + ), + ids=lambda case: case.description, +) +def test_given_empty_input_only_tests_when_compiling_then_rules_report_filler_and_coverage( + test_case: EmptyInputTestRuleCompileTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="empty_input_orders", + repo_files=build_empty_input_test_project_files( + select=test_case.select, allowed_tests_toml=_ALLOWED_TESTS + ), + ) + + text_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + json_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + + text_output: str = text_result.stdout + text_result.stderr + assert text_result.returncode == 1, text_output + for code, path, message in test_case.expected_diagnostics: + assert f"error[{code}]: {message}" in text_result.stdout, text_output + assert f"--> {path}:1:1" in text_result.stdout, text_output + assert ( + f"4 models, 0 seeds, 0 functions, {len(test_case.expected_diagnostics)} error" + in text_result.stdout + ), text_output + assert json_result.returncode == 1, json_result.stdout + json_result.stderr + payload: dict[str, Any] = json.loads(json_result.stdout) + assert payload["summary"]["errors"] == len(test_case.expected_diagnostics) + assert { + (diagnostic["code"], diagnostic["path"], diagnostic["message"]) + for diagnostic in payload["diagnostics"] + } == test_case.expected_diagnostics + + +@pytest.mark.parametrize( + "test_case", + ( + EmptyInputTestRuleOptionErrorTestCase( + description="string instead of a list", + allowed_tests_toml='"customers__reviewed_empty_input"', + expected_error="rule SQBRTEST203 option allowed_tests has the wrong type", + ), + EmptyInputTestRuleOptionErrorTestCase( + description="non-string list items", + allowed_tests_toml="[1]", + expected_error=( + "rule SQBRTEST203 option allowed_tests has item values of the wrong type" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_invalid_allowed_tests_option_when_compiling_then_reports_config_error( + test_case: EmptyInputTestRuleOptionErrorTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="empty_input_orders", + repo_files=build_empty_input_test_project_files( + select=("SQBRTEST202", "SQBRTEST203"), + allowed_tests_toml=test_case.allowed_tests_toml, + ), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 1, output + assert test_case.expected_error in output, output + + +@pytest.mark.parametrize( + "test_case", + ( + EmptyInputTestRuleCacheTestCase( + description="filler edited to mock rows", + replacement_mock="SELECT 1 AS order_id, 5 AS amount", + expected_first_codes=("SQBRTEST202",), + expected_second_codes=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_warm_cache_when_filler_test_gains_rows_then_minimum_tests_recounts( + test_case: EmptyInputTestRuleCacheTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="empty_input_orders", + repo_files=build_empty_input_test_project_files( + select=("SQBRTEST202",), allowed_tests_toml='["customers__reviewed_empty_input"]' + ), + ) + test_path: Path = project_dir / "tests/unit/test_orders__empty_inputs_produce_no_rows.sql" + + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + source: str = test_path.read_text(encoding="utf-8") + empty_mock: str = ( + "SELECT CAST(NULL AS INTEGER) AS order_id, CAST(NULL AS INTEGER) AS amount\n WHERE FALSE" + ) + assert empty_mock in source + test_path.write_text(source.replace(empty_mock, test_case.replacement_mock), encoding="utf-8") + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + + first_payload: dict[str, Any] = json.loads(first.stdout) + second_payload: dict[str, Any] = json.loads(second.stdout) + assert ( + tuple(diagnostic["code"] for diagnostic in first_payload["diagnostics"]) + == test_case.expected_first_codes + ), first.stdout + assert ( + tuple(diagnostic["code"] for diagnostic in second_payload["diagnostics"]) + == test_case.expected_second_codes + ), second.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_rule_gate.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_rule_gate.py new file mode 100644 index 0000000000..969fdb1ae3 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_rule_gate.py @@ -0,0 +1,86 @@ +"""E2E tests for SQL test planning errors reported when compile rules fail.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + RuleGatedTestDiagnosticsTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + build_rule_gated_test_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_MISSING_MOCK_MESSAGE: str = ( + "test 'orders_case': model 'orders' references __source('raw_refunds') which has no mock" +) +_EXPECTED_DIAGNOSTICS: tuple[tuple[str, str, str], ...] = ( + ("rule", "SQBRMODEL102", "SELECT * is allowed only inside dependency import CTEs"), + ("test", "S000", _MISSING_MOCK_MESSAGE), +) +_EXPECTED_TEXT_LINES: tuple[str, ...] = ( + "error[SQBRMODEL102]: SELECT * is allowed only inside dependency import CTEs", + f"error[S000]: {_MISSING_MOCK_MESSAGE}", +) + + +@pytest.mark.parametrize( + "test_case", + ( + RuleGatedTestDiagnosticsTestCase( + description="inline artifact writer", + filler_model_count=0, + extra_args=(), + expected_diagnostics=_EXPECTED_DIAGNOSTICS, + expected_text_lines=_EXPECTED_TEXT_LINES, + ), + RuleGatedTestDiagnosticsTestCase( + description="background artifact preparation", + filler_model_count=130, + extra_args=("--no-cache",), + expected_diagnostics=_EXPECTED_DIAGNOSTICS, + expected_text_lines=_EXPECTED_TEXT_LINES, + ), + ), + ids=lambda case: case.description, +) +def test_given_rule_error_and_test_missing_mock_when_compiling_then_both_errors_are_reported( + test_case: RuleGatedTestDiagnosticsTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="rule_gated_project", + repo_files=build_rule_gated_test_project_files( + filler_model_count=test_case.filler_model_count + ), + ) + + text_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", *test_case.extra_args), project_dir=project_dir + ) + json_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json", *test_case.extra_args), + project_dir=project_dir, + ) + + text_output: str = text_result.stdout + text_result.stderr + assert text_result.returncode == 1, text_output + for line in test_case.expected_text_lines: + assert text_result.stdout.count(line) == 1, text_output + assert "2 errors, 0 warnings" in text_result.stdout, text_output + assert json_result.returncode == 1, json_result.stdout + json_result.stderr + payload: dict[str, Any] = json.loads(json_result.stdout) + assert payload["has_errors"] is True + assert payload["summary"]["errors"] == len(test_case.expected_diagnostics) + assert [ + (diagnostic["phase"], diagnostic["code"], diagnostic["message"]) + for diagnostic in payload["diagnostics"] + ] == list(test_case.expected_diagnostics) + assert not (project_dir / "target" / "compiled" / "tests").exists() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index e1cc71bf57..d75c81bf04 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -156,3 +156,23 @@ class MissingMockCliE2ETestCase: expected_error: str compile_error_line: str compile_summary_fragment: str + + +@dataclass(frozen=True) +class HelperScopeE2ETestCase: + """Test case for helper CTEs referenced by expected and assertion SQL.""" + + description: str + command: tuple[str, ...] + expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class CursorWindowE2ETestCase: + """Test case for SQL tests of models that use cursor intrinsics.""" + + description: str + tests: dict[str, str] + expected_exit_code: int + expected_output_fragments: tuple[str, ...] + expected_compiled_fragments: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 19f394a14f..e196fd3377 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -1185,3 +1185,260 @@ def build_expected_column_subset_project_files( "SELECT 1\n" ) return files + + +_HELPER_SCOPE_SOURCE_MOCK: str = ( + "__source__raw_orders AS (SELECT 1 AS id, 10 AS amount UNION ALL SELECT 2 AS id, 20 AS amount)" +) +_HELPER_SCOPE_TESTS: dict[str, str] = { + "helper_in_assert_passes": ( + f"{_HELPER_SCOPE_SOURCE_MOCK},\n" + "expected_rows AS (SELECT 1 AS order_id, 20 AS amount UNION ALL SELECT 2, 40),\n" + "__assert__all_expected_rows_built AS (\n" + " SELECT order_id, amount FROM expected_rows\n" + ' EXCEPT SELECT order_id, amount FROM __ref("orders")\n' + ")\n" + ), + "helper_in_assert_fails": ( + f"{_HELPER_SCOPE_SOURCE_MOCK},\n" + "expected_rows AS (SELECT 1 AS order_id, 20 AS amount UNION ALL SELECT 3, 60),\n" + "__assert__all_expected_rows_built AS (\n" + " SELECT order_id, amount FROM expected_rows\n" + ' EXCEPT SELECT order_id, amount FROM __ref("orders")\n' + ")\n" + ), + "helper_in_expected_passes": ( + f"{_HELPER_SCOPE_SOURCE_MOCK},\n" + "expected_rows AS (SELECT 1 AS order_id, 20 AS amount UNION ALL SELECT 2, 40),\n" + "__expected__orders AS (SELECT order_id, amount FROM expected_rows)\n" + ), + "helper_in_expected_fails": ( + f"{_HELPER_SCOPE_SOURCE_MOCK},\n" + "expected_rows AS (SELECT 1 AS order_id, 20 AS amount UNION ALL SELECT 2, 41),\n" + "__expected__orders AS (SELECT order_id, amount FROM expected_rows)\n" + ), + "mock_reads_helper_reading_mock": ( + "__source__raw_orders AS (SELECT 1 AS id, 10 AS amount UNION ALL SELECT 2, 20),\n" + "base_rows AS (SELECT id AS order_id, amount FROM __source__raw_orders),\n" + "__ref__stg_orders AS (SELECT order_id, amount FROM base_rows),\n" + "expected_rows AS (SELECT order_id, amount * 2 AS amount FROM __ref__stg_orders),\n" + "__expected__orders AS (SELECT order_id, amount FROM expected_rows),\n" + "__assert__every_order_built AS (\n" + " SELECT order_id, amount FROM expected_rows\n" + ' EXCEPT SELECT order_id, amount FROM __ref("orders")\n' + ")\n" + ), + "helper_reads_mock_and_helper": ( + "__ref__stg_orders AS (SELECT 1 AS order_id, 10 AS amount UNION ALL SELECT 2, 20),\n" + "doubled AS (SELECT order_id, amount * 2 AS amount FROM __ref__stg_orders),\n" + "expected_rows AS (SELECT order_id, amount FROM doubled),\n" + "__expected__orders AS (SELECT order_id, amount FROM expected_rows),\n" + "__assert__every_order_doubled AS (\n" + " SELECT order_id, amount FROM expected_rows\n" + ' EXCEPT SELECT order_id, amount FROM __ref("orders")\n' + ")\n" + ), +} + + +def build_helper_scope_project_files() -> dict[str, str]: + """Build an orders chain whose SQL tests read helper CTEs from expected and assertion SQL.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "helper_scope_demo"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "helper_scope_demo.duckdb"\n\n' + "[defaults]\n" + 'materialized = "table"\n' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + ), + "models/stg_orders.sql": ( + "MODEL (materialized table);\n\n" + 'SELECT id AS order_id, amount FROM __source("raw_orders")\n' + ), + "models/orders.sql": ( + "MODEL (materialized table);\n\n" + 'SELECT order_id, amount * 2 AS amount FROM __ref("stg_orders")\n' + ), + } + for test_name, ctes in _HELPER_SCOPE_TESTS.items(): + files[f"tests/unit/{test_name}.sql"] = ( + f'TEST (name "{test_name}");\n\nWITH\n{ctes}SELECT 1\n' + ) + return files + + +_CURSOR_ORDER_ROWS: str = ( + "__source__raw_orders AS (\n" + " SELECT TIMESTAMP '1990-06-01' AS order_date, 1 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-01-31' AS order_date, 2 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-02-01' AS order_date, 3 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-02-02' AS order_date, 4 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-02-03' AS order_date, 5 AS amount\n" + ")" +) +_CURSOR_EVENT_ROWS: str = ( + "__source__raw_events AS (\n" + " SELECT 5 AS event_id, 1 AS amount\n" + " UNION ALL SELECT 10 AS event_id, 2 AS amount\n" + " UNION ALL SELECT 20 AS event_id, 3 AS amount\n" + ")" +) +_ALL_ORDER_DATES: str = ( + "SELECT TIMESTAMP '1990-06-01' AS order_date, 1 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-01-31' AS order_date, 2 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-02-01' AS order_date, 3 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-02-02' AS order_date, 4 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-02-03' AS order_date, 5 AS amount" +) +CURSOR_WINDOW_PASSING_TESTS: dict[str, str] = { + "default_window_keeps_every_row": ( + 'TEST (name "default_window_keeps_every_row");\n\nWITH\n' + f"{_CURSOR_ORDER_ROWS},\n" + f"__expected__daily_orders AS (\n {_ALL_ORDER_DATES}\n)\n" + "SELECT 1\n" + ), + "default_window_supports_date_arithmetic": ( + 'TEST (name "default_window_supports_date_arithmetic");\n\nWITH\n' + f"{_CURSOR_ORDER_ROWS},\n" + "__expected__order_lookback AS (\n" + " SELECT DATE '1899-12-25' AS scan_start, DATE '3000-01-01' AS scan_end, 15 AS amount\n" + ")\n" + "SELECT 1\n" + ), + "declared_window_excludes_rows_outside": ( + 'TEST (name "declared_window_excludes_rows_outside", ' + 'cursor_start "2026-02-01", cursor_end "2026-02-03");\n\nWITH\n' + f"{_CURSOR_ORDER_ROWS},\n" + "__expected__daily_orders AS (\n" + " SELECT TIMESTAMP '2026-02-01' AS order_date, 3 AS amount\n" + " UNION ALL SELECT TIMESTAMP '2026-02-02' AS order_date, 4 AS amount\n" + ")\n" + "SELECT 1\n" + ), + "integer_default_window_keeps_every_row": ( + 'TEST (name "integer_default_window_keeps_every_row");\n\nWITH\n' + f"{_CURSOR_EVENT_ROWS},\n" + "__expected__event_totals AS (\n" + " SELECT 5 AS event_id, 1 AS amount\n" + " UNION ALL SELECT 10 AS event_id, 2 AS amount\n" + " UNION ALL SELECT 20 AS event_id, 3 AS amount\n" + ")\n" + "SELECT 1\n" + ), + "integer_declared_window_excludes_rows_outside": ( + 'TEST (name "integer_declared_window_excludes_rows_outside", ' + "cursor_start 10, cursor_end 20);\n\nWITH\n" + f"{_CURSOR_EVENT_ROWS},\n" + "__expected__event_totals AS (SELECT 10 AS event_id, 2 AS amount)\n" + "SELECT 1\n" + ), + "microbatch_runs_one_pass": ( + 'TEST (name "microbatch_runs_one_pass");\n\nWITH\n' + f"{_CURSOR_ORDER_ROWS},\n" + "__expected__hourly_orders AS (SELECT 5 AS order_count)\n" + "SELECT 1\n" + ), +} + + +def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str]: + """Build intrinsic-using timestamp, integer and microbatch models plus one plain model.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "cursor_window_demo"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "cursor_window_demo.duckdb"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n schema: main\n table: raw_orders\n" + " - name: raw_events\n schema: main\n table: raw_events\n" + ), + "models/daily_orders.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor order_date,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + ' cursor_start "2026-01-01",\n' + " cursor_inputs (raw_orders order_date,),\n" + ");\n\n" + "SELECT order_date, amount\n" + 'FROM __source("raw_orders")\n' + "WHERE order_date >= __cursor_start() AND order_date < __cursor_end()\n" + ), + "models/order_lookback.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor scan_start,\n" + " cursor_type timestamp,\n" + " cursor_grain day,\n" + " cursor_inputs (raw_orders order_date,),\n" + ");\n\n" + "SELECT\n" + " CAST(__cursor_start() AS DATE) - INTERVAL 7 DAY AS scan_start,\n" + " CAST(__cursor_end() + INTERVAL 1 DAY AS DATE) AS scan_end,\n" + " SUM(amount) AS amount\n" + 'FROM __source("raw_orders")\n' + "WHERE order_date >= CAST(__cursor_start() AS DATE) - INTERVAL 7 DAY\n" + ), + "models/event_totals.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " cursor event_id,\n" + " cursor_type integer,\n" + " cursor_inputs (raw_events event_id,),\n" + ");\n\n" + "SELECT event_id, amount\n" + 'FROM __source("raw_events")\n' + "WHERE event_id >= __cursor_start() AND event_id < __cursor_end()\n" + " AND event_id - 1 < __cursor_end() + 1\n" + ), + "models/plain_orders.sql": ( + 'MODEL (materialized table);\n\nSELECT amount FROM __source("raw_orders")\n' + ), + "models/hourly_orders.sql": ( + "MODEL (\n" + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n" + " cursor_watermark_mode all,\n" + " cursor order_hour,\n" + " cursor_type timestamp,\n" + " cursor_grain hour,\n" + " cursor_inputs (raw_orders (column order_date, roles [filter, watermark]),),\n" + " batch_size 1h,\n" + ");\n\n" + "SELECT COUNT(*) AS order_count\n" + 'FROM __source("raw_orders")\n' + "WHERE order_date >= __cursor_start() AND order_date < __cursor_end()\n" + ), + } + for test_name, sql in tests.items(): + files[f"tests/unit/{test_name}.sql"] = sql + return files + + +def build_declared_window_test(*, window: str, model_name: str) -> dict[str, str]: + """Build one SQL test declaring a cursor window and expecting rows from one model.""" + + return { + "declared_window_case": ( + f'TEST (name "declared_window_case", {window});\n\nWITH\n' + f"{_CURSOR_ORDER_ROWS},\n" + f"{_CURSOR_EVENT_ROWS},\n" + f"__expected__{model_name} AS (SELECT 1 AS amount)\n" + "SELECT 1\n" + ) + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py new file mode 100644 index 0000000000..cac1a1ad82 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_cursor_window.py @@ -0,0 +1,179 @@ +"""E2E tests for SQL tests of models that use `__cursor_start()` and `__cursor_end()`.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import CursorWindowE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import ( + CURSOR_WINDOW_PASSING_TESTS, + build_cursor_window_project_files, + build_declared_window_test, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_DECLARED_ERROR_PREFIX: str = ( + "SQL test 'declared_window_case' in 'tests/unit/declared_window_case.sql'" +) + + +@pytest.mark.parametrize( + "test_case", + ( + CursorWindowE2ETestCase( + description="default and declared windows evaluate intrinsic models", + tests=CURSOR_WINDOW_PASSING_TESTS, + expected_exit_code=0, + expected_output_fragments=("PASS=6 FAIL=0 TOTAL=6",), + expected_compiled_fragments=( + "TIMESTAMP '1900-01-01 00:00:00'", + "TIMESTAMP '2999-12-31 00:00:00'", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_models_using_cursor_intrinsics_when_testing_then_window_is_rendered( + test_case: CursorWindowE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="cursor_window_project", + repo_files=build_cursor_window_project_files(tests=test_case.tests), + ) + test_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + compile_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = test_result.stdout + test_result.stderr + assert test_result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + assert compile_result.returncode == 0, compile_result.stdout + compile_result.stderr + runtime_sql: str = next( + (project_dir / "target" / "run" / "tests").rglob("default_window_keeps_every_row.sql") + ).read_text(encoding="utf-8") + compiled_sql: str = next( + (project_dir / "target" / "compiled" / "tests").rglob("default_window_keeps_every_row.sql") + ).read_text(encoding="utf-8") + assert runtime_sql == compiled_sql + assert "__cursor_start" not in compiled_sql, compiled_sql + for fragment in test_case.expected_compiled_fragments: + assert fragment in compiled_sql, compiled_sql + declared_sql: str = next( + (project_dir / "target" / "compiled" / "tests").rglob( + "declared_window_excludes_rows_outside.sql" + ) + ).read_text(encoding="utf-8") + assert "TIMESTAMP '2026-02-01'" in declared_sql, declared_sql + assert "TIMESTAMP '2026-02-03'" in declared_sql, declared_sql + + +@pytest.mark.parametrize( + "test_case", + ( + CursorWindowE2ETestCase( + description="inverted declared window", + tests=build_declared_window_test( + window='cursor_start "2026-02-03", cursor_end "2026-02-01"', + model_name="daily_orders", + ), + expected_exit_code=1, + expected_output_fragments=( + ( + _DECLARED_ERROR_PREFIX + + ": cursor_start '2026-02-03' must be before the exclusive cursor_end " + "'2026-02-01' for model 'daily_orders'" + ), + ), + ), + CursorWindowE2ETestCase( + description="invalid timestamp bound", + tests=build_declared_window_test( + window='cursor_start "first of february"', + model_name="daily_orders", + ), + expected_exit_code=1, + expected_output_fragments=( + ( + _DECLARED_ERROR_PREFIX + + ": cursor_start 'first of february' must be an ISO timestamp or date " + "because model 'daily_orders' uses cursor_type timestamp" + ), + ), + ), + CursorWindowE2ETestCase( + description="bound not aligned to the model grain", + tests=build_declared_window_test( + window='cursor_end "2026-02-03 12:00:00"', + model_name="daily_orders", + ), + expected_exit_code=1, + expected_output_fragments=( + ( + _DECLARED_ERROR_PREFIX + + ": cursor_end '2026-02-03 12:00:00' is not aligned to cursor_grain day of " + "model 'daily_orders'" + ), + ), + ), + CursorWindowE2ETestCase( + description="timestamp bound for an integer cursor", + tests=build_declared_window_test( + window='cursor_start "2026-02-01"', + model_name="event_totals", + ), + expected_exit_code=1, + expected_output_fragments=( + ( + _DECLARED_ERROR_PREFIX + + ": cursor_start '2026-02-01' must be an integer because model " + "'event_totals' uses cursor_type integer" + ), + ), + ), + CursorWindowE2ETestCase( + description="declared window without an intrinsic model", + tests=build_declared_window_test( + window='cursor_start "2026-02-01"', model_name="plain_orders" + ), + expected_exit_code=1, + expected_output_fragments=( + ( + _DECLARED_ERROR_PREFIX + + " declares cursor_start or cursor_end, but no model it evaluates uses " + "__cursor_start() or __cursor_end()" + ), + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_invalid_declared_cursor_window_when_testing_and_compiling_then_errors_clearly( + test_case: CursorWindowE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="cursor_window_error_project", + repo_files=build_cursor_window_project_files(tests=test_case.tests), + ) + + results: tuple[subprocess.CompletedProcess[str], ...] = tuple( + run_sqb(command=command, project_dir=project_dir) + for command in (("--no-color", "test"), ("--no-color", "compile")) + ) + + for result in results: + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + assert "Connecting to" not in output, output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_scope.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_scope.py new file mode 100644 index 0000000000..e05e1081f5 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_helper_scope.py @@ -0,0 +1,65 @@ +"""E2E tests for helper CTEs used by expected and assertion SQL in `sqb test`.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import HelperScopeE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import build_helper_scope_project_files +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_OUTCOME_FRAGMENTS: tuple[str, ...] = ( + "PASS=4 FAIL=2 TOTAL=6", + "unexpected sample 1: order_id=2, amount=40; missing sample 1: order_id=2, amount=41", + "expect assertion all_expected_rows_built", +) + + +@pytest.mark.parametrize( + "test_case", + ( + HelperScopeE2ETestCase( + description="helpers are in scope with sql analysis", + command=("--no-color", "test"), + expected_output_fragments=_OUTCOME_FRAGMENTS, + ), + HelperScopeE2ETestCase( + description="helpers are in scope without sql analysis", + command=("--no-color", "test", "--no-sql-analysis"), + expected_output_fragments=_OUTCOME_FRAGMENTS, + ), + ), + ids=lambda case: case.description, +) +def test_given_helper_ctes_when_expected_and_assertions_use_them_then_tests_compare_rows( + test_case: HelperScopeE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="helper_scope_project", + repo_files=build_helper_scope_project_files(), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + compile_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 1, output + for fragment in test_case.expected_output_fragments: + assert fragment in output, output + assert "Catalog Error" not in output, output + assert "unreachable" not in output, output + assert compile_result.returncode == 0, compile_result.stdout + compile_result.stderr + compiled_sql: str = next( + (project_dir / "target" / "compiled" / "tests").rglob("helper_reads_mock_and_helper.sql") + ).read_text(encoding="utf-8") + assert compiled_sql.count("\ndoubled AS (") == 1, compiled_sql + assert compiled_sql.count("\nexpected_rows AS (") == 1, compiled_sql diff --git a/tests/unit/scripts/dupscore/_helpers/clones/clustering/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/clustering/_test_types.py new file mode 100644 index 0000000000..c71f739cab --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/clustering/_test_types.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ClusterPairsTestCase: + description: str + pairs: tuple[tuple[int, int, float, str], ...] + expected_clusters: tuple[tuple[str, tuple[str, ...]], ...] + + +@dataclass(frozen=True) +class ClusterEstimateTestCase: + description: str + pairs: tuple[tuple[int, int, float, str], ...] + expected_links: tuple[tuple[int, int], ...] + expected_similarity_range: tuple[float, float] + expected_duplicated_tokens: int diff --git a/tests/unit/scripts/dupscore/_helpers/clones/clustering/helpers.py b/tests/unit/scripts/dupscore/_helpers/clones/clustering/helpers.py new file mode 100644 index 0000000000..f07e07943f --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/clustering/helpers.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from scripts.dupscore._helpers.clones.tokens import build_clone_unit +from scripts.dupscore.models import CloneCluster, ClonePair, CloneUnit + +_UNIT_SIZES: tuple[int, ...] = (100, 100, 90, 40, 40) + + +def build_units() -> list[CloneUnit]: + return [ + build_clone_unit( + language="python", + path=f"src/sqlbuild/module_{position}.py", + name=f"function_{position}", + start_line=1, + end_line=10, + normalized=["$id"] * size, + concrete=["name"] * size, + ) + for position, size in enumerate(_UNIT_SIZES) + ] + + +def build_pairs(specs: tuple[tuple[int, int, float, str], ...]) -> list[ClonePair]: + return [ + ClonePair(left=left, right=right, similarity=similarity, category=category) + for left, right, similarity, category in specs + ] + + +def no_change(unit: CloneUnit) -> str | None: + return None + + +def summarize_clusters(clusters: list[CloneCluster]) -> tuple[tuple[str, tuple[str, ...]], ...]: + summaries: list[tuple[str, tuple[str, ...]]] = [] + for cluster in clusters: + names: tuple[str, ...] = tuple(member.name for member in cluster.members) + summaries.append((cluster.category, names)) + return tuple(summaries) diff --git a/tests/unit/scripts/dupscore/_helpers/clones/clustering/test_cluster_clone_pairs.py b/tests/unit/scripts/dupscore/_helpers/clones/clustering/test_cluster_clone_pairs.py new file mode 100644 index 0000000000..09553f46ca --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/clustering/test_cluster_clone_pairs.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import pytest + +from scripts.dupscore._helpers.clones.clustering import cluster_clone_pairs +from scripts.dupscore.constants import CATEGORY_EXACT, CATEGORY_NEAR_MISS, CATEGORY_RENAMED +from scripts.dupscore.models import CloneCluster +from tests.unit.scripts.dupscore._helpers.clones.clustering._test_types import ( + ClusterEstimateTestCase, + ClusterPairsTestCase, +) +from tests.unit.scripts.dupscore._helpers.clones.clustering.helpers import ( + build_pairs, + build_units, + no_change, + summarize_clusters, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ClusterPairsTestCase( + description="transitive pairs merge and rank by duplicated tokens", + pairs=( + (3, 4, 1.0, CATEGORY_EXACT), + (0, 1, 1.0, CATEGORY_RENAMED), + (1, 2, 0.9, CATEGORY_NEAR_MISS), + ), + expected_clusters=( + (CATEGORY_NEAR_MISS, ("function_0", "function_1", "function_2")), + (CATEGORY_EXACT, ("function_3", "function_4")), + ), + ), + ClusterPairsTestCase( + description="no pairs means no clusters", + pairs=(), + expected_clusters=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_clone_pairs_when_clustering_then_groups_transitively( + test_case: ClusterPairsTestCase, +) -> None: + clusters: list[CloneCluster] = cluster_clone_pairs( + units=build_units(), pairs=build_pairs(test_case.pairs), change_of=no_change + ) + + assert summarize_clusters(clusters) == test_case.expected_clusters + + +@pytest.mark.parametrize( + "test_case", + [ + ClusterEstimateTestCase( + description="chain reports member links and duplicated token estimate", + pairs=((0, 1, 1.0, CATEGORY_RENAMED), (1, 2, 0.9, CATEGORY_NEAR_MISS)), + expected_links=((0, 1), (1, 2)), + expected_similarity_range=(0.9, 1.0), + expected_duplicated_tokens=181, + ) + ], + ids=lambda case: case.description, +) +def test_given_cluster_when_clustering_then_reports_links_and_estimates( + test_case: ClusterEstimateTestCase, +) -> None: + cluster: CloneCluster = cluster_clone_pairs( + units=build_units(), pairs=build_pairs(test_case.pairs), change_of=no_change + )[0] + + assert tuple((link.left, link.right) for link in cluster.links) == test_case.expected_links + assert (cluster.similarity_min, cluster.similarity_max) == test_case.expected_similarity_range + assert cluster.duplicated_tokens == test_case.expected_duplicated_tokens diff --git a/tests/unit/scripts/dupscore/_helpers/clones/filters/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/filters/_test_types.py new file mode 100644 index 0000000000..650c3bbf97 --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/filters/_test_types.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.dupscore.models import FileChanges + + +@dataclass(frozen=True) +class ClassifyChangeTestCase: + description: str + changes: FileChanges | None + expected_change: str | None + + +@dataclass(frozen=True) +class ParseDiffTestCase: + description: str + diff_text: str + expected_changes: dict[str, FileChanges] diff --git a/tests/unit/scripts/dupscore/_helpers/clones/filters/test_clone_changes.py b/tests/unit/scripts/dupscore/_helpers/clones/filters/test_clone_changes.py new file mode 100644 index 0000000000..2dd70eb62a --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/filters/test_clone_changes.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import pytest + +from scripts.dupscore._helpers.clones.filters import classify_unit_change, parse_unified_diff +from scripts.dupscore._helpers.clones.tokens import build_clone_unit +from scripts.dupscore.constants import CHANGE_CHANGED, CHANGE_NEW +from scripts.dupscore.models import CloneUnit, FileChanges +from tests.unit.scripts.dupscore._helpers.clones.filters._test_types import ( + ClassifyChangeTestCase, + ParseDiffTestCase, +) + +_DIFF_TEXT: str = """\ +diff --git a/src/sqlbuild/orders.py b/src/sqlbuild/orders.py +index 1111111..2222222 100644 +--- a/src/sqlbuild/orders.py ++++ b/src/sqlbuild/orders.py +@@ -10,0 +11,3 @@ def summarize(): ++ one ++ two ++ three +@@ -40 +43 @@ def other(): +- old ++ new +@@ -60,2 +62,0 @@ def removed(): +- gone +- gone +diff --git a/src/sqlbuild/deleted.py b/src/sqlbuild/deleted.py +deleted file mode 100644 +--- a/src/sqlbuild/deleted.py ++++ /dev/null +@@ -1,2 +0,0 @@ +-x +-y +""" + +_UNIT: CloneUnit = build_clone_unit( + language="python", + path="src/sqlbuild/orders.py", + name="summarize", + start_line=10, + end_line=20, + normalized=["$id"], + concrete=["name"], +) + + +@pytest.mark.parametrize( + "test_case", + [ + ParseDiffTestCase( + description="added ranges and pure deletions; deleted files are ignored", + diff_text=_DIFF_TEXT, + expected_changes={ + "src/sqlbuild/orders.py": FileChanges( + added_ranges=((11, 13), (43, 43)), + deletion_points=(62,), + ) + }, + ), + ParseDiffTestCase( + description="empty diff has no changes", + diff_text="", + expected_changes={}, + ), + ], + ids=lambda case: case.description, +) +def test_given_unified_diff_when_parsing_then_returns_added_ranges_and_deletions( + test_case: ParseDiffTestCase, +) -> None: + assert parse_unified_diff(test_case.diff_text) == test_case.expected_changes + + +@pytest.mark.parametrize( + "test_case", + [ + ClassifyChangeTestCase( + description="untouched file", + changes=None, + expected_change=None, + ), + ClassifyChangeTestCase( + description="untracked file is new", + changes=FileChanges(whole_file=True), + expected_change=CHANGE_NEW, + ), + ClassifyChangeTestCase( + description="fully added range is new", + changes=FileChanges(added_ranges=((5, 12), (13, 25))), + expected_change=CHANGE_NEW, + ), + ClassifyChangeTestCase( + description="partially added range is changed", + changes=FileChanges(added_ranges=((18, 30),)), + expected_change=CHANGE_CHANGED, + ), + ClassifyChangeTestCase( + description="deletion inside the unit is changed", + changes=FileChanges(deletion_points=(15,)), + expected_change=CHANGE_CHANGED, + ), + ClassifyChangeTestCase( + description="deletion after the last line is untouched", + changes=FileChanges(deletion_points=(20,)), + expected_change=None, + ), + ClassifyChangeTestCase( + description="edits outside the unit are untouched", + changes=FileChanges(added_ranges=((1, 9), (21, 40))), + expected_change=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_file_changes_when_classifying_unit_then_returns_expected_status( + test_case: ClassifyChangeTestCase, +) -> None: + assert classify_unit_change(unit=_UNIT, changes=test_case.changes) == test_case.expected_change diff --git a/tests/unit/scripts/dupscore/_helpers/clones/python_units/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/python_units/_test_types.py new file mode 100644 index 0000000000..1c69f17c63 --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/python_units/_test_types.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class PythonUnitsTestCase: + description: str + source: str + expected_units: tuple[tuple[str, int, int], ...] + + +@dataclass(frozen=True) +class PythonKeyTestCase: + description: str + left_source: str + right_source: str + expected_same_concrete: bool + expected_same_normalized: bool diff --git a/tests/unit/scripts/dupscore/_helpers/clones/python_units/test_extract_python_units.py b/tests/unit/scripts/dupscore/_helpers/clones/python_units/test_extract_python_units.py new file mode 100644 index 0000000000..2b187c1986 --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/python_units/test_extract_python_units.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import pytest + +from scripts.dupscore._helpers.clones.python_units import extract_python_units +from scripts.dupscore.models import CloneUnit +from tests.unit.scripts.dupscore._helpers.clones.python_units._test_types import ( + PythonKeyTestCase, + PythonUnitsTestCase, +) + +_LAYOUT_SOURCE: str = """\ +import functools + + +@functools.cache +def load_orders(path): + def parse(line): + return line.split(",") + return [parse(line) for line in open(path)] + + +class Warehouse: + class Shelf: + def label(self): + return "shelf" + + async def restock(self, count): + return count + 1 + + +if True: + def conditional_helper(): + return None +""" + + +@pytest.mark.parametrize( + "test_case", + [ + PythonUnitsTestCase( + description="functions, nested class methods, and conditional defs are units", + source=_LAYOUT_SOURCE, + expected_units=( + ("load_orders", 5, 8), + ("Warehouse.Shelf.label", 13, 14), + ("Warehouse.restock", 16, 17), + ("conditional_helper", 21, 22), + ), + ), + PythonUnitsTestCase( + description="unparseable source yields no units", + source="def broken(:\n pass\n", + expected_units=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_source_when_extracting_units_then_returns_expected_units( + test_case: PythonUnitsTestCase, +) -> None: + units: tuple[CloneUnit, ...] = extract_python_units( + relative_path="src/sqlbuild/demo.py", source=test_case.source + ) + + assert ( + tuple((unit.name, unit.start_line, unit.end_line) for unit in units) + == test_case.expected_units + ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonKeyTestCase( + description="keyword-only markers and defaults still change identity", + left_source='def f(a, b):\n """Doc."""\n total = a + b # sum\n return total\n', + right_source=( + "def f(a: int, *, b: int = 0) -> int:\n total: int = a + b\n return total\n" + ), + expected_same_concrete=False, + expected_same_normalized=False, + ), + PythonKeyTestCase( + description="annotations and docstring only differences are exact", + left_source='def f(a, b):\n """Doc."""\n total = a + b # sum\n return total\n', + right_source=( + "def f(a: int, b: list[int]) -> int:\n total: int = a + b\n return total\n" + ), + expected_same_concrete=True, + expected_same_normalized=True, + ), + PythonKeyTestCase( + description="renamed identifiers and literals share the normalised stream", + left_source='def f(a):\n return g(a, "x", 1)\n', + right_source='def other(value):\n return h(value, "y", 2)\n', + expected_same_concrete=False, + expected_same_normalized=True, + ), + PythonKeyTestCase( + description="different literal kinds keep distinct placeholders", + left_source='def f(a):\n return g(a, "x")\n', + right_source="def f(a):\n return g(a, 1)\n", + expected_same_concrete=False, + expected_same_normalized=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_two_functions_when_extracting_then_identity_keys_reflect_normalisation( + test_case: PythonKeyTestCase, +) -> None: + left: CloneUnit = extract_python_units(relative_path="a.py", source=test_case.left_source)[0] + right: CloneUnit = extract_python_units(relative_path="b.py", source=test_case.right_source)[0] + + assert (left.concrete_key == right.concrete_key) is test_case.expected_same_concrete + assert (left.normalized_key == right.normalized_key) is test_case.expected_same_normalized diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/_test_types.py new file mode 100644 index 0000000000..bcdfac43da --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/_test_types.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class LexRustTestCase: + description: str + source: str + expected_tokens: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class LexLineTestCase: + description: str + source: str + token_text: str + expected_line: int diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/test_lex_rust.py b/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/test_lex_rust.py new file mode 100644 index 0000000000..92c3b3dbbd --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/rust_lexer/test_lex_rust.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import pytest + +from scripts.dupscore._helpers.clones.rust_lexer import lex_rust +from scripts.dupscore.constants import ( + RUST_KIND_BYTE, + RUST_KIND_BYTE_STRING, + RUST_KIND_CHAR, + RUST_KIND_IDENTIFIER, + RUST_KIND_LIFETIME, + RUST_KIND_NUMBER, + RUST_KIND_PUNCTUATION, + RUST_KIND_STRING, +) +from scripts.dupscore.models import RustToken +from tests.unit.scripts.dupscore._helpers.clones.rust_lexer._test_types import ( + LexLineTestCase, + LexRustTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + LexRustTestCase( + description="raw string containing braces and quotes is one token", + source='let s = r#"{ "a" } }"#;', + expected_tokens=( + (RUST_KIND_IDENTIFIER, "let"), + (RUST_KIND_IDENTIFIER, "s"), + (RUST_KIND_PUNCTUATION, "="), + (RUST_KIND_STRING, 'r#"{ "a" } }"#'), + (RUST_KIND_PUNCTUATION, ";"), + ), + ), + LexRustTestCase( + description="lifetimes are distinguished from char literals", + source="fn f<'a>(x: &'a str) -> char { '{' }", + expected_tokens=( + (RUST_KIND_IDENTIFIER, "fn"), + (RUST_KIND_IDENTIFIER, "f"), + (RUST_KIND_PUNCTUATION, "<"), + (RUST_KIND_LIFETIME, "'a"), + (RUST_KIND_PUNCTUATION, ">"), + (RUST_KIND_PUNCTUATION, "("), + (RUST_KIND_IDENTIFIER, "x"), + (RUST_KIND_PUNCTUATION, ":"), + (RUST_KIND_PUNCTUATION, "&"), + (RUST_KIND_LIFETIME, "'a"), + (RUST_KIND_IDENTIFIER, "str"), + (RUST_KIND_PUNCTUATION, ")"), + (RUST_KIND_PUNCTUATION, "->"), + (RUST_KIND_IDENTIFIER, "char"), + (RUST_KIND_PUNCTUATION, "{"), + (RUST_KIND_CHAR, "'{'"), + (RUST_KIND_PUNCTUATION, "}"), + ), + ), + LexRustTestCase( + description="nested block comments and line comments are dropped", + source="/* outer /* inner { */ still } */ x // trailing {\ny", + expected_tokens=((RUST_KIND_IDENTIFIER, "x"), (RUST_KIND_IDENTIFIER, "y")), + ), + LexRustTestCase( + description="escaped chars and byte literals are single tokens", + source=r"""'\'' '\u{7B}' b'}' b"{\"}" br#"}"# 'static""", + expected_tokens=( + (RUST_KIND_CHAR, r"'\''"), + (RUST_KIND_CHAR, r"'\u{7B}'"), + (RUST_KIND_BYTE, "b'}'"), + (RUST_KIND_BYTE_STRING, r'b"{\"}"'), + (RUST_KIND_BYTE_STRING, 'br#"}"#'), + (RUST_KIND_LIFETIME, "'static"), + ), + ), + LexRustTestCase( + description="numbers keep suffixes and ranges stay separate", + source="0..10u8 1.5e-3 0xFF_u32 t.0", + expected_tokens=( + (RUST_KIND_NUMBER, "0"), + (RUST_KIND_PUNCTUATION, ".."), + (RUST_KIND_NUMBER, "10u8"), + (RUST_KIND_NUMBER, "1.5e-3"), + (RUST_KIND_NUMBER, "0xFF_u32"), + (RUST_KIND_IDENTIFIER, "t"), + (RUST_KIND_PUNCTUATION, "."), + (RUST_KIND_NUMBER, "0"), + ), + ), + LexRustTestCase( + description="raw identifiers and path separators", + source="r#type::new()", + expected_tokens=( + (RUST_KIND_IDENTIFIER, "r#type"), + (RUST_KIND_PUNCTUATION, "::"), + (RUST_KIND_IDENTIFIER, "new"), + (RUST_KIND_PUNCTUATION, "("), + (RUST_KIND_PUNCTUATION, ")"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_rust_source_when_lexing_then_returns_expected_tokens( + test_case: LexRustTestCase, +) -> None: + tokens: list[RustToken] = lex_rust(test_case.source) + + assert tuple((item.kind, item.text) for item in tokens) == test_case.expected_tokens + + +@pytest.mark.parametrize( + "test_case", + [ + LexLineTestCase( + description="multi-line string advances the line counter", + source='let a = "one\ntwo\nthree";\nlet b = 1;', + token_text="b", + expected_line=4, + ), + LexLineTestCase( + description="multi-line block comment advances the line counter", + source="/* one\n/* two\n*/ three\n*/\nafter", + token_text="after", + expected_line=5, + ), + ], + ids=lambda case: case.description, +) +def test_given_multiline_tokens_when_lexing_then_tracks_lines( + test_case: LexLineTestCase, +) -> None: + tokens: list[RustToken] = lex_rust(test_case.source) + + lines_by_text: dict[str, int] = {item.text: item.line for item in tokens} + assert lines_by_text[test_case.token_text] == test_case.expected_line diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rust_units/_test_types.py b/tests/unit/scripts/dupscore/_helpers/clones/rust_units/_test_types.py new file mode 100644 index 0000000000..0179c682bc --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/rust_units/_test_types.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RustUnitsTestCase: + description: str + source: str + include_tests: bool + expected_units: tuple[tuple[str, int, int], ...] + + +@dataclass(frozen=True) +class RustTestModuleTestCase: + description: str + relative_path: str + source: str + expected_prefixes: tuple[str, ...] + + +@dataclass(frozen=True) +class RustKeyTestCase: + description: str + left_source: str + right_source: str + expected_same_concrete: bool + expected_same_normalized: bool diff --git a/tests/unit/scripts/dupscore/_helpers/clones/rust_units/test_extract_rust_units.py b/tests/unit/scripts/dupscore/_helpers/clones/rust_units/test_extract_rust_units.py new file mode 100644 index 0000000000..825cca5b80 --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/clones/rust_units/test_extract_rust_units.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import pytest + +from scripts.dupscore._helpers.clones.rust_units import extract_rust_units +from scripts.dupscore.models import RustFileUnits +from tests.unit.scripts.dupscore._helpers.clones.rust_units._test_types import ( + RustKeyTestCase, + RustTestModuleTestCase, + RustUnitsTestCase, +) + +_IMPL_AND_TRAIT_SOURCE: str = """\ +pub struct Inventory { items: Vec } + +impl Inventory where T: Default { + #[must_use] + pub fn first(&self) -> Option<&T> { + self.items.first() + } +} + +pub trait Fulfill { + fn ship(&self) -> bool; + fn describe(&self) -> String { + String::from("pending") + } +} + +impl<'a> Fulfill for &'a Inventory { + fn ship(&self) -> bool { true } +} + +fn helper(pointer: fn(u8) -> u8) -> u8 { + fn nested() -> u8 { 1 } + pointer(nested()) +} +""" + +_BRACE_TRAPS_SOURCE: str = """\ +fn braces() -> &'static str { + let open = '{'; + let text = r#"}}} "{" }"#; + /* } /* } */ } */ + let bytes = b"}"; + text +} + +fn after() {} +""" + +_TEST_ITEMS_SOURCE: str = """\ +fn kept() {} + +#[cfg(test)] +fn test_helper() {} + +#[cfg(not(test))] +fn production_only() {} + +#[cfg(test)] +mod tests { + #[test] + fn checks_orders() {} +} + +#[tokio::test] +async fn async_check() {} + +macro_rules! make_fn { + () => { fn generated() {} }; +} +""" + + +@pytest.mark.parametrize( + "test_case", + [ + RustUnitsTestCase( + description="impl and trait methods are qualified and bodiless decls skipped", + source=_IMPL_AND_TRAIT_SOURCE, + include_tests=False, + expected_units=( + ("Inventory::first", 5, 7), + ("Fulfill::describe", 12, 14), + ("Inventory::ship", 18, 18), + ("helper", 21, 24), + ), + ), + RustUnitsTestCase( + description="braces inside chars, raw strings, comments, and bytes are ignored", + source=_BRACE_TRAPS_SOURCE, + include_tests=False, + expected_units=(("braces", 1, 7), ("after", 9, 9)), + ), + RustUnitsTestCase( + description="cfg(test) items, test functions, and macro bodies are skipped", + source=_TEST_ITEMS_SOURCE, + include_tests=False, + expected_units=(("kept", 1, 1), ("production_only", 7, 7)), + ), + RustUnitsTestCase( + description="include-tests keeps test items", + source=_TEST_ITEMS_SOURCE, + include_tests=True, + expected_units=( + ("kept", 1, 1), + ("test_helper", 4, 4), + ("production_only", 7, 7), + ("checks_orders", 12, 12), + ("async_check", 16, 16), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_rust_source_when_extracting_units_then_returns_fn_items( + test_case: RustUnitsTestCase, +) -> None: + extracted: RustFileUnits = extract_rust_units( + relative_path="crates/demo/src/lib.rs", + source=test_case.source, + include_tests=test_case.include_tests, + ) + + assert ( + tuple((unit.name, unit.start_line, unit.end_line) for unit in extracted.units) + == test_case.expected_units + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RustTestModuleTestCase( + description="module root declares sibling test module", + relative_path="crates/demo/src/engine/mod.rs", + source="#[cfg(test)]\nmod tests;\nmod kept;\n", + expected_prefixes=( + "crates/demo/src/engine/tests.rs", + "crates/demo/src/engine/tests/", + ), + ), + RustTestModuleTestCase( + description="non-root file declares nested test module", + relative_path="crates/demo/src/engine.rs", + source="#[cfg(test)]\nmod checks;\n", + expected_prefixes=( + "crates/demo/src/engine/checks.rs", + "crates/demo/src/engine/checks/", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_cfg_test_module_declaration_when_extracting_then_reports_test_paths( + test_case: RustTestModuleTestCase, +) -> None: + extracted: RustFileUnits = extract_rust_units( + relative_path=test_case.relative_path, + source=test_case.source, + include_tests=False, + ) + + assert extracted.test_module_prefixes == test_case.expected_prefixes + + +@pytest.mark.parametrize( + "test_case", + [ + RustKeyTestCase( + description="renamed identifiers, literals, and visibility share the normalised stream", + left_source='fn count_orders(x: u8) -> u8 { let y = x + 1; println!("{y}"); y }', + right_source='pub fn tally(value: u16) -> u16 { let n = value + 9; print!("n"); n }', + expected_same_concrete=False, + expected_same_normalized=True, + ), + RustKeyTestCase( + description="comments and attributes do not change identity", + left_source="fn f(x: u8) -> u8 { x /* note */ }", + right_source="#[inline]\npub(crate) fn f(x: u8) -> u8 {\n // note\n x\n}", + expected_same_concrete=True, + expected_same_normalized=True, + ), + RustKeyTestCase( + description="char and string literals keep distinct placeholders", + left_source="fn f() -> char { 'x' }", + right_source='fn f() -> char { "x" }', + expected_same_concrete=False, + expected_same_normalized=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_two_rust_functions_when_extracting_then_identity_keys_reflect_normalisation( + test_case: RustKeyTestCase, +) -> None: + left: RustFileUnits = extract_rust_units( + relative_path="crates/demo/src/a.rs", source=test_case.left_source, include_tests=False + ) + right: RustFileUnits = extract_rust_units( + relative_path="crates/demo/src/b.rs", source=test_case.right_source, include_tests=False + ) + + assert ( + left.units[0].concrete_key == right.units[0].concrete_key + ) is test_case.expected_same_concrete + assert ( + left.units[0].normalized_key == right.units[0].normalized_key + ) is test_case.expected_same_normalized diff --git a/tests/unit/scripts/dupscore/_helpers/config/test_load_config.py b/tests/unit/scripts/dupscore/_helpers/config/test_load_config.py deleted file mode 100644 index 2c59b08b08..0000000000 --- a/tests/unit/scripts/dupscore/_helpers/config/test_load_config.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from scripts.dupscore._helpers.config import load_config -from scripts.dupscore.exceptions import DupscoreConfigError -from scripts.dupscore.models import DupscoreConfig -from tests.unit.scripts.dupscore._helpers.config._test_types import ( - InvalidConfigTestCase, - LoadConfigTestCase, - MissingConfigTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - LoadConfigTestCase( - description="loads surfaces and normalizes allowlist pair order", - toml_text=( - 'persisted_state_surfaces = ["sqlbuild.virtual.state"]\n' - "\n" - "[[allowlist]]\n" - 'pair = ["sqlbuild.b.right", "sqlbuild.a.left"]\n' - 'reason = "intentional mirror"\n' - ), - expected_surfaces=("sqlbuild.virtual.state",), - expected_allowlisted_pair=("sqlbuild.a.left", "sqlbuild.b.right"), - expected_reason="intentional mirror", - ) - ], - ids=lambda case: case.description, -) -def test_given_config_file_when_loading_then_returns_expected_config( - test_case: LoadConfigTestCase, - tmp_path: Path, -) -> None: - config_path: Path = tmp_path / "dupscore.toml" - config_path.write_text(test_case.toml_text, encoding="utf-8") - - config: DupscoreConfig = load_config(config_path) - - assert config.persisted_state_surfaces == test_case.expected_surfaces - assert ( - config.allowlisted_pairs[test_case.expected_allowlisted_pair] == test_case.expected_reason - ) - - -@pytest.mark.parametrize( - "test_case", - [ - MissingConfigTestCase( - description="returns empty config when the file is missing", - expected_surfaces=(), - expected_allowlist_size=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_config_file_when_loading_then_returns_empty_config( - test_case: MissingConfigTestCase, - tmp_path: Path, -) -> None: - config_path: Path = tmp_path / "dupscore.toml" - - config: DupscoreConfig = load_config(config_path) - - assert config.persisted_state_surfaces == test_case.expected_surfaces - assert len(config.allowlisted_pairs) == test_case.expected_allowlist_size - - -@pytest.mark.parametrize( - "test_case", - [ - InvalidConfigTestCase( - description="rejects allowlist entries without a reason", - toml_text='[[allowlist]]\npair = ["sqlbuild.a", "sqlbuild.b"]\n', - expected_error_fragment="reason", - ), - InvalidConfigTestCase( - description="rejects non-list surfaces", - toml_text='persisted_state_surfaces = "sqlbuild.virtual.state"\n', - expected_error_fragment="persisted_state_surfaces", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_config_when_loading_then_raises_config_error( - test_case: InvalidConfigTestCase, - tmp_path: Path, -) -> None: - config_path: Path = tmp_path / "dupscore.toml" - config_path.write_text(test_case.toml_text, encoding="utf-8") - - with pytest.raises(DupscoreConfigError) as raised: - _ = load_config(config_path) - - assert test_case.expected_error_fragment in str(raised.value) diff --git a/tests/unit/scripts/dupscore/_helpers/config/_test_types.py b/tests/unit/scripts/dupscore/_helpers/inputs/config/_test_types.py similarity index 58% rename from tests/unit/scripts/dupscore/_helpers/config/_test_types.py rename to tests/unit/scripts/dupscore/_helpers/inputs/config/_test_types.py index 7871520774..02e969847e 100644 --- a/tests/unit/scripts/dupscore/_helpers/config/_test_types.py +++ b/tests/unit/scripts/dupscore/_helpers/inputs/config/_test_types.py @@ -2,6 +2,8 @@ from dataclasses import dataclass +from scripts.dupscore.models import CloneAllowlistEntry + @dataclass(frozen=True) class LoadConfigTestCase: @@ -24,3 +26,18 @@ class InvalidConfigTestCase: description: str toml_text: str expected_error_fragment: str + + +@dataclass(frozen=True) +class CloneAllowlistTestCase: + description: str + toml_text: str + expected_entries: tuple[CloneAllowlistEntry, ...] + + +@dataclass(frozen=True) +class ShippedConfigTestCase: + description: str + relative_path: str + expected_pair_allowlist_size: int + expected_clone_allowlist_size: int diff --git a/tests/unit/scripts/dupscore/_helpers/inputs/config/test_load_config.py b/tests/unit/scripts/dupscore/_helpers/inputs/config/test_load_config.py new file mode 100644 index 0000000000..af112d19c5 --- /dev/null +++ b/tests/unit/scripts/dupscore/_helpers/inputs/config/test_load_config.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.dupscore._helpers.inputs.config import load_config +from scripts.dupscore.exceptions import DupscoreConfigError +from scripts.dupscore.models import CloneAllowlistEntry, DupscoreConfig +from tests.unit.scripts.dupscore._helpers.inputs.config._test_types import ( + CloneAllowlistTestCase, + InvalidConfigTestCase, + LoadConfigTestCase, + MissingConfigTestCase, + ShippedConfigTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + LoadConfigTestCase( + description="loads surfaces and normalizes allowlist pair order", + toml_text=( + 'persisted_state_surfaces = ["sqlbuild.virtual.state"]\n' + "\n" + "[[allowlist]]\n" + 'pair = ["sqlbuild.b.right", "sqlbuild.a.left"]\n' + 'reason = "intentional mirror"\n' + ), + expected_surfaces=("sqlbuild.virtual.state",), + expected_allowlisted_pair=("sqlbuild.a.left", "sqlbuild.b.right"), + expected_reason="intentional mirror", + ) + ], + ids=lambda case: case.description, +) +def test_given_config_file_when_loading_then_returns_expected_config( + test_case: LoadConfigTestCase, + tmp_path: Path, +) -> None: + config_path: Path = tmp_path / "dupscore.toml" + config_path.write_text(test_case.toml_text, encoding="utf-8") + + config: DupscoreConfig = load_config(config_path) + + assert config.persisted_state_surfaces == test_case.expected_surfaces + assert ( + config.allowlisted_pairs[test_case.expected_allowlisted_pair] == test_case.expected_reason + ) + + +@pytest.mark.parametrize( + "test_case", + [ + MissingConfigTestCase( + description="returns empty config when the file is missing", + expected_surfaces=(), + expected_allowlist_size=0, + ) + ], + ids=lambda case: case.description, +) +def test_given_missing_config_file_when_loading_then_returns_empty_config( + test_case: MissingConfigTestCase, + tmp_path: Path, +) -> None: + config_path: Path = tmp_path / "dupscore.toml" + + config: DupscoreConfig = load_config(config_path) + + assert config.persisted_state_surfaces == test_case.expected_surfaces + assert len(config.allowlisted_pairs) == test_case.expected_allowlist_size + + +@pytest.mark.parametrize( + "test_case", + [ + InvalidConfigTestCase( + description="rejects allowlist entries without a reason", + toml_text='[[allowlist]]\npair = ["sqlbuild.a", "sqlbuild.b"]\n', + expected_error_fragment="reason", + ), + InvalidConfigTestCase( + description="rejects non-list surfaces", + toml_text='persisted_state_surfaces = "sqlbuild.virtual.state"\n', + expected_error_fragment="persisted_state_surfaces", + ), + InvalidConfigTestCase( + description="rejects clone allowlist entries without a reason", + toml_text='[[clone_allowlist]]\npaths = ["crates/*"]\nreason = " "\n', + expected_error_fragment="clone_allowlist entry 1 needs a non-empty reason", + ), + InvalidConfigTestCase( + description="rejects clone allowlist entries with empty paths", + toml_text='[[clone_allowlist]]\npaths = []\nreason = "mirror"\n', + expected_error_fragment="clone_allowlist entry 1 needs a non-empty paths list", + ), + InvalidConfigTestCase( + description="rejects clone allowlist entries with blank globs", + toml_text='[[clone_allowlist]]\npaths = ["src/*", ""]\nreason = "mirror"\n', + expected_error_fragment="non-empty glob strings", + ), + InvalidConfigTestCase( + description="rejects clone allowlist entries with unknown keys", + toml_text=( + '[[clone_allowlist]]\npaths = ["src/*"]\nreason = "mirror"\npair = ["a", "b"]\n' + ), + expected_error_fragment="unknown keys: pair", + ), + InvalidConfigTestCase( + description="rejects a non-table clone allowlist", + toml_text='clone_allowlist = ["src/*"]\n', + expected_error_fragment="clone_allowlist entry 1 must be a table", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_config_when_loading_then_raises_config_error( + test_case: InvalidConfigTestCase, + tmp_path: Path, +) -> None: + config_path: Path = tmp_path / "dupscore.toml" + config_path.write_text(test_case.toml_text, encoding="utf-8") + + with pytest.raises(DupscoreConfigError) as raised: + _ = load_config(config_path) + + assert test_case.expected_error_fragment in str(raised.value) + + +@pytest.mark.parametrize( + "test_case", + [ + CloneAllowlistTestCase( + description="loads clone allowlist globs and strips the reason", + toml_text=( + '[[clone_allowlist]]\npaths = ["src/sqlbuild/example/*", "crates/*"]\n' + 'reason = " Backends mirror one protocol. "\n' + ), + expected_entries=( + CloneAllowlistEntry( + paths=("src/sqlbuild/example/*", "crates/*"), + reason="Backends mirror one protocol.", + ), + ), + ), + CloneAllowlistTestCase( + description="missing clone allowlist is empty", + toml_text="", + expected_entries=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_clone_allowlist_when_loading_then_returns_entries( + test_case: CloneAllowlistTestCase, + tmp_path: Path, +) -> None: + config_path: Path = tmp_path / "dupscore.toml" + config_path.write_text(test_case.toml_text, encoding="utf-8") + + config: DupscoreConfig = load_config(config_path) + + assert config.clone_allowlist == test_case.expected_entries + + +@pytest.mark.parametrize( + "test_case", + [ + ShippedConfigTestCase( + description="shipped dupscore.toml is valid", + relative_path="scripts/dupscore/dupscore.toml", + expected_pair_allowlist_size=9, + expected_clone_allowlist_size=0, + ) + ], + ids=lambda case: case.description, +) +def test_given_shipped_config_when_loading_then_it_is_valid( + test_case: ShippedConfigTestCase, +) -> None: + config_path: Path = Path(__file__).resolve().parents[7] / test_case.relative_path + + config: DupscoreConfig = load_config(config_path) + + assert len(config.allowlisted_pairs) == test_case.expected_pair_allowlist_size + assert len(config.clone_allowlist) == test_case.expected_clone_allowlist_size diff --git a/tests/unit/scripts/dupscore/_helpers/facts/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/_test_types.py similarity index 100% rename from tests/unit/scripts/dupscore/_helpers/facts/_test_types.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/facts/_test_types.py diff --git a/tests/unit/scripts/dupscore/_helpers/facts/helpers.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/helpers.py similarity index 100% rename from tests/unit/scripts/dupscore/_helpers/facts/helpers.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/facts/helpers.py diff --git a/tests/unit/scripts/dupscore/_helpers/facts/test_extract_project_facts.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/test_extract_project_facts.py similarity index 93% rename from tests/unit/scripts/dupscore/_helpers/facts/test_extract_project_facts.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/facts/test_extract_project_facts.py index bbd9f99332..206731804a 100644 --- a/tests/unit/scripts/dupscore/_helpers/facts/test_extract_project_facts.py +++ b/tests/unit/scripts/dupscore/_helpers/package_pairs/facts/test_extract_project_facts.py @@ -2,14 +2,17 @@ import pytest -from scripts.dupscore._helpers.facts import extract_project_facts +from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts from scripts.dupscore.models import ClassFact, FunctionFact, ProjectFacts -from tests.unit.scripts.dupscore._helpers.facts._test_types import ( +from tests.unit.scripts.dupscore._helpers.package_pairs.facts._test_types import ( ExtractCallsTestCase, ExtractFieldsTestCase, ExtractModulesTestCase, ) -from tests.unit.scripts.dupscore._helpers.facts.helpers import find_class, find_function +from tests.unit.scripts.dupscore._helpers.package_pairs.facts.helpers import ( + find_class, + find_function, +) _PLANNER_SOURCE: str = ( "from sqlbuild.alpha.shared_leaves.leaves import load_rows\n" diff --git a/tests/unit/scripts/dupscore/_helpers/fusion/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/_test_types.py similarity index 100% rename from tests/unit/scripts/dupscore/_helpers/fusion/_test_types.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/_test_types.py diff --git a/tests/unit/scripts/dupscore/_helpers/fusion/helpers.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/helpers.py similarity index 100% rename from tests/unit/scripts/dupscore/_helpers/fusion/helpers.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/helpers.py diff --git a/tests/unit/scripts/dupscore/_helpers/fusion/test_fuse_rankings.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/test_fuse_rankings.py similarity index 92% rename from tests/unit/scripts/dupscore/_helpers/fusion/test_fuse_rankings.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/test_fuse_rankings.py index f82071d4f9..11ede8cbe5 100644 --- a/tests/unit/scripts/dupscore/_helpers/fusion/test_fuse_rankings.py +++ b/tests/unit/scripts/dupscore/_helpers/package_pairs/fusion/test_fuse_rankings.py @@ -2,18 +2,18 @@ import pytest -from scripts.dupscore._helpers.fusion import filter_report_to_domain, fuse_rankings +from scripts.dupscore._helpers.package_pairs.fusion import filter_report_to_domain, fuse_rankings from scripts.dupscore.constants import ( SIGNAL_NAME_CALLGRAPH, SIGNAL_NAME_DATACLASS_OVERLAP, SIGNAL_NAME_SAME_NAME, ) from scripts.dupscore.models import DupscoreConfig, DupscoreReport -from tests.unit.scripts.dupscore._helpers.fusion._test_types import ( +from tests.unit.scripts.dupscore._helpers.package_pairs.fusion._test_types import ( DomainFilterTestCase, FuseRankingsTestCase, ) -from tests.unit.scripts.dupscore._helpers.fusion.helpers import ranking_of +from tests.unit.scripts.dupscore._helpers.package_pairs.fusion.helpers import ranking_of _PAIR_A: tuple[str, str] = ("sqlbuild.alpha.executor", "sqlbuild.alpha.planner") _PAIR_B: tuple[str, str] = ("sqlbuild.alpha.cli", "sqlbuild.alpha.executor") diff --git a/tests/unit/scripts/dupscore/_helpers/signal_dataclasses/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/_test_types.py similarity index 100% rename from tests/unit/scripts/dupscore/_helpers/signal_dataclasses/_test_types.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/_test_types.py diff --git a/tests/unit/scripts/dupscore/_helpers/signal_dataclasses/test_dataclass_overlap_signal.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/test_dataclass_overlap_signal.py similarity index 89% rename from tests/unit/scripts/dupscore/_helpers/signal_dataclasses/test_dataclass_overlap_signal.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/test_dataclass_overlap_signal.py index e141caf474..9c51f75e73 100644 --- a/tests/unit/scripts/dupscore/_helpers/signal_dataclasses/test_dataclass_overlap_signal.py +++ b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_dataclasses/test_dataclass_overlap_signal.py @@ -2,10 +2,10 @@ import pytest -from scripts.dupscore._helpers.facts import extract_project_facts -from scripts.dupscore._helpers.signal_dataclasses import score_dataclass_overlap +from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts +from scripts.dupscore._helpers.package_pairs.signal_dataclasses import score_dataclass_overlap from scripts.dupscore.models import SignalRanking -from tests.unit.scripts.dupscore._helpers.signal_dataclasses._test_types import ( +from tests.unit.scripts.dupscore._helpers.package_pairs.signal_dataclasses._test_types import ( DataclassOverlapPairTestCase, DataclassOverlapSuppressionTestCase, ) diff --git a/tests/unit/scripts/dupscore/_helpers/signal_same_names/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/_test_types.py similarity index 100% rename from tests/unit/scripts/dupscore/_helpers/signal_same_names/_test_types.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/_test_types.py diff --git a/tests/unit/scripts/dupscore/_helpers/signal_same_names/test_same_name_signal.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/test_same_name_signal.py similarity index 89% rename from tests/unit/scripts/dupscore/_helpers/signal_same_names/test_same_name_signal.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/test_same_name_signal.py index 8fd547d567..b174c6c6c1 100644 --- a/tests/unit/scripts/dupscore/_helpers/signal_same_names/test_same_name_signal.py +++ b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_same_names/test_same_name_signal.py @@ -2,10 +2,10 @@ import pytest -from scripts.dupscore._helpers.facts import extract_project_facts -from scripts.dupscore._helpers.signal_same_names import score_same_name_symbols +from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts +from scripts.dupscore._helpers.package_pairs.signal_same_names import score_same_name_symbols from scripts.dupscore.models import SignalRanking -from tests.unit.scripts.dupscore._helpers.signal_same_names._test_types import ( +from tests.unit.scripts.dupscore._helpers.package_pairs.signal_same_names._test_types import ( SameNamePairTestCase, SameNameSuppressionTestCase, ) diff --git a/tests/unit/scripts/dupscore/_helpers/signal_state_fanin/_test_types.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/_test_types.py similarity index 100% rename from tests/unit/scripts/dupscore/_helpers/signal_state_fanin/_test_types.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/_test_types.py diff --git a/tests/unit/scripts/dupscore/_helpers/signal_state_fanin/test_state_fanin_signal.py b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/test_state_fanin_signal.py similarity index 92% rename from tests/unit/scripts/dupscore/_helpers/signal_state_fanin/test_state_fanin_signal.py rename to tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/test_state_fanin_signal.py index 344ad983cf..7380eadf57 100644 --- a/tests/unit/scripts/dupscore/_helpers/signal_state_fanin/test_state_fanin_signal.py +++ b/tests/unit/scripts/dupscore/_helpers/package_pairs/signal_state_fanin/test_state_fanin_signal.py @@ -2,10 +2,10 @@ import pytest -from scripts.dupscore._helpers.facts import extract_project_facts -from scripts.dupscore._helpers.signal_state_fanin import score_state_fanin +from scripts.dupscore._helpers.package_pairs.facts import extract_project_facts +from scripts.dupscore._helpers.package_pairs.signal_state_fanin import score_state_fanin from scripts.dupscore.models import DupscoreConfig, SignalRanking -from tests.unit.scripts.dupscore._helpers.signal_state_fanin._test_types import ( +from tests.unit.scripts.dupscore._helpers.package_pairs.signal_state_fanin._test_types import ( StateFaninPairTestCase, StateFaninSuppressionTestCase, ) diff --git a/tests/unit/scripts/dupscore/main/build_clone_report/_test_types.py b/tests/unit/scripts/dupscore/main/build_clone_report/_test_types.py new file mode 100644 index 0000000000..a917578c06 --- /dev/null +++ b/tests/unit/scripts/dupscore/main/build_clone_report/_test_types.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.dupscore.models import CloneAllowlistEntry + + +@dataclass(frozen=True) +class SeededCloneTestCase: + description: str + left_path: str + right_path: str + expected_category: str + + +@dataclass(frozen=True) +class ClusterShapeTestCase: + description: str + expected_languages: list[tuple[str, ...]] + expected_sizes: list[int] + + +@dataclass(frozen=True) +class ExcludedUnitTestCase: + description: str + include_tests: bool + name: str + expected_reported: bool + + +@dataclass(frozen=True) +class ReportedPathTestCase: + description: str + include_tests: bool + path: str + expected_reported: bool + + +@dataclass(frozen=True) +class AllowlistTestCase: + description: str + entries: tuple[CloneAllowlistEntry, ...] + expected_languages: list[tuple[str, ...]] + expected_allowlisted_pairs: int + + +@dataclass(frozen=True) +class PathFilterTestCase: + description: str + path_globs: tuple[str, ...] + expected_languages: list[tuple[str, ...]] + + +@dataclass(frozen=True) +class SinceFilterTestCase: + description: str + changed_files: dict[str, str] + expected_changes: dict[str, str | None] + + +@dataclass(frozen=True) +class SinceUnchangedTestCase: + description: str + changed_files: dict[str, str] + expected_cluster_count: int diff --git a/tests/unit/scripts/dupscore/main/build_clone_report/helpers.py b/tests/unit/scripts/dupscore/main/build_clone_report/helpers.py new file mode 100644 index 0000000000..6abd5c492c --- /dev/null +++ b/tests/unit/scripts/dupscore/main/build_clone_report/helpers.py @@ -0,0 +1,269 @@ +from __future__ import annotations + +from pathlib import Path + +from scripts.dupscore.constants import ( + DEFAULT_CLONE_MIN_SIMILARITY, + DEFAULT_CLONE_MIN_TOKENS, + SUPPORTED_LANGUAGES, +) +from scripts.dupscore.models import CloneCluster, CloneMember, CloneOptions, CloneReport +from tests.unit.scripts.dupscore.main.build_report.helpers import write_project_files + +PYTHON_BASE: str = ''' +def summarize_orders(orders, threshold): + """Rank customers by order totals.""" + totals = {} + for order in orders: + customer = order["customer"] + amount = order["amount"] + if amount < threshold: + continue + totals[customer] = totals.get(customer, 0) + amount + ranked = sorted(totals.items(), key=lambda item: item[1], reverse=True) + result = [] + for customer, total in ranked: + result.append(f"{customer}: {total}") + return result +''' + +PYTHON_EXACT_COPY: str = """ +import logging + + +@staticmethod +def summarize_orders(orders: list[dict[str, int]], threshold: int) -> list[str]: + # Copied without the docstring, with annotations and a comment. + totals: dict[str, int] = {} + for order in orders: + customer = order["customer"] + amount = order["amount"] + if amount < threshold: + continue + totals[customer] = totals.get(customer, 0) + amount + ranked = sorted(totals.items(), key=lambda item: item[1], reverse=True) + result = [] + for customer, total in ranked: + result.append(f"{customer}: {total}") + return result +""" + +PYTHON_RENAMED_COPY: str = """ +class InventoryReport: + @staticmethod + def rank_products(products, minimum): + per_product = {} + for product in products: + sku = product["sku"] + quantity = product["quantity"] + if quantity < minimum: + continue + per_product[sku] = per_product.get(sku, 1) + quantity + ordered = sorted(per_product.items(), key=lambda entry: entry[0], reverse=True) + lines = [] + for sku, count in ordered: + lines.append(f"{sku} = {count}") + return lines +""" + +PYTHON_NEAR_MISS_COPY: str = """ +def summarize_tickets(tickets, threshold): + totals = {} + for ticket in tickets: + queue = ticket["queue"] + minutes = ticket["minutes"] + if not queue: + continue + if minutes < threshold: + continue + totals[queue] = totals.get(queue, 0) + minutes + ranked = sorted(totals.items(), key=lambda item: item[1], reverse=True) + result = ["support queues"] + for queue, total in ranked: + result.append(f"{queue}: {total}") + return result +""" + +PYTHON_UNRELATED: str = """ +def parse_fulfillment_window(text): + parts = text.split("-") + if len(parts) != 2: + raise ValueError(text) + start, end = (int(value) for value in parts) + while start > end: + end += 24 + with open("windows.log", "a") as handle: + handle.write(str((start, end))) + try: + return {"start": start, "end": end, "span": end - start} + except KeyError: + return None +""" + +PYTHON_SHORT_HELPER: str = """ +def add_quantities(left, right): + return left + right +""" + +RUST_BASE: str = """ +use std::collections::HashMap; + +/// Rank inventory by quantity. +pub fn total_inventory(items: &[Item], minimum: u32) -> Vec { + let mut totals: HashMap = HashMap::new(); + for item in items { + if item.quantity < minimum { + continue; + } + *totals.entry(item.sku.clone()).or_insert(0) += item.quantity; + } + let mut ranked: Vec<(String, u32)> = totals.into_iter().collect(); + ranked.sort_by(|left, right| right.1.cmp(&left.1)); + ranked + .iter() + .map(|(sku, total)| format!("{sku}: {total}")) + .collect() +} +""" + +RUST_RENAMED_COPY: str = """ +pub struct OrderBook; + +impl OrderBook { + #[inline] + pub(crate) fn rank_orders(orders: &[Order], floor: u64) -> Vec { + let mut per_customer: HashMap = HashMap::new(); + for order in orders { + if order.amount < floor { + continue; + } + *per_customer.entry(order.customer.clone()).or_insert(7) += order.amount; + } + let mut sorted: Vec<(String, u64)> = per_customer.into_iter().collect(); + sorted.sort_by(|a, b| b.1.cmp(&a.1)); + sorted + .iter() + .map(|(customer, amount)| format!("{customer} -> {amount}")) + .collect() + } +} +""" + +RUST_NEAR_MISS_COPY: str = """ +/* A block comment /* with a nested { brace */ that must not confuse matching. */ +fn tally_products(products: &[Product], minimum: u32) -> Vec { + let mut totals: HashMap = HashMap::new(); + for product in products { + if product.sku.is_empty() { + continue; + } + if product.quantity < minimum { + continue; + } + *totals.entry(product.sku.clone()).or_insert(0) += product.quantity; + } + let mut ranked: Vec<(String, u32)> = totals.into_iter().collect(); + ranked.sort_by(|left, right| right.1.cmp(&left.1)); + ranked + .iter() + .map(|(sku, total)| format!(r#"{sku}: "{total}""#)) + .collect() +} +""" + +RUST_TEST_ONLY_COPIES: str = ( + """ +#[cfg(test)] +mod checks; + +#[cfg(test)] +mod tests { + use super::*; +""" + + RUST_BASE.replace("total_inventory", "inline_test_copy") + + """ +} +""" +) + +RUST_UNRELATED: str = """ +pub fn parse_window<'a>(text: &'a str) -> Option<(u32, u32)> { + let mut parts = text.split('-'); + let start: u32 = parts.next()?.trim().parse().ok()?; + let end: u32 = parts.next()?.trim().parse().ok()?; + match (start, end) { + (s, e) if s <= e => Some((s, e)), + (s, e) => Some((s, e + 24)), + } +} +""" + +RUST_EXACT_COPY: str = ( + RUST_BASE.replace("/// Rank inventory by quantity.", "// Kept in sync by hand.") + .replace("pub fn", "pub(crate) fn") + .replace(" .collect()", " .collect() // done") +) + +SEEDED_FILES: dict[str, str] = { + "src/sqlbuild/alpha/orders.py": PYTHON_BASE + PYTHON_SHORT_HELPER, + "src/sqlbuild/beta/orders_copy.py": PYTHON_EXACT_COPY, + "src/sqlbuild/beta/inventory.py": PYTHON_RENAMED_COPY + PYTHON_SHORT_HELPER, + "src/sqlbuild/gamma/support.py": PYTHON_NEAR_MISS_COPY, + "src/sqlbuild/gamma/windows.py": PYTHON_UNRELATED, + "crates/demo-rules/src/lib.rs": RUST_BASE + RUST_TEST_ONLY_COPIES, + "crates/demo-rules/src/checks.rs": RUST_BASE.replace("total_inventory", "external_test_copy"), + "crates/demo-rules/src/copy.rs": RUST_EXACT_COPY, + "crates/demo-rules/src/orders.rs": RUST_RENAMED_COPY, + "crates/demo-rules/src/products.rs": RUST_NEAR_MISS_COPY, + "crates/demo-rules/src/windows.rs": RUST_UNRELATED, + "crates/demo-rules/tests/integration.rs": RUST_BASE, +} + + +def clone_options( + *, + include_tests: bool = False, + since: str | None = None, + path_globs: tuple[str, ...] = (), +) -> CloneOptions: + return CloneOptions( + languages=SUPPORTED_LANGUAGES, + include_tests=include_tests, + min_tokens=DEFAULT_CLONE_MIN_TOKENS, + min_similarity=DEFAULT_CLONE_MIN_SIMILARITY, + path_globs=path_globs, + since=since, + ) + + +def seed_repository(tmp_path: Path) -> Path: + repo_root: Path = tmp_path / "repo" + write_project_files(repo_root=repo_root, files=SEEDED_FILES) + return repo_root + + +def link_categories(report: CloneReport) -> dict[frozenset[str], str]: + categories: dict[frozenset[str], str] = {} + for cluster in report.clusters: + paths: list[str] = [member.path for member in cluster.members] + for link in cluster.links: + categories[frozenset((paths[link.left], paths[link.right]))] = link.category + return categories + + +def reported_members(report: CloneReport) -> list[CloneMember]: + members: list[CloneMember] = [] + for cluster in report.clusters: + members.extend(cluster.members) + return members + + +def cluster_languages(report: CloneReport) -> list[tuple[str, ...]]: + languages: list[tuple[str, ...]] = [] + for cluster in report.clusters: + languages.append(tuple(sorted({member.language for member in cluster.members}))) + return sorted(languages) + + +def member_changes(cluster: CloneCluster) -> dict[str, str | None]: + return {member.path: member.change for member in cluster.members} diff --git a/tests/unit/scripts/dupscore/main/build_clone_report/test_build_clone_report.py b/tests/unit/scripts/dupscore/main/build_clone_report/test_build_clone_report.py new file mode 100644 index 0000000000..d9da41dda9 --- /dev/null +++ b/tests/unit/scripts/dupscore/main/build_clone_report/test_build_clone_report.py @@ -0,0 +1,368 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.dupscore.constants import ( + CATEGORY_EXACT, + CATEGORY_NEAR_MISS, + CATEGORY_RENAMED, + CHANGE_CHANGED, + CHANGE_NEW, +) +from scripts.dupscore.main.build_clone_report import build_clone_report +from scripts.dupscore.models import CloneAllowlistEntry, CloneReport, DupscoreConfig +from tests.unit.scripts.dupscore.main.build_clone_report._test_types import ( + AllowlistTestCase, + ClusterShapeTestCase, + ExcludedUnitTestCase, + PathFilterTestCase, + ReportedPathTestCase, + SeededCloneTestCase, + SinceFilterTestCase, + SinceUnchangedTestCase, +) +from tests.unit.scripts.dupscore.main.build_clone_report.helpers import ( + PYTHON_BASE, + PYTHON_NEAR_MISS_COPY, + clone_options, + cluster_languages, + link_categories, + member_changes, + reported_members, + seed_repository, +) +from tests.unit.scripts.dupscore.main.build_report.helpers import ( + commit_all, + initialize_repo, + write_project_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SeededCloneTestCase( + description="python copy with annotations and comments is exact", + left_path="src/sqlbuild/alpha/orders.py", + right_path="src/sqlbuild/beta/orders_copy.py", + expected_category=CATEGORY_EXACT, + ), + SeededCloneTestCase( + description="python method with renamed identifiers and literals is renamed", + left_path="src/sqlbuild/alpha/orders.py", + right_path="src/sqlbuild/beta/inventory.py", + expected_category=CATEGORY_RENAMED, + ), + SeededCloneTestCase( + description="python copy with extra statements is a near miss", + left_path="src/sqlbuild/alpha/orders.py", + right_path="src/sqlbuild/gamma/support.py", + expected_category=CATEGORY_NEAR_MISS, + ), + SeededCloneTestCase( + description="rust copy with different comments and visibility is exact", + left_path="crates/demo-rules/src/lib.rs", + right_path="crates/demo-rules/src/copy.rs", + expected_category=CATEGORY_EXACT, + ), + SeededCloneTestCase( + description="rust impl method with renamed identifiers is renamed", + left_path="crates/demo-rules/src/lib.rs", + right_path="crates/demo-rules/src/orders.rs", + expected_category=CATEGORY_RENAMED, + ), + SeededCloneTestCase( + description="rust copy with an extra guard and raw string is a near miss", + left_path="crates/demo-rules/src/lib.rs", + right_path="crates/demo-rules/src/products.rs", + expected_category=CATEGORY_NEAR_MISS, + ), + ], + ids=lambda case: case.description, +) +def test_given_seeded_clones_when_building_report_then_links_expected_category( + test_case: SeededCloneTestCase, + tmp_path: Path, +) -> None: + repo_root: Path = seed_repository(tmp_path) + + report: CloneReport = build_clone_report( + repo_root=repo_root, options=clone_options(), config=DupscoreConfig() + ) + + categories: dict[frozenset[str], str] = link_categories(report) + assert categories[frozenset((test_case.left_path, test_case.right_path))] == ( + test_case.expected_category + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ClusterShapeTestCase( + description="each language forms one four-member cluster", + expected_languages=[("python",), ("rust",)], + expected_sizes=[4, 4], + ) + ], + ids=lambda case: case.description, +) +def test_given_seeded_clones_when_building_report_then_groups_transitive_clusters( + test_case: ClusterShapeTestCase, + tmp_path: Path, +) -> None: + repo_root: Path = seed_repository(tmp_path) + + report: CloneReport = build_clone_report( + repo_root=repo_root, options=clone_options(), config=DupscoreConfig() + ) + + assert cluster_languages(report) == test_case.expected_languages + assert sorted(len(cluster.members) for cluster in report.clusters) == test_case.expected_sizes + + +@pytest.mark.parametrize( + "test_case", + [ + ExcludedUnitTestCase( + description="unrelated python function is not reported", + include_tests=False, + name="parse_fulfillment_window", + expected_reported=False, + ), + ExcludedUnitTestCase( + description="short duplicated helper is below the size threshold", + include_tests=False, + name="add_quantities", + expected_reported=False, + ), + ExcludedUnitTestCase( + description="unrelated rust function is not reported", + include_tests=False, + name="parse_window", + expected_reported=False, + ), + ExcludedUnitTestCase( + description="copy inside a cfg(test) module is skipped by default", + include_tests=False, + name="inline_test_copy", + expected_reported=False, + ), + ExcludedUnitTestCase( + description="copy in a cfg(test) external module file is skipped by default", + include_tests=False, + name="external_test_copy", + expected_reported=False, + ), + ExcludedUnitTestCase( + description="copy inside a cfg(test) module is reported with include-tests", + include_tests=True, + name="inline_test_copy", + expected_reported=True, + ), + ExcludedUnitTestCase( + description="copy in a cfg(test) external module is reported with include-tests", + include_tests=True, + name="external_test_copy", + expected_reported=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_seeded_tree_when_building_report_then_applies_unit_exclusions( + test_case: ExcludedUnitTestCase, + tmp_path: Path, +) -> None: + repo_root: Path = seed_repository(tmp_path) + + report: CloneReport = build_clone_report( + repo_root=repo_root, + options=clone_options(include_tests=test_case.include_tests), + config=DupscoreConfig(), + ) + + names: set[str] = {member.name for member in reported_members(report)} + assert (test_case.name in names) is test_case.expected_reported + + +@pytest.mark.parametrize( + "test_case", + [ + ReportedPathTestCase( + description="rust tests directory is skipped by default", + include_tests=False, + path="crates/demo-rules/tests/integration.rs", + expected_reported=False, + ), + ReportedPathTestCase( + description="rust tests directory is analyzed with include-tests", + include_tests=True, + path="crates/demo-rules/tests/integration.rs", + expected_reported=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_rust_test_directory_when_building_report_then_respects_include_tests( + test_case: ReportedPathTestCase, + tmp_path: Path, +) -> None: + repo_root: Path = seed_repository(tmp_path) + + report: CloneReport = build_clone_report( + repo_root=repo_root, + options=clone_options(include_tests=test_case.include_tests), + config=DupscoreConfig(), + ) + + paths: set[str] = {member.path for member in reported_members(report)} + assert (test_case.path in paths) is test_case.expected_reported + + +@pytest.mark.parametrize( + "test_case", + [ + AllowlistTestCase( + description="entry covering both sides suppresses the rust clones", + entries=(CloneAllowlistEntry(paths=("crates/*",), reason="Mirrored rule helpers."),), + expected_languages=[("python",)], + expected_allowlisted_pairs=6, + ), + AllowlistTestCase( + description="entry covering only one side keeps the pair", + entries=( + CloneAllowlistEntry( + paths=("crates/demo-rules/src/lib.rs",), reason="Only one side." + ), + ), + expected_languages=[("python",), ("rust",)], + expected_allowlisted_pairs=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_clone_allowlist_when_building_report_then_suppresses_covered_pairs( + test_case: AllowlistTestCase, + tmp_path: Path, +) -> None: + repo_root: Path = seed_repository(tmp_path) + + report: CloneReport = build_clone_report( + repo_root=repo_root, + options=clone_options(), + config=DupscoreConfig(clone_allowlist=test_case.entries), + ) + + assert report.allowlisted_pairs == test_case.expected_allowlisted_pairs + assert cluster_languages(report) == test_case.expected_languages + + +@pytest.mark.parametrize( + "test_case", + [ + PathFilterTestCase( + description="glob keeps only clusters with a matching member", + path_globs=("*/gamma/*",), + expected_languages=[("python",)], + ), + PathFilterTestCase( + description="glob without matches keeps nothing", + path_globs=("docs/*",), + expected_languages=[], + ), + ], + ids=lambda case: case.description, +) +def test_given_path_filter_when_building_report_then_keeps_matching_clusters( + test_case: PathFilterTestCase, + tmp_path: Path, +) -> None: + repo_root: Path = seed_repository(tmp_path) + + report: CloneReport = build_clone_report( + repo_root=repo_root, + options=clone_options(path_globs=test_case.path_globs), + config=DupscoreConfig(), + ) + + assert cluster_languages(report) == test_case.expected_languages + + +@pytest.mark.parametrize( + "test_case", + [ + SinceUnchangedTestCase( + description="unchanged worktree reports nothing", + changed_files={}, + expected_cluster_count=0, + ), + SinceUnchangedTestCase( + description="edits outside clone members report nothing", + changed_files={"src/sqlbuild/gamma/notes.py": "VALUE = 1\n"}, + expected_cluster_count=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_since_revision_without_clone_edits_when_building_report_then_reports_nothing( + test_case: SinceUnchangedTestCase, + tmp_path: Path, +) -> None: + repo_root: Path = seed_repository(tmp_path) + initialize_repo(repo_root) + base_revision: str = commit_all(repo_root=repo_root, message="seed clones") + write_project_files(repo_root=repo_root, files=test_case.changed_files) + + report: CloneReport = build_clone_report( + repo_root=repo_root, options=clone_options(since=base_revision), config=DupscoreConfig() + ) + + assert len(report.clusters) == test_case.expected_cluster_count + + +@pytest.mark.parametrize( + "test_case", + [ + SinceFilterTestCase( + description="untracked copy is new and keeps its cluster", + changed_files={"src/sqlbuild/delta/branch_copy.py": PYTHON_BASE}, + expected_changes={ + "src/sqlbuild/delta/branch_copy.py": CHANGE_NEW, + "src/sqlbuild/alpha/orders.py": None, + }, + ), + SinceFilterTestCase( + description="edited clone member is marked changed", + changed_files={ + "src/sqlbuild/gamma/support.py": PYTHON_NEAR_MISS_COPY.replace( + 'result = ["support queues"]', 'result = ["support queue totals"]' + ) + }, + expected_changes={ + "src/sqlbuild/gamma/support.py": CHANGE_CHANGED, + "src/sqlbuild/alpha/orders.py": None, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_since_revision_when_building_report_then_marks_touched_members( + test_case: SinceFilterTestCase, + tmp_path: Path, +) -> None: + repo_root: Path = seed_repository(tmp_path) + initialize_repo(repo_root) + base_revision: str = commit_all(repo_root=repo_root, message="seed clones") + write_project_files(repo_root=repo_root, files=test_case.changed_files) + + report: CloneReport = build_clone_report( + repo_root=repo_root, options=clone_options(since=base_revision), config=DupscoreConfig() + ) + + assert len(report.clusters) == 1 + changes: dict[str, str | None] = member_changes(report.clusters[0]) + assert {path: changes[path] for path in test_case.expected_changes} == ( + test_case.expected_changes + ) diff --git a/tests/unit/scripts/dupscore/main/dupscore/_test_types.py b/tests/unit/scripts/dupscore/main/dupscore/_test_types.py new file mode 100644 index 0000000000..707a8253c8 --- /dev/null +++ b/tests/unit/scripts/dupscore/main/dupscore/_test_types.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CliJsonTestCase: + description: str + arguments: tuple[str, ...] + expected_total_clusters: int + expected_unit_counts: dict[str, int] + expected_cluster_keys: frozenset[str] + expected_member_keys: frozenset[str] + expected_progress_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class CliTextTestCase: + description: str + arguments: tuple[str, ...] + expected_prefix: str + expected_fragments: tuple[str, ...] + absent_fragment: str + + +@dataclass(frozen=True) +class CliSinceTestCase: + description: str + new_files: dict[str, str] + expected_fragments: tuple[str, ...] + expected_new_markers: int + + +@dataclass(frozen=True) +class CliUsageErrorTestCase: + description: str + arguments: tuple[str, ...] + expected_error_fragment: str + + +@dataclass(frozen=True) +class CliHelpTestCase: + description: str + expected_fragments: tuple[str, ...] diff --git a/tests/unit/scripts/dupscore/main/dupscore/test_dupscore_cli.py b/tests/unit/scripts/dupscore/main/dupscore/test_dupscore_cli.py new file mode 100644 index 0000000000..615e556dc1 --- /dev/null +++ b/tests/unit/scripts/dupscore/main/dupscore/test_dupscore_cli.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from scripts.dupscore.main.dupscore import dupscore +from tests.unit.scripts.dupscore.main.build_clone_report.helpers import ( + PYTHON_BASE, + seed_repository, +) +from tests.unit.scripts.dupscore.main.build_report.helpers import ( + commit_all, + initialize_repo, + write_project_files, +) +from tests.unit.scripts.dupscore.main.dupscore._test_types import ( + CliHelpTestCase, + CliJsonTestCase, + CliSinceTestCase, + CliTextTestCase, + CliUsageErrorTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CliJsonTestCase( + description="json output lists ranked clusters with member locations", + arguments=("clones", "--json", "--top", "1"), + expected_total_clusters=2, + expected_unit_counts={"python": 5, "rust": 5}, + expected_cluster_keys=frozenset( + { + "category", + "duplicated_tokens", + "links", + "members", + "rank", + "similarity_max", + "similarity_min", + } + ), + expected_member_keys=frozenset( + {"change", "end_line", "language", "name", "path", "start_line", "tokens"} + ), + expected_progress_fragments=( + "detecting function clones", + "found 2 clone clusters across 10 units", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_seeded_tree_when_running_json_cli_then_prints_cluster_document( + test_case: CliJsonTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + repo_root: Path = seed_repository(tmp_path) + + exit_code: int = dupscore([*test_case.arguments, "--repo-root", str(repo_root)]) + + output: str + errors: str + output, errors = capsys.readouterr() + payload: dict[str, Any] = json.loads(output) + clusters: list[dict[str, Any]] = payload["clusters"] + members: list[dict[str, Any]] = clusters[0]["members"] + assert exit_code == 0 + assert payload["total_clusters"] == test_case.expected_total_clusters + assert payload["unit_counts"] == test_case.expected_unit_counts + assert payload["since"] is None + assert [set(cluster) for cluster in clusters] == [set(test_case.expected_cluster_keys)] + assert all(set(member) == test_case.expected_member_keys for member in members) + assert all(fragment in errors for fragment in test_case.expected_progress_fragments) + + +@pytest.mark.parametrize( + "test_case", + [ + CliTextTestCase( + description="default mode prints python clone locations", + arguments=("--lang", "python"), + expected_prefix="dupscore clones: 1 clusters (python 5 units;", + expected_fragments=( + "src/sqlbuild/alpha/orders.py:2-15 summarize_orders", + "src/sqlbuild/beta/inventory.py:4-16 InventoryReport.rank_products", + ), + absent_fragment="crates/", + ), + CliTextTestCase( + description="rust mode prints impl-qualified names", + arguments=("clones", "--lang", "rust"), + expected_prefix="dupscore clones: 1 clusters (rust 5 units;", + expected_fragments=("crates/demo-rules/src/orders.rs:6-20 OrderBook::rank_orders",), + absent_fragment="src/sqlbuild/", + ), + ], + ids=lambda case: case.description, +) +def test_given_seeded_tree_when_running_text_cli_then_prints_locations( + test_case: CliTextTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + repo_root: Path = seed_repository(tmp_path) + + exit_code: int = dupscore([*test_case.arguments, "--repo-root", str(repo_root)]) + + output: str = capsys.readouterr().out + assert exit_code == 0 + assert output.startswith(test_case.expected_prefix) + assert all(fragment in output for fragment in test_case.expected_fragments) + assert test_case.absent_fragment not in output + + +@pytest.mark.parametrize( + "test_case", + [ + CliSinceTestCase( + description="branch copy is the only new member", + new_files={"src/sqlbuild/delta/branch_copy.py": PYTHON_BASE}, + expected_fragments=( + ": 1 clusters (", + "src/sqlbuild/delta/branch_copy.py:2-15 summarize_orders (118 tokens) [new]", + ), + expected_new_markers=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_branch_copy_when_running_cli_since_base_then_marks_new_member( + test_case: CliSinceTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + repo_root: Path = seed_repository(tmp_path) + initialize_repo(repo_root) + base_revision: str = commit_all(repo_root=repo_root, message="seed clones") + write_project_files(repo_root=repo_root, files=test_case.new_files) + + exit_code: int = dupscore(["--repo-root", str(repo_root), "--since", base_revision]) + + output: str = capsys.readouterr().out + assert exit_code == 0 + assert output.startswith(f"dupscore clones changed since {base_revision}") + assert all(fragment in output for fragment in test_case.expected_fragments) + assert output.count("[new]") == test_case.expected_new_markers + + +@pytest.mark.parametrize( + "test_case", + [ + CliUsageErrorTestCase( + description="clone options are rejected in report mode", + arguments=("report", "--min-tokens", "10"), + expected_error_fragment="--min-tokens only apply to the clones mode", + ), + CliUsageErrorTestCase( + description="similarity outside the unit interval is rejected", + arguments=("clones", "--min-similarity", "1.5"), + expected_error_fragment="--min-similarity must be in (0, 1]", + ), + CliUsageErrorTestCase( + description="unknown languages are rejected", + arguments=("--lang", "go"), + expected_error_fragment="invalid choice: 'go'", + ), + CliUsageErrorTestCase( + description="domain filter is rejected in clones mode", + arguments=("--domain", "sqlbuild.compiler"), + expected_error_fragment="--domain only applies to the report mode", + ), + CliUsageErrorTestCase( + description="clones mode takes no package names", + arguments=("clones", "sqlbuild.alpha"), + expected_error_fragment="takes no positional package names", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_arguments_when_running_cli_then_exits_with_usage_error( + test_case: CliUsageErrorTestCase, + capsys: pytest.CaptureFixture[str], +) -> None: + with pytest.raises(SystemExit) as raised: + _ = dupscore(list(test_case.arguments)) + + assert raised.value.code == 2 + assert test_case.expected_error_fragment in capsys.readouterr().err + + +@pytest.mark.parametrize( + "test_case", + [ + CliHelpTestCase( + description="help lists the clone mode and its options", + expected_fragments=("clones", "report", "--include-tests", "--since", "--path"), + ) + ], + ids=lambda case: case.description, +) +def test_given_help_flag_when_running_cli_then_lists_clone_mode( + test_case: CliHelpTestCase, + capsys: pytest.CaptureFixture[str], +) -> None: + with pytest.raises(SystemExit) as raised: + _ = dupscore(["--help"]) + + help_text: str = capsys.readouterr().out + assert raised.value.code == 0 + assert all(fragment in help_text for fragment in test_case.expected_fragments) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py index 619c23f7d2..a3fc989973 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py @@ -35,3 +35,11 @@ class QueryDiffParsingTestCase: expected_right_query: str | None expected_keys: tuple[str, ...] expected_unkeyed: bool + + +@dataclass(frozen=True) +class EventExportWarningTestCase: + description: str + exporter_counts: tuple[tuple[str, int, int, int, int], ...] + command_succeeded: bool + expected_message: str | None diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py new file mode 100644 index 0000000000..422af92b1a --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/helpers.py @@ -0,0 +1,33 @@ +"""Local helpers for CLI entry tests.""" + +from __future__ import annotations + +from sqlbuild.runtime.event_exporting.models import ( + EventExporterAccounting, + EventExporterCounts, + EventExportSummary, +) + + +def build_export_summary(counts: tuple[tuple[str, int, int, int, int], ...]) -> EventExportSummary: + per_exporter: tuple[EventExporterAccounting, ...] = tuple( + EventExporterAccounting( + exporter_name=name, + counts=EventExporterCounts( + accepted=accepted, delivered=delivered, failed=failed, dropped=dropped + ), + ) + for name, accepted, delivered, failed, dropped in counts + ) + return EventExportSummary( + aggregate=EventExporterCounts( + accepted=sum(item.counts.accepted for item in per_exporter), + delivered=sum(item.counts.delivered for item in per_exporter), + failed=sum(item.counts.failed for item in per_exporter), + dropped=sum(item.counts.dropped for item in per_exporter), + ), + per_exporter=per_exporter, + queue_depth=0, + queue_capacity=1024, + flush_complete=all(item.counts.dropped == 0 for item in per_exporter), + ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py new file mode 100644 index 0000000000..2a445718e5 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_format_event_export_warning.py @@ -0,0 +1,67 @@ +"""Incomplete lifecycle-event export warning formatting.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.cli.commands._helpers.entry.observability import ( + format_event_export_warning, +) +from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( + EventExportWarningTestCase, +) +from tests.unit.src.sqlbuild.cli.commands._helpers.entry.helpers import build_export_summary + + +@pytest.mark.parametrize( + "test_case", + ( + EventExportWarningTestCase( + "all delivered", + (("orders_events", 248, 248, 0, 0),), + True, + None, + ), + EventExportWarningTestCase( + "successful command with drops", + (("orders_events", 248, 108, 0, 140),), + True, + "Warning: command completed successfully, but lifecycle event export was " + "incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). " + "Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + EventExportWarningTestCase( + "failed command with failures", + (("orders_events", 10, 9, 1, 0),), + False, + "Warning: lifecycle event export was incomplete: 0 of 10 events dropped, " + "1 failed (sink 'orders_events'). " + "Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + EventExportWarningTestCase( + "multiple sinks names only affected sinks", + ( + ("customer_events", 20, 20, 0, 0), + ("orders_events", 20, 15, 1, 4), + ), + True, + "Warning: command completed successfully, but lifecycle event export was " + "incomplete: 4 of 40 event deliveries dropped, 1 failed (sink 'orders_events'). " + "Increase sinks.lifecycle.shutdown_timeout or check sink health.", + ), + ), + ids=lambda case: case.description, +) +def test_given_final_export_summary_when_formatting_warning_then_reports_loss_only( + test_case: EventExportWarningTestCase, +) -> None: + message: str | None = format_event_export_warning( + summary=build_export_summary(test_case.exporter_counts), + command_succeeded=test_case.command_succeeded, + ) + + assert message == test_case.expected_message + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py index c827d6d2af..0d0a17d142 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -157,3 +157,13 @@ class ResolveEffectiveAdapterNameTestCase: class ExpectedMessageTestCase: description: str expected_message: str + + +@dataclass(frozen=True) +class RuleGatedTestPlanningTestCase: + description: str + min_prepared_models: int + test_header: str + expected_codes: tuple[str, ...] + expected_message_fragment: str + expected_inline_planning_calls: int diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 33ca4f65a2..5434cc973c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -522,3 +522,38 @@ def build_runtime_target_execution_result() -> BuildExecutionResult: ), success_count=1, ) + + +def prepare_rule_gated_compile_project(root: Path, *, test_header: str) -> Path: + """Create a project with a rule error and a SQL test that has a planning error.""" + + project_dir: Path = prepare_static_compile_project(root) + with (project_dir / "sqlbuild_project.toml").open("a", encoding="utf-8") as config: + config.write('\n[rules]\nselect = ["SQBRMODEL102"]\n') + (project_dir / "sources").mkdir() + (project_dir / "sources" / "raw.yml").write_text( + "sources:\n" + " - name: raw_orders\n schema: main\n table: raw_orders\n" + " - name: raw_refunds\n schema: main\n table: raw_refunds\n", + encoding="utf-8", + ) + (project_dir / "models" / "orders.sql").write_text( + "MODEL (materialized view);\n\n" + 'SELECT o.order_id, r.refund_id FROM __source("raw_orders") AS o\n' + 'LEFT JOIN __source("raw_refunds") AS r ON r.order_id = o.order_id\n', + encoding="utf-8", + ) + (project_dir / "models" / "bad_star.sql").write_text( + 'MODEL (materialized view);\n\nSELECT * FROM __source("raw_orders")\n', + encoding="utf-8", + ) + (project_dir / "tests" / "unit").mkdir(parents=True) + (project_dir / "tests" / "unit" / "orders_case.sql").write_text( + f"TEST ({test_header});\n" + "WITH\n" + "__source__raw_orders AS (SELECT 1 AS order_id),\n" + "__expected__orders AS (SELECT 1 AS order_id)\n" + "SELECT 1\n", + encoding="utf-8", + ) + return project_dir diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 9dfd8d72dc..252ef119be 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -1,19 +1,22 @@ from __future__ import annotations import json +from collections.abc import Callable from pathlib import Path -from typing import cast +from typing import Any, cast import pytest from sqlbuild.cli.commands._helpers.compile import lineage as compile_lineage from sqlbuild.cli.commands._helpers.compile import pipeline as compile_pipeline from sqlbuild.cli.commands._helpers.compile import status as compile_status +from sqlbuild.cli.commands.classes import prepared_compile_artifacts from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.main.run import run_compile from sqlbuild.cli.compile.models import ( CompileCommandRequest, ) +from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( @@ -23,10 +26,12 @@ CompileJsonDiagnosticsTestCase, CompileLineageModeTestCase, CompilePythonDagArtifactTestCase, + RuleGatedTestPlanningTestCase, ) from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( NoConnectDuckDbAdapter, prepare_python_compile_project, + prepare_rule_gated_compile_project, prepare_static_compile_project, ) @@ -537,3 +542,86 @@ def test_given_lineage_disabled_when_building_compile_lineage_then_skips_analyze assert lineage is None assert test_case.expected_lineage_mode_values == () + + +@pytest.mark.parametrize( + "test_case", + [ + RuleGatedTestPlanningTestCase( + description="inline writer plans once for withheld artifacts", + min_prepared_models=128, + test_header='name "orders_case"', + expected_codes=("SQBRMODEL102", "S000"), + expected_message_fragment="references __source('raw_refunds') which has no mock", + expected_inline_planning_calls=1, + ), + RuleGatedTestPlanningTestCase( + description="background preparation reuses staged planning", + min_prepared_models=1, + test_header='name "orders_case"', + expected_codes=("SQBRMODEL102", "S000"), + expected_message_fragment="references __source('raw_refunds') which has no mock", + expected_inline_planning_calls=0, + ), + RuleGatedTestPlanningTestCase( + description="inline planning input error becomes a diagnostic", + min_prepared_models=128, + test_header='name "orders_case", cursor_start "2026-02-01"', + expected_codes=("SQBRMODEL102", "P001"), + expected_message_fragment="declares cursor_start or cursor_end", + expected_inline_planning_calls=1, + ), + RuleGatedTestPlanningTestCase( + description="background planning input error becomes a diagnostic", + min_prepared_models=1, + test_header='name "orders_case", cursor_start "2026-02-01"', + expected_codes=("SQBRMODEL102", "P001"), + expected_message_fragment="declares cursor_start or cursor_end", + expected_inline_planning_calls=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_rule_error_when_compiling_json_then_withheld_test_planning_errors_are_reported( + test_case: RuleGatedTestPlanningTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = prepare_rule_gated_compile_project( + tmp_path, test_header=test_case.test_header + ) + planning_calls: list[Path] = [] + static_planning: Callable[..., tuple[CompilerDiagnostic, ...]] = ( + compile_pipeline.static_sql_test_planning_diagnostics + ) + + def counted_static_planning(**kwargs: Any) -> tuple[CompilerDiagnostic, ...]: + planning_calls.append(kwargs["target_dir"]) + return static_planning(**kwargs) + + monkeypatch.setattr( + compile_pipeline, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter() + ) + monkeypatch.setattr( + compile_pipeline, "static_sql_test_planning_diagnostics", counted_static_planning + ) + monkeypatch.setattr( + prepared_compile_artifacts, + "_MIN_PREPARED_ARTIFACT_MODELS", + test_case.min_prepared_models, + ) + + exit_code: int = run_compile( + CompileCommandRequest(project_dir=project_dir, json_output=True, no_cache=True) + ) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert exit_code == 1 + assert tuple(diagnostic["code"] for diagnostic in payload["diagnostics"]) == ( + test_case.expected_codes + ) + assert test_case.expected_message_fragment in payload["diagnostics"][-1]["message"] + assert payload["summary"]["errors"] == len(test_case.expected_codes) + assert len(planning_calls) == test_case.expected_inline_planning_calls + assert not (project_dir / "target" / "compiled" / "tests").exists() diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 7463f1b6b3..db48cfebc5 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -580,6 +580,14 @@ class ParseSqlTestFileErrorTestCase: expected_error_fragment: str +@dataclass(frozen=True) +class ParseSqlTestCursorWindowTestCase: + description: str + contents: str + expected_cursor_start: str | None + expected_cursor_end: str | None + + @dataclass(frozen=True) class ParseSqlScenarioFileTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py index 3eb50c8684..04869fef72 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_tests.py @@ -23,6 +23,7 @@ ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ExpectedCountTestCase, + ParseSqlTestCursorWindowTestCase, ParseSqlTestFileErrorTestCase, ParseSqlTestFileTestCase, ) @@ -410,6 +411,19 @@ def test_given_sql_test_file_variants_when_parsing_then_it_returns_expected_raw_ contents="TEST (parameters (status string), cases (one (status null)));\nSELECT 1\n", expected_error_fragment="is not nullable", ), + ParseSqlTestFileErrorTestCase( + description="rejects a cursor window outside model tests", + contents='TEST (mode macro, cursor_start "2026-02-01");\nSELECT 1\n', + expected_error_fragment=( + "declares `cursor_start` or `cursor_end`, which are only supported for model " + "tests, not mode 'macro'" + ), + ), + ParseSqlTestFileErrorTestCase( + description="rejects a boolean cursor bound", + contents="TEST (cursor_end true);\nSELECT 1\n", + expected_error_fragment="cursor_end in '.*' must be a non-empty string or an integer", + ), ParseSqlTestFileErrorTestCase( description="rejects malformed case names", contents=( @@ -426,3 +440,43 @@ def test_given_invalid_sql_test_file_contents_when_parsing_then_it_raises_clear_ ) -> None: with pytest.raises(SqlTestParseError, match=test_case.expected_error_fragment): parse_sql_test_file(contents=test_case.contents, file_path=Path("tests/unit/orders.sql")) + + +@pytest.mark.parametrize( + "test_case", + [ + ParseSqlTestCursorWindowTestCase( + description="omitted keys leave the default window", + contents='TEST (name "orders");\nSELECT 1\n', + expected_cursor_start=None, + expected_cursor_end=None, + ), + ParseSqlTestCursorWindowTestCase( + description="timestamp bounds are kept as authored", + contents=( + 'TEST (name "orders", cursor_start "2026-02-01", cursor_end "2026-02-03");\n' + "SELECT 1\n" + ), + expected_cursor_start="2026-02-01", + expected_cursor_end="2026-02-03", + ), + ParseSqlTestCursorWindowTestCase( + description="integer bounds are normalized to text independently", + contents='TEST (name "orders", cursor_end 20);\nSELECT 1\n', + expected_cursor_start=None, + expected_cursor_end="20", + ), + ], + ids=lambda case: case.description, +) +def test_given_test_cursor_window_header_when_parsing_then_bounds_are_recorded( + test_case: ParseSqlTestCursorWindowTestCase, +) -> None: + blocks: tuple[DiscoveredSqlTestBlock, ...] = parse_sql_test_file( + contents=test_case.contents, file_path=Path("tests/unit/orders.sql") + ) + + assert (blocks[0].cursor_start, blocks[0].cursor_end) == ( + test_case.expected_cursor_start, + test_case.expected_cursor_end, + ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py index 12eb5852b7..9a8304483f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_test_types.py @@ -16,6 +16,13 @@ class EventExporterConfigTestCase: expected_min_severity: str +@dataclass(frozen=True) +class LifecycleShutdownTimeoutTestCase: + description: str + config: str + expected_seconds: int | None + + @dataclass(frozen=True) class DiscoverProjectInputsTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py b/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py index b1b6696d78..ea76c13484 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_runtime_event_exporter_config.py @@ -4,10 +4,14 @@ from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.discovery.main.runtime_extensions import discover_runtime_extensions -from sqlbuild.compiler.discovery.models import DiscoveredEventExporter +from sqlbuild.compiler.discovery.models import ( + DiscoveredEventExporter, + DiscoveredRuntimeExtensions, +) from tests.unit.src.sqlbuild.compiler.discovery._test_types import ( EventExporterConfigTestCase, InvalidEventExporterConfigTestCase, + LifecycleShutdownTimeoutTestCase, ) from tests.unit.src.sqlbuild.compiler.discovery.helpers import write_lifecycle_event_sink_project @@ -37,14 +41,40 @@ def test_given_global_and_named_filters_when_discovering_then_effective_filter_i exporter_config=test_case.config, ) - _, exporters, output_sinks = discover_runtime_extensions(project_dir=tmp_path) + extensions: DiscoveredRuntimeExtensions = discover_runtime_extensions(project_dir=tmp_path) - assert output_sinks == () - exporter: DiscoveredEventExporter = exporters[0] + assert extensions.command_output_sinks == () + assert extensions.lifecycle_shutdown_timeout_seconds is None + exporter: DiscoveredEventExporter = extensions.event_exporters[0] assert exporter.event_kinds == test_case.expected_kinds assert exporter.min_severity == test_case.expected_min_severity +@pytest.mark.parametrize( + "test_case", + ( + LifecycleShutdownTimeoutTestCase("default", "", None), + LifecycleShutdownTimeoutTestCase("zero", '[sinks.lifecycle]\nshutdown_timeout = "0s"\n', 0), + LifecycleShutdownTimeoutTestCase( + "seconds", '[sinks.lifecycle]\nshutdown_timeout = "30s"\n', 30 + ), + LifecycleShutdownTimeoutTestCase( + "upper bound", '[sinks.lifecycle]\nshutdown_timeout = "10m"\n', 600 + ), + ), + ids=lambda case: case.description, +) +def test_given_lifecycle_shutdown_timeout_when_discovering_then_exposes_drain_seconds( + tmp_path: Path, test_case: LifecycleShutdownTimeoutTestCase +) -> None: + write_lifecycle_event_sink_project(project_dir=tmp_path, exporter_config=test_case.config) + + extensions: DiscoveredRuntimeExtensions = discover_runtime_extensions(project_dir=tmp_path) + + assert extensions.lifecycle_shutdown_timeout_seconds == test_case.expected_seconds + assert len(extensions.event_exporters) == 1 + + @pytest.mark.parametrize( "test_case", ( @@ -71,6 +101,31 @@ def test_given_global_and_named_filters_when_discovering_then_effective_filter_i '[sinks.lifecycle.named.publish]\nminimum_severity = "info"\n', "unknown key.*minimum_severity", ), + InvalidEventExporterConfigTestCase( + "negative shutdown timeout", + '[sinks.lifecycle]\nshutdown_timeout = "-1s"\n', + "sinks.lifecycle.shutdown_timeout must be a fixed duration from '0s' to '10m'", + ), + InvalidEventExporterConfigTestCase( + "numeric shutdown timeout", + "[sinks.lifecycle]\nshutdown_timeout = 5\n", + "sinks.lifecycle.shutdown_timeout must be a fixed duration", + ), + InvalidEventExporterConfigTestCase( + "calendar shutdown timeout", + '[sinks.lifecycle]\nshutdown_timeout = "1mo"\n', + "sinks.lifecycle.shutdown_timeout must be a fixed duration", + ), + InvalidEventExporterConfigTestCase( + "shutdown timeout above upper bound", + '[sinks.lifecycle]\nshutdown_timeout = "11m"\n', + "sinks.lifecycle.shutdown_timeout must be a fixed duration", + ), + InvalidEventExporterConfigTestCase( + "named shutdown timeout", + '[sinks.lifecycle.named.publish]\nshutdown_timeout = "5s"\n', + "unknown key.*shutdown_timeout", + ), InvalidEventExporterConfigTestCase( "legacy exporter config", '[event_exporters]\nmin_severity = "info"\n', diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/_test_types.py index b0ad04bf83..6c50339415 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/_test_types.py @@ -74,3 +74,14 @@ class NativePlanningDifferentialTestCase: planning_case: PlanTestChainTestCase sql_analysis_enabled: bool = True expected_matches: bool = True + + +@dataclass(frozen=True) +class CursorWindowRenderingTestCase: + description: str + adapter_name: str + cursor_type: str + cursor_grain: str | None + cursor_start: str | None + cursor_end: str | None + expected_sql: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py index 59871f47f3..295fcc27ba 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/helpers.py @@ -10,6 +10,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile._helpers.render.macros import expand_sql_macros from sqlbuild.compiler.compile.constants import ( ASSERT_TEST_CTE_PREFIX, @@ -493,3 +494,62 @@ def _build_test_sql_body( for name, body in expected_bodies.items(): parts.append(f"{EXPECTED_TEST_CTE_PREFIX}{name} AS ({body})") return ("WITH " + ", ".join(parts) + " SELECT 1", "SELECT 1")[not parts] + + +CURSOR_WINDOW_MODEL_SQL: str = ( + "SELECT order_date FROM orders " + "WHERE order_date >= __cursor_start() AND order_date < __cursor_end()" +) +CURSOR_WINDOW_ADAPTERS: dict[str, BaseAdapter] = { + "duckdb": DuckDbAdapter(), + "snowflake": SnowflakeAdapter(), +} + + +def build_cursor_model(*, cursor_type: str, cursor_grain: str | None) -> CompiledModel: + """Build one incremental model whose query uses both cursor intrinsics.""" + + values: dict[str, object] = { + "materialized": "incremental", + "cursor": "order_date", + "cursor_type": cursor_type, + "cursor_grain": cursor_grain, + } + return CompiledModel( + key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="daily_orders"), + deps=(), + name="daily_orders", + relative_path=Path("models/daily_orders.sql"), + query_sql=CURSOR_WINDOW_MODEL_SQL, + config=CompileModelConfig(values=values), + destination=CompiledRelationLocation( + database=None, + schema="main", + name="daily_orders", + qualified_name="main.daily_orders", + ), + ) + + +def build_windowed_sql_test(*, cursor_start: str | None, cursor_end: str | None) -> CompiledSqlTest: + """Build one model SQL test whose header declares the given cursor window.""" + + return CompiledSqlTest( + key=CompiledObjectKey(resource_type=CompiledResourceType.SQL_TEST, name="orders_case"), + scope_deps=(), + name="orders_case", + test_file=DiscoveredSqlTestFile( + file_path=Path("tests/unit/orders_case.sql"), + relative_path=Path("tests/unit/orders_case.sql"), + contents="", + blocks=(), + ), + test_block=DiscoveredSqlTestBlock( + test_index=1, + header_values={}, + sql_body="", + cursor_start=cursor_start, + cursor_end=cursor_end, + ), + sql_body="", + ) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py new file mode 100644 index 0000000000..6532187bc8 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_cursor_window.py @@ -0,0 +1,110 @@ +"""Unit tests for cursor intrinsics rendered in SQL-test model expansion.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.planner._helpers.sql_tests.cursor_window import ( + render_test_cursor_intrinsics, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly._test_types import ( + CursorWindowRenderingTestCase, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.sql_test_assembly.helpers import ( + CURSOR_WINDOW_ADAPTERS, + CURSOR_WINDOW_MODEL_SQL, + build_cursor_model, + build_windowed_sql_test, +) + + +@pytest.mark.parametrize( + "test_case", + ( + CursorWindowRenderingTestCase( + description="duckdb timestamp default window", + adapter_name="duckdb", + cursor_type="timestamp", + cursor_grain="day", + cursor_start=None, + cursor_end=None, + expected_sql=( + "SELECT order_date FROM orders WHERE order_date >= " + "TIMESTAMP '1900-01-01 00:00:00' AND order_date < TIMESTAMP '2999-12-31 00:00:00'" + ), + ), + CursorWindowRenderingTestCase( + description="snowflake timestamp default window uses its typed literal", + adapter_name="snowflake", + cursor_type="timestamp", + cursor_grain="day", + cursor_start=None, + cursor_end=None, + expected_sql=( + "SELECT order_date FROM orders WHERE order_date >= " + "'1900-01-01 00:00:00' AND order_date < '2999-12-31 00:00:00'" + ), + ), + CursorWindowRenderingTestCase( + description="integer default window", + adapter_name="snowflake", + cursor_type="integer", + cursor_grain=None, + cursor_start=None, + cursor_end=None, + expected_sql=( + "SELECT order_date FROM orders WHERE order_date >= -1000000000000000 " + "AND order_date < 1000000000000000" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_model_using_cursor_intrinsics_when_rendering_default_window_then_bounds_are_typed( + test_case: CursorWindowRenderingTestCase, +) -> None: + rendered: str = render_test_cursor_intrinsics( + sql=CURSOR_WINDOW_MODEL_SQL, + model=build_cursor_model( + cursor_type=test_case.cursor_type, cursor_grain=test_case.cursor_grain + ), + adapter=CURSOR_WINDOW_ADAPTERS[test_case.adapter_name], + test=None, + ) + + assert rendered == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + ( + CursorWindowRenderingTestCase( + description="declared start keeps the default end", + adapter_name="duckdb", + cursor_type="timestamp", + cursor_grain="day", + cursor_start="2026-02-01", + cursor_end=None, + expected_sql=( + "SELECT order_date FROM orders WHERE order_date >= TIMESTAMP '2026-02-01' " + "AND order_date < TIMESTAMP '2999-12-31 00:00:00'" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_declared_test_window_when_rendering_then_declared_bounds_replace_defaults( + test_case: CursorWindowRenderingTestCase, +) -> None: + rendered: str = render_test_cursor_intrinsics( + sql=CURSOR_WINDOW_MODEL_SQL, + model=build_cursor_model( + cursor_type=test_case.cursor_type, cursor_grain=test_case.cursor_grain + ), + adapter=CURSOR_WINDOW_ADAPTERS[test_case.adapter_name], + test=build_windowed_sql_test( + cursor_start=test_case.cursor_start, cursor_end=test_case.cursor_end + ), + ) + + assert rendered == test_case.expected_sql diff --git a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py index 276d0eeb4a..304ed6fc8f 100644 --- a/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py +++ b/tests/unit/src/sqlbuild/executor/auditing/main/test_project_results.py @@ -15,6 +15,13 @@ from sqlbuild.executor.audit_results.exceptions import AuditResultStorageError from sqlbuild.executor.audit_results.models import AuditResultRecord from sqlbuild.executor.auditing.main._project_results import project_audit_result_batch +from sqlbuild.executor.auditing.main.audit_result_publication_scope import ( + audit_result_publication_scope, +) +from sqlbuild.executor.auditing.main.publish_completed_audit_results import ( + publish_completed_audit_results, +) +from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.runtime.observability.classes.event_dispatcher import EventDispatcher from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError from sqlbuild.runtime.observability.main.dispatcher_scope import dispatcher_scope @@ -178,11 +185,11 @@ def test_given_record_build_failure_when_projected_then_reports_degradation_with monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, ) -> None: - def fail_record_build(**kwargs: object) -> tuple[AuditResultRecord, ...]: + def fail_record_build(**kwargs: object) -> AuditResultRecord: del kwargs raise RuntimeError("invalid record") - monkeypatch.setattr(projection_module, "_build_records", fail_record_build) + monkeypatch.setattr(projection_module, "_build_record", fail_record_build) with identity_scope(ExecutionIdentity(invocation_id="invocation", run_id="run")): projection: Any = project_audit_result_batch( @@ -287,5 +294,54 @@ def test_given_lifecycle_validation_failure_when_projected_then_continues_and_pe assert build_projection_result().outcome == test_case.expected_outcome +@pytest.mark.parametrize( + "test_case", + [AuditExecutionCase("incremental publication", AuditOutcome.WARN)], + ids=lambda case: case.description, +) +def test_given_publication_scope_when_audits_complete_then_events_precede_batch_without_duplicates( + test_case: AuditExecutionCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + writer_calls: list[dict[str, Any]] = [] + monkeypatch.setattr( + projection_module, + "write_audit_result_records", + lambda **kwargs: writer_calls.append(kwargs), + ) + dispatcher: EventDispatcher = EventDispatcher() + events: list[LifecycleEvent] = [] + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + plan: PlanOutput = PlanOutput(audit_entries=(build_projection_entry(),)) + early: AuditExecutionResult = build_projection_result() + late: AuditExecutionResult = build_projection_result() + + with ( + identity_scope(ExecutionIdentity(invocation_id="invocation", run_id="run")), + dispatcher_scope(dispatcher), + audit_result_publication_scope(plan=plan, storage_schema="analytics"), + ): + publish_completed_audit_results((early,)) + publish_completed_audit_results((early,)) + events_before_batch: int = len(events) + projection: Any = project_audit_result_batch( + plan=plan, + results=(early, late), + adapter=writer_adapter(), + connection=object(), + storage_schema="analytics", + ) + + records: tuple[AuditResultRecord, ...] = writer_calls[0]["records"] + assert events_before_batch == 1 + assert len(events) == 2 + assert [event.payload["result_id"] for event in events] == [ + record.result_id for record in records + ] + assert records[0].result_id != records[1].result_id + assert projection.written_count == 2 + assert early.outcome == test_case.expected_outcome + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 57ad62bd63..900a6762ce 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -63,6 +63,7 @@ "SQBRTEST105", "SQBRTEST201", "SQBRTEST202", + "SQBRTEST203", "SQBRTEST301", ) SQL_CODES: tuple[str, ...] = tuple(f"SQBRSQL{number:03d}" for number in range(1, 40)) @@ -81,6 +82,7 @@ "SQBRTEST105", "SQBRTEST201", "SQBRTEST202", + "SQBRTEST203", "SQBRTEST301", ) NON_STRUCTURE_CODES: tuple[str, ...] = tuple( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index 0154526824..a170d9e678 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -119,6 +119,11 @@ def test_given_compiled_sql_facts_when_evaluating_native_then_exact_safe_rows_ar "assertion_target_model_names": ["orders"], "target_model_names": ["orders"], "tested_resources": [], + "authored_ctes": [], + "expected_ctes": [], + "assertion_ctes": [], + "has_macro_mocks": False, + "has_model_query_overrides": False, } assert len(request["sql_scenarios"]) == test_case.expected_scenario_count assert request["sql_scenarios"] == [