diff --git a/Cargo.lock b/Cargo.lock index 44860e0dd9..a4d63b81c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1317,6 +1317,7 @@ dependencies = [ "serde_json", "sha2", "sqlbuild-analysis", + "sqlbuild-attachments", "sqlbuild-core", "sqlbuild-sqltext", "sqlparser", diff --git a/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs b/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs index de75e3bad9..a8a487eb9a 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/formatting.rs @@ -1,6 +1,8 @@ //! `argparse.HelpFormatter`'s help, usage and version text in the Python 3.13 layout. -use crate::argv::_helpers::text_wrap::{normalize_whitespace, wrap}; +use sqlbuild_core::text::main::text_wrap::text_wrap; + +use crate::argv::_helpers::text_wrap::normalize_whitespace; use crate::argv::constants::{ INDENT_INCREMENT, MAX_HELP_POSITION, MAXIMUM_BLANK_RUN, MINIMUM_TEXT_WIDTH, USAGE_PREFIX, VERSION_PLACEHOLDER, @@ -130,7 +132,11 @@ fn help_text(help: &HelpText) -> Option<&str> { fn format_text(formatter: &Formatter, text: &str, indent: i64) -> String { let width: i64 = (formatter.width - indent).max(MINIMUM_TEXT_WIDTH); let padding: String = " ".repeat(indent as usize); - let lines: Vec = wrap(&normalize_whitespace(text), width as usize, &padding); + let lines: Vec = text_wrap( + &normalize_whitespace(text), + width as usize, + (&padding, &padding), + ); format!("{}\n\n", lines.join("\n")) } @@ -179,7 +185,7 @@ fn format_action(formatter: &Formatter, entry: &HelpEntry, indent: i64) -> Strin } }; let lines: Vec = help - .map(|help| wrap(&normalize_whitespace(help), help_width as usize, "")) + .map(|help| text_wrap(&normalize_whitespace(help), help_width as usize, ("", ""))) .unwrap_or_default(); if let Some((first, rest)) = lines.split_first() { text.push_str(&format!("{}{first}\n", " ".repeat(indent_first as usize))); diff --git a/crates/sqlbuild-cli/src/argv/_helpers/text_wrap.rs b/crates/sqlbuild-cli/src/argv/_helpers/text_wrap.rs index 552185d67d..daaf2df65e 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/text_wrap.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/text_wrap.rs @@ -1,9 +1,4 @@ -//! Python's `textwrap.wrap` with its default options, as `argparse` uses it. - -use sqlbuild_core::text::main::is_decimal_digit::is_decimal_digit; -use sqlbuild_core::text::main::python_isalnum::python_isalnum; - -use crate::argv::constants::MINIMUM_EM_DASH; +//! Whitespace normalization `argparse` applies before wrapping help text. /// `_whitespace_matcher.sub(' ', text).strip()`: ASCII whitespace runs become one space. pub(crate) fn normalize_whitespace(text: &str) -> String { @@ -22,165 +17,3 @@ pub(crate) fn normalize_whitespace(text: &str) -> String { } normalized } - -/// `textwrap.wrap(text, width, initial_indent=indent, subsequent_indent=indent)`. -pub(crate) fn wrap(text: &str, width: usize, indent: &str) -> Vec { - let characters: Vec = text.chars().collect(); - let mut chunks: Vec> = split_chunks(&characters); - chunks.reverse(); - let indent_length: usize = indent.chars().count(); - let mut lines: Vec = Vec::new(); - while !chunks.is_empty() { - let mut line: Vec> = Vec::new(); - let mut length: usize = 0; - let available: i64 = width as i64 - indent_length as i64; - if !lines.is_empty() && chunks.last().is_some_and(|chunk| is_blank(chunk)) { - chunks.pop(); - } - while let Some(chunk) = chunks.last() { - if (length + chunk.len()) as i64 <= available { - length += chunk.len(); - line.push(chunks.pop().unwrap_or_default()); - } else { - break; - } - } - if let Some(chunk) = chunks.pop_if(|chunk| chunk.len() as i64 > available) { - let (head, rest) = split_long_word(chunk, line.is_empty(), length, available); - line.extend(head); - chunks.extend(rest); - } - if line.last().is_some_and(|chunk| is_blank(chunk)) { - line.pop(); - } - if !line.is_empty() { - let joined: String = line.iter().flatten().collect(); - lines.push(format!("{indent}{joined}")); - } - } - lines -} - -/// `_handle_long_word`: the part of a too-long chunk the line takes, and what stays queued. -fn split_long_word( - mut chunk: Vec, - line_is_empty: bool, - length: usize, - width: i64, -) -> (Option>, Option>) { - let space_left: i64 = if width < 1 { 1 } else { width - length as i64 }; - if space_left <= 0 { - return if line_is_empty { - (Some(chunk), None) - } else { - (None, Some(chunk)) - }; - } - let space_left: usize = space_left as usize; - let mut end: usize = space_left.min(chunk.len()); - if chunk.len() > space_left - && let Some(hyphen) = chunk[..space_left] - .iter() - .rposition(|character| *character == '-') - && hyphen > 0 - && chunk[..hyphen].iter().any(|character| *character != '-') - { - end = hyphen + 1; - } - let rest: Vec = chunk.split_off(end); - (Some(chunk), Some(rest)) -} - -fn is_blank(chunk: &[char]) -> bool { - chunk.iter().all(|character| character.is_whitespace()) -} - -fn is_space(character: char) -> bool { - matches!(character, '\t' | '\n' | '\x0b' | '\x0c' | '\r' | ' ') -} - -fn is_word(character: char) -> bool { - character == '_' || python_isalnum(character) -} - -fn is_word_punct(character: char) -> bool { - is_word(character) || matches!(character, '!' | '"' | '\'' | '&' | '.' | ',' | '?') -} - -fn is_letter(character: char) -> bool { - is_word(character) && !is_decimal_digit(character) -} - -/// `TextWrapper.wordsep_re.split`: whitespace runs, em-dashes and possibly hyphenated words. -fn split_chunks(text: &[char]) -> Vec> { - let mut chunks: Vec> = Vec::new(); - let mut start: usize = 0; - while start < text.len() { - let end: usize = chunk_end(text, start); - chunks.push(text[start..end].to_vec()); - start = end; - } - chunks -} - -fn at(text: &[char], index: usize) -> Option { - text.get(index).copied() -} - -fn letter_at(text: &[char], index: Option) -> bool { - index - .and_then(|index| at(text, index)) - .is_some_and(is_letter) -} - -/// The end of a `-{2,}(?=\w)` match at `start`, if there is one. -fn em_dash_end(text: &[char], start: usize) -> Option { - let count: usize = text[start..] - .iter() - .take_while(|character| **character == '-') - .count(); - (count >= MINIMUM_EM_DASH && at(text, start + count).is_some_and(is_word)) - .then_some(start + count) -} - -fn chunk_end(text: &[char], start: usize) -> usize { - if is_space(text[start]) { - return start - + text[start..] - .iter() - .take_while(|character| is_space(**character)) - .count(); - } - if start > 0 - && is_word_punct(text[start - 1]) - && let Some(end) = em_dash_end(text, start) - { - return end; - } - let mut position: usize = start + 1; - loop { - if at(text, position) == Some('-') { - let two_letters: bool = letter_at(text, position.checked_sub(2)) - && letter_at(text, position.checked_sub(1)); - let letter_hyphen_letter: bool = letter_at(text, position.checked_sub(3)) - && at(text, position.wrapping_sub(2)) == Some('-') - && letter_at(text, position.checked_sub(1)); - let ahead: bool = letter_at(text, Some(position + 1)) - && (letter_at(text, Some(position + 2)) - || (at(text, position + 2) == Some('-') - && letter_at(text, Some(position + 3)))); - if (two_letters || letter_hyphen_letter) && ahead { - return position + 1; - } - } - match at(text, position) { - None => return position, - Some(character) if is_space(character) => return position, - Some(_) => {} - } - if is_word_punct(text[position - 1]) && em_dash_end(text, position).is_some() { - return position; - } - position += 1; - } -} diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 9396f84758..1094ff3510 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -20,8 +20,6 @@ pub(crate) const MAX_HELP_POSITION: i64 = 24; pub(crate) const MINIMUM_TEXT_WIDTH: i64 = 11; /// `_long_break_matcher` folds longer runs of newlines into this many. pub(crate) const MAXIMUM_BLANK_RUN: usize = 2; -/// `textwrap`'s em-dash is at least this many hyphens. -pub(crate) const MINIMUM_EM_DASH: usize = 2; /// `CliNamespace` attributes the post-parse rules read and write. pub(crate) const COMMAND_DEST: &str = "command"; diff --git a/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs b/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs index 4f936a0aef..ce6cb6fe10 100644 --- a/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs +++ b/crates/sqlbuild-cli/src/argv/tests/test_text_wrap.rs @@ -1,4 +1,6 @@ -use crate::argv::_helpers::text_wrap::{normalize_whitespace, wrap}; +use sqlbuild_core::text::main::text_wrap::text_wrap; + +use crate::argv::_helpers::text_wrap::normalize_whitespace; use crate::argv::tests::test_types::WrapTestCase; #[test] @@ -29,10 +31,10 @@ fn given_help_text_when_wrapping_then_lines_match_python_textwrap() { for test_case in test_cases { assert_eq!( - wrap( + text_wrap( &normalize_whitespace(test_case.text), test_case.width, - test_case.indent + (test_case.indent, test_case.indent) ), test_case.expected_lines, "{}", diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs index c7e36b2e4f..fb0073575c 100644 --- a/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/layout.rs @@ -1,6 +1,8 @@ //! The subset of Rich's console layout the diff summary uses: text, groups, tables and columns. -use crate::diff_report::_helpers::text_cells::{cell_len, wrap_line}; +use sqlbuild_core::text::main::cell_len::cell_len; + +use crate::diff_report::_helpers::text_cells::wrap_line; /// Text attributes and one standard foreground color, as Rich renders them. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] diff --git a/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs b/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs index 6c87a93a03..50fe798ca3 100644 --- a/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs +++ b/crates/sqlbuild-cli/src/diff_report/_helpers/text_cells.rs @@ -1,42 +1,7 @@ //! Terminal cell widths and Rich's word wrapping of one line of text. -/// The terminal cells one character occupies: zero for combining marks, two for wide characters. -pub(crate) fn char_cells(character: char) -> usize { - let code: u32 = u32::from(character); - let zero_width: bool = matches!( - code, - 0x0300..=0x036F | 0x200B..=0x200F | 0x20D0..=0x20FF | 0xFE00..=0xFE0F | 0xFE20..=0xFE2F - ); - let wide: bool = matches!( - code, - 0x1100..=0x115F - | 0x2E80..=0x303E - | 0x3041..=0x33FF - | 0x3400..=0x4DBF - | 0x4E00..=0x9FFF - | 0xA000..=0xA4CF - | 0xAC00..=0xD7A3 - | 0xF900..=0xFAFF - | 0xFE30..=0xFE4F - | 0xFF00..=0xFF60 - | 0xFFE0..=0xFFE6 - | 0x1F300..=0x1F64F - | 0x1F900..=0x1F9FF - | 0x20000..=0x3FFFD - ); - if zero_width { - 0 - } else if wide { - 2 - } else { - 1 - } -} - -/// The terminal cells a string occupies. -pub(crate) fn cell_len(text: &str) -> usize { - text.chars().map(char_cells).sum() -} +use sqlbuild_core::text::main::cell_len::cell_len; +use sqlbuild_core::text::main::char_cells::char_cells; /// The longest prefix of `text` that fits `width` cells, padded with spaces to exactly `width`. pub(crate) fn set_cell_size(text: &str, width: usize) -> String { diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index b46c6facef..e740334be6 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -108,7 +108,7 @@ pub(crate) const SCOPE_ROUTE: Route = routed( const DAG_ARGUMENTS: [Field; 4] = [PROJECT_DIR, NO_SQL_VALIDATION, JSON_OUTPUT, CLI_VARS]; pub(crate) const DAG_ROUTE: Route = routed( "run_dag", - ("sqlbuild.cli.commands.main.project._dag", "run_dag"), + ("sqlbuild._native", "run_dag_command"), &DAG_ARGUMENTS, CallShape { slot_positional: 0, @@ -190,9 +190,9 @@ const BUILD_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const BUILD_ROUTE: Route = routed( "run_build", - ("sqlbuild.cli.commands.main.execution._build", "run_build"), + ("sqlbuild._native", "run_build_command"), &BUILD_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const FRESHNESS_FIELDS: [Field; 12] = [ @@ -286,9 +286,9 @@ const AUDIT_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const AUDIT_ROUTE: Route = routed( "run_audit", - ("sqlbuild.cli.commands.main.execution._audit", "run_audit"), + ("sqlbuild._native", "run_audit_command"), &AUDIT_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const LOAD_FIELDS: [Field; 11] = [ @@ -453,7 +453,7 @@ const QUERY_ARGUMENTS: [Field; 6] = [ ]; pub(crate) const QUERY_ROUTE: Route = routed( "run_query", - ("sqlbuild.cli.commands.main.inspection._query", "run_query"), + ("sqlbuild._native", "run_query_command"), &QUERY_ARGUMENTS, CallShape { slot_positional: 0, @@ -480,7 +480,7 @@ const COST_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const COST_ROUTE: Route = routed( "run_cost", - ("sqlbuild.cli.commands.main.inspection._cost", "run_cost"), + ("sqlbuild._native", "run_cost_command"), &COST_ARGUMENTS, REQUEST_SHAPE, ); @@ -571,12 +571,9 @@ pub(crate) const CONTRACT_ROUTE: Route = Route { unavailable_code: Some("C470"), ..routed( "run_contract", - ( - "sqlbuild.cli.commands.main.inspection._contract", - "run_contract", - ), + ("sqlbuild._native", "run_contract_command"), &CONTRACT_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ) }; @@ -629,7 +626,7 @@ pub(crate) const COMPILE_ROUTE: Route = routed( const INIT_ARGUMENTS: [Field; 1] = [PROJECT_DIR]; pub(crate) const INIT_ROUTE: Route = routed( "run_init", - ("sqlbuild.cli.commands.main.workspace._init", "run_init"), + ("sqlbuild._native", "run_init_command"), &INIT_ARGUMENTS, REQUEST_SHAPE, ); @@ -679,10 +676,7 @@ const FORMAT_ARGUMENTS: [Field; 9] = [ ]; pub(crate) const FORMAT_ROUTE: Route = routed( "run_format", - ( - "sqlbuild.cli.commands.main.project._format", - "run_format_command", - ), + ("sqlbuild._native", "run_format_command"), &FORMAT_ARGUMENTS, CallShape { slot_positional: 1, @@ -786,12 +780,9 @@ const SCENARIO_TEST_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const SCENARIO_TEST_ROUTE: Route = routed( "run_scenario", - ( - "sqlbuild.cli.commands.main.execution._scenario", - "run_scenario", - ), + ("sqlbuild._native", "run_scenario_test_command"), &SCENARIO_TEST_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); const SCENARIO_CAPTURE_FIELDS: [Field; 8] = [ @@ -810,10 +801,7 @@ const SCENARIO_CAPTURE_ARGUMENTS: [Field; 1] = [request( )]; pub(crate) const SCENARIO_CAPTURE_ROUTE: Route = routed( "run_scenario_capture", - ( - "sqlbuild.cli.commands._helpers.scenario_capture.capture", - "run_scenario_capture", - ), + ("sqlbuild._native", "run_scenario_capture_command"), &SCENARIO_CAPTURE_ARGUMENTS, - REQUEST_SHAPE, + REQUEST_KEYWORD_SHAPE, ); diff --git a/crates/sqlbuild-core/src/audit_command/main/audit_exit_code.rs b/crates/sqlbuild-core/src/audit_command/main/audit_exit_code.rs new file mode 100644 index 0000000000..8050bb231d --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/main/audit_exit_code.rs @@ -0,0 +1,7 @@ +//! The exit code of `sqb audit`. + +/// 1 when any audit failed with error severity, else 0. +#[must_use] +pub fn audit_exit_code(failed: usize) -> i32 { + i32::from(failed > 0) +} diff --git a/crates/sqlbuild-core/src/audit_command/main/mod.rs b/crates/sqlbuild-core/src/audit_command/main/mod.rs new file mode 100644 index 0000000000..06b39a9eb2 --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/main/mod.rs @@ -0,0 +1,3 @@ +pub mod audit_exit_code; +pub mod render_audit_header; +pub mod render_audit_summary; diff --git a/crates/sqlbuild-core/src/audit_command/main/render_audit_header.rs b/crates/sqlbuild-core/src/audit_command/main/render_audit_header.rs new file mode 100644 index 0000000000..173660e115 --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/main/render_audit_header.rs @@ -0,0 +1,12 @@ +//! The header announcing the selected audits. + +use crate::cli_text::models::CliStyle; + +/// A blank line, `Audit ( selected, models)`, and a blank line. +#[must_use] +pub fn render_audit_header((audits, models): (usize, usize), style: CliStyle) -> String { + format!( + "\n{}\n\n", + style.success_strong(&format!("Audit ({audits} selected, {models} models)")) + ) +} diff --git a/crates/sqlbuild-core/src/audit_command/main/render_audit_summary.rs b/crates/sqlbuild-core/src/audit_command/main/render_audit_summary.rs new file mode 100644 index 0000000000..3cf437a738 --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/main/render_audit_summary.rs @@ -0,0 +1,31 @@ +//! The audit summary footer. + +use crate::audit_command::models::AuditCounts; +use crate::cli_text::main::format_summary_footer::format_summary_footer; +use crate::cli_text::models::{CliStyle, SummaryCount}; + +fn count(label: &str, value: usize) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value: i64::try_from(value).unwrap_or(i64::MAX), + } +} + +/// A blank line and the PASS, WARN, FAIL, INSUFFICIENT and TOTAL counts. +#[must_use] +pub fn render_audit_summary(counts: AuditCounts, style: CliStyle) -> String { + format!( + "\n{}\n", + format_summary_footer( + style, + &[ + count("PASS", counts.passed), + count("WARN", counts.warned), + count("FAIL", counts.failed), + count("INSUFFICIENT", counts.insufficient), + count("TOTAL", counts.total), + ], + None, + ) + ) +} diff --git a/crates/sqlbuild-core/src/audit_command/mod.rs b/crates/sqlbuild-core/src/audit_command/mod.rs new file mode 100644 index 0000000000..801aaecd8d --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/mod.rs @@ -0,0 +1,7 @@ +//! `sqb audit`: the selected-audits header, the summary footer and the exit code. + +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/audit_command/models.rs b/crates/sqlbuild-core/src/audit_command/models.rs new file mode 100644 index 0000000000..041fc73d69 --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/models.rs @@ -0,0 +1,11 @@ +//! Audit outcome counts as the summary reports them. + +/// How many audits ended in each outcome. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AuditCounts { + pub passed: usize, + pub warned: usize, + pub failed: usize, + pub insufficient: usize, + pub total: usize, +} diff --git a/crates/sqlbuild-core/src/audit_command/tests.rs b/crates/sqlbuild-core/src/audit_command/tests.rs new file mode 100644 index 0000000000..022331259d --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_audit_command.rs"] +mod audit_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/audit_command/tests/test_audit_command.rs b/crates/sqlbuild-core/src/audit_command/tests/test_audit_command.rs new file mode 100644 index 0000000000..72ba254e18 --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/tests/test_audit_command.rs @@ -0,0 +1,77 @@ +use crate::audit_command::main::audit_exit_code::audit_exit_code; +use crate::audit_command::main::render_audit_header::render_audit_header; +use crate::audit_command::main::render_audit_summary::render_audit_summary; +use crate::audit_command::models::AuditCounts; +use crate::audit_command::tests::test_types::{HeaderTestCase, SummaryTestCase}; +use crate::cli_text::models::CliStyle; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_selected_audits_when_rendering_header_then_counts_audits_and_models() { + let test_cases = [ + HeaderTestCase { + description: "audits across models", + counts: (28, 7), + expected_text: "\nAudit (28 selected, 7 models)\n\n", + }, + HeaderTestCase { + description: "nothing selected", + counts: (0, 0), + expected_text: "\nAudit (0 selected, 0 models)\n\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_audit_header(test_case.counts, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_outcomes_when_summarizing_then_counts_and_fails_only_on_errors() { + let test_cases = [ + SummaryTestCase { + description: "a warning keeps a zero exit code", + counts: AuditCounts { + passed: 27, + warned: 1, + total: 28, + ..AuditCounts::default() + }, + expected_text: "\nPASS=27 WARN=1 FAIL=0 INSUFFICIENT=0 TOTAL=28\n", + expected_exit_code: 0, + }, + SummaryTestCase { + description: "an error fails the run", + counts: AuditCounts { + passed: 26, + failed: 1, + insufficient: 1, + total: 28, + ..AuditCounts::default() + }, + expected_text: "\nPASS=26 WARN=0 FAIL=1 INSUFFICIENT=1 TOTAL=28\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_audit_summary(test_case.counts, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!( + audit_exit_code(test_case.counts.failed), + test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/audit_command/tests/test_types.rs b/crates/sqlbuild-core/src/audit_command/tests/test_types.rs new file mode 100644 index 0000000000..fb773bd620 --- /dev/null +++ b/crates/sqlbuild-core/src/audit_command/tests/test_types.rs @@ -0,0 +1,14 @@ +use crate::audit_command::models::AuditCounts; + +pub(super) struct HeaderTestCase { + pub(super) description: &'static str, + pub(super) counts: (usize, usize), + pub(super) expected_text: &'static str, +} + +pub(super) struct SummaryTestCase { + pub(super) description: &'static str, + pub(super) counts: AuditCounts, + pub(super) expected_text: &'static str, + pub(super) expected_exit_code: i32, +} diff --git a/crates/sqlbuild-core/src/build_command/_helpers/migrations.rs b/crates/sqlbuild-core/src/build_command/_helpers/migrations.rs new file mode 100644 index 0000000000..677e24e7b6 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/_helpers/migrations.rs @@ -0,0 +1,188 @@ +//! Model and column migration gates and the old-name refusal. + +use std::collections::BTreeSet; + +use crate::build_command::constants::{ + COLUMN_BLOCK_CODES, COLUMN_BLOCK_DEFAULT_CODE, COLUMN_MIGRATION_HELP, HIDDEN_ORIGIN_REMEDY, + MIGRATION_CODE, MIGRATION_CONFLICT, MIGRATION_CONFLICT_CODE, MIGRATION_CONFLICT_HELP, + MIGRATION_INCOMPATIBLE_CODE, MIGRATION_INCOMPATIBLE_HELP, MIGRATION_ORIGIN_MISSING, + MIGRATION_PLAN_HELP, OLD_NAME_CODES, OLD_NAME_DEFAULT_CODE, OLD_NAME_HELP, +}; +use crate::build_command::errors::BuildCommandError; +use crate::build_command::models::{ + ColumnMigrationFact, Confirmation, ModelMigrationFact, PlanWarningFact, +}; + +fn origins(entries: &[&ModelMigrationFact]) -> String { + entries + .iter() + .map(|entry| { + format!( + "'{}' (migrate_from {})", + entry.model_name, entry.origin_label + ) + }) + .collect::>() + .join(", ") +} + +/// Refuse to build while any planned migration conflicts or is incompatible. +pub(crate) fn model_migration_refusal(entries: &[ModelMigrationFact]) -> Option { + let blocked: Vec<&ModelMigrationFact> = + entries.iter().filter(|entry| entry.blocks_build).collect(); + if blocked.is_empty() { + return None; + } + let hidden: Vec<&ModelMigrationFact> = blocked + .iter() + .copied() + .filter(|entry| entry.origin_hidden) + .collect(); + if !hidden.is_empty() { + return Some(BuildCommandError::new( + Some(MIGRATION_CODE), + format!( + "model migration origin was built in this target but is not visible for {}; {HIDDEN_ORIGIN_REMEDY}", + origins(&hidden) + ), + Some(MIGRATION_PLAN_HELP), + )); + } + let missing: Vec<&ModelMigrationFact> = blocked + .iter() + .copied() + .filter(|entry| entry.decision == MIGRATION_ORIGIN_MISSING) + .collect(); + if !missing.is_empty() { + return Some(BuildCommandError::new( + Some(MIGRATION_CODE), + format!( + "model migration origin does not exist for {} and no recorded migration into it was found, and this target sets missing_migration_origin = deny; if the migration already happened elsewhere or is no longer needed, remove migrate_from from the model header", + origins(&missing) + ), + Some(MIGRATION_PLAN_HELP), + )); + } + let names: String = blocked + .iter() + .map(|entry| format!("'{}'", entry.model_name)) + .collect::>() + .join(", "); + if blocked + .iter() + .any(|entry| entry.decision == MIGRATION_CONFLICT) + { + return Some(BuildCommandError::new( + Some(MIGRATION_CONFLICT_CODE), + format!( + "model migration conflict for {names}: the destination already exists with its own build history and no recorded migration" + ), + Some(MIGRATION_CONFLICT_HELP), + )); + } + Some(BuildCommandError::new( + Some(MIGRATION_INCOMPATIBLE_CODE), + format!("model migration source is incompatible with {names}"), + Some(MIGRATION_INCOMPATIBLE_HELP), + )) +} + +/// Refuse to build while any declared column rename cannot be applied safely. +pub(crate) fn column_migration_refusal( + entries: &[ColumnMigrationFact], +) -> Option { + let blocked: Vec<&ColumnMigrationFact> = + entries.iter().filter(|entry| entry.blocks_build).collect(); + let first: &ColumnMigrationFact = blocked.first()?; + let code: &str = COLUMN_BLOCK_CODES + .iter() + .find(|(decision, _)| *decision == first.decision) + .map_or(COLUMN_BLOCK_DEFAULT_CODE, |(_, code)| code); + let listed: String = blocked + .iter() + .map(|entry| { + format!( + "'{}' ({} migrate_from {}: {})", + entry.model_name, + entry.destination_column, + entry.origin_column, + entry.decision_label + ) + }) + .collect::>() + .join(", "); + Some(BuildCommandError::new( + Some(code), + format!("column migrations are blocked for {listed}"), + Some(COLUMN_MIGRATION_HELP), + )) +} + +/// Confirm before building past declared migrations whose origin is missing. +pub(crate) fn missing_origin_confirmation( + (models, columns): (&[ModelMigrationFact], &[ColumnMigrationFact]), + help: &str, +) -> Option { + let mut names: BTreeSet<&str> = BTreeSet::new(); + for entry in models { + if entry.origin_missing && !entry.blocks_build && entry.confirm_missing_origin { + names.insert(&entry.model_name); + } + } + for entry in columns { + if entry.origin_missing && !entry.blocks_build && entry.confirm_missing_origin { + names.insert(&entry.model_name); + } + } + let first: &str = names.iter().next()?; + let listed: String = names + .iter() + .map(|name| format!("'{name}'")) + .collect::>() + .join(", "); + let action: String = format!("building {listed} without its missing migrate_from origin"); + let expected: String = if names.len() == 1 { + format!("build {first} without its migration") + } else { + format!("build {} models without their migrations", names.len()) + }; + Some(Confirmation { + warning: format!( + "The migrate_from origin of {listed} does not exist in this target; nothing will be migrated." + ), + expected, + refusal: BuildCommandError::new( + Some(MIGRATION_CODE), + format!("{action} requires confirmation"), + Some(help), + ), + cancellation: BuildCommandError::new( + Some(MIGRATION_CODE), + format!("{action} cancelled"), + None, + ), + }) +} + +/// Refuse to build over a compatibility view or while code reads an old name. +pub(crate) fn old_name_refusal(warnings: &[PlanWarningFact]) -> Option { + let errors: Vec<&PlanWarningFact> = warnings + .iter() + .filter(|warning| { + warning + .code + .as_deref() + .is_some_and(|code| OLD_NAME_CODES.contains(&code)) + }) + .collect(); + let first: &PlanWarningFact = errors.first()?; + Some(BuildCommandError::new( + Some(first.code.as_deref().unwrap_or(OLD_NAME_DEFAULT_CODE)), + errors + .iter() + .map(|warning| warning.message.as_str()) + .collect::>() + .join("; "), + Some(OLD_NAME_HELP), + )) +} diff --git a/crates/sqlbuild-core/src/build_command/_helpers/mod.rs b/crates/sqlbuild-core/src/build_command/_helpers/mod.rs new file mode 100644 index 0000000000..3c92a4b9dd --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod migrations; +pub(crate) mod storage; +pub(crate) mod text; diff --git a/crates/sqlbuild-core/src/build_command/_helpers/storage.rs b/crates/sqlbuild-core/src/build_command/_helpers/storage.rs new file mode 100644 index 0000000000..5ba23ba62b --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/_helpers/storage.rs @@ -0,0 +1,220 @@ +//! Snapshot full-refresh, table-type downgrade and retention decrease gates. + +use crate::build_command::constants::{ + POLICY_ALLOW, POLICY_DENY, POLICY_REQUIRE_CONFIRMATION, RETENTION_DENIED_HELP, + SNAPSHOT_CANCELLED_CODE, SNAPSHOT_CONFIRMATION_CODE, SNAPSHOT_DENIED_CODE, + SNAPSHOT_DENIED_HELP, +}; +use crate::build_command::errors::BuildCommandError; +use crate::build_command::models::{ + Confirmation, PolicyStep, RetentionFact, SnapshotPolicies, SnapshotRefreshFact, TableTypeFact, +}; + +fn strictness(policy: &str) -> Result { + match policy { + POLICY_ALLOW => Ok(0), + POLICY_REQUIRE_CONFIRMATION => Ok(1), + POLICY_DENY => Ok(2), + _ => Err(BuildCommandError::new( + None, + format!("'{policy}' is not a valid SnapshotFullRefreshPolicy"), + None, + )), + } +} + +fn effective_snapshot_policy<'a>( + entry: &'a SnapshotRefreshFact, + policies: &'a SnapshotPolicies, +) -> Result<&'a str, BuildCommandError> { + let project: &str = if entry.historical { + &policies.historical + } else { + &policies.current_state + }; + let project_strictness: u8 = strictness(project)?; + let Some(model) = entry.model_policy.as_deref() else { + return Ok(project); + }; + if strictness(model)? > project_strictness { + return Ok(model); + } + Ok(project) +} + +fn quoted(names: &[&str]) -> String { + names + .iter() + .map(|name| format!("'{name}'")) + .collect::>() + .join(", ") +} + +fn confirmation( + (warning, expected): (String, String), + (subject, help): (&str, &str), + codes: (Option<&str>, Option<&str>), +) -> Confirmation { + Confirmation { + warning, + expected, + refusal: BuildCommandError::new( + codes.0, + format!("{subject} requires confirmation"), + Some(help), + ), + cancellation: BuildCommandError::new(codes.1, format!("{subject} cancelled"), None), + } +} + +/// Fail or confirm before executing protected snapshot full refreshes. +pub(crate) fn snapshot_steps( + entries: &[SnapshotRefreshFact], + policies: &SnapshotPolicies, + (allowed, help): (bool, &str), +) -> Result, BuildCommandError> { + let mut denied: Vec<&str> = Vec::new(); + let mut confirm: Vec<&str> = Vec::new(); + for entry in entries { + match effective_snapshot_policy(entry, policies)? { + POLICY_DENY => denied.push(&entry.name), + POLICY_REQUIRE_CONFIRMATION => confirm.push(&entry.name), + _ => {} + } + } + if !denied.is_empty() { + return Ok(Some(PolicyStep::Refuse(BuildCommandError::new( + Some(SNAPSHOT_DENIED_CODE), + format!( + "full refresh is denied for snapshot model {} by snapshot_full_refresh policy", + quoted(&denied) + ), + Some(SNAPSHOT_DENIED_HELP), + )))); + } + if confirm.is_empty() || allowed { + return Ok(None); + } + let expected: String = if confirm.len() == 1 { + format!("discard snapshot history for {}", confirm[0]) + } else { + format!("discard snapshot history for {} models", confirm.len()) + }; + Ok(Some(PolicyStep::Confirm(confirmation( + ( + format!( + "Full refresh of snapshot model {} may permanently discard unrecoverable history.", + quoted(&confirm) + ), + expected, + ), + ("snapshot full refresh", help), + ( + Some(SNAPSHOT_CONFIRMATION_CODE), + Some(SNAPSHOT_CANCELLED_CODE), + ), + )))) +} + +/// Fail or confirm before executing permanent-to-transient conversions. +pub(crate) fn table_type_step( + entries: &[TableTypeFact], + (allowed, help): (bool, &str), +) -> Option { + let denied: Vec<&str> = entries + .iter() + .filter(|entry| entry.policy == POLICY_DENY) + .map(|entry| entry.model_name.as_str()) + .collect(); + if !denied.is_empty() { + return Some(PolicyStep::Refuse(BuildCommandError::new( + None, + format!( + "table-type downgrade is denied for {} by table_type_downgrade policy", + quoted(&denied) + ), + None, + ))); + } + let confirm: Vec<&str> = entries + .iter() + .filter(|entry| entry.policy == POLICY_REQUIRE_CONFIRMATION) + .map(|entry| entry.model_name.as_str()) + .collect(); + if confirm.is_empty() || allowed { + return None; + } + let expected: String = if confirm.len() == 1 { + format!("downgrade table type for {}", confirm[0]) + } else { + format!("downgrade table type for {} models", confirm.len()) + }; + Some(PolicyStep::Confirm(confirmation( + ( + format!( + "Downgrading {} to transient may discard up to 90 days of time-travel history.", + quoted(&confirm) + ), + expected, + ), + ("table-type downgrade", help), + (None, None), + ))) +} + +fn retention_names<'a>(entries: &[&'a RetentionFact]) -> Vec<&'a str> { + let mut names: Vec<&str> = Vec::new(); + for entry in entries { + for name in &entry.model_names { + if !names.contains(&name.as_str()) { + names.push(name); + } + } + } + names +} + +/// Fail or confirm before lowering live time-travel retention. +pub(crate) fn retention_step( + entries: &[RetentionFact], + (allowed, help): (bool, &str), +) -> Option { + let denied: Vec<&RetentionFact> = entries + .iter() + .filter(|entry| entry.policy == POLICY_DENY) + .collect(); + if !denied.is_empty() { + return Some(PolicyStep::Refuse(BuildCommandError::new( + None, + format!( + "time travel retention decrease is denied for {}", + quoted(&retention_names(&denied)) + ), + Some(RETENTION_DENIED_HELP), + ))); + } + let confirm: Vec<&RetentionFact> = entries + .iter() + .filter(|entry| entry.policy == POLICY_REQUIRE_CONFIRMATION) + .collect(); + if confirm.is_empty() || allowed { + return None; + } + let names: Vec<&str> = retention_names(&confirm); + let expected: String = if names.len() == 1 { + format!("decrease retention for {}", names[0]) + } else { + format!("decrease retention for {} models", names.len()) + }; + Some(PolicyStep::Confirm(confirmation( + ( + format!( + "Decreasing time travel retention for {} permanently discards history older than the new retention.", + quoted(&names) + ), + expected, + ), + ("time travel retention decrease", help), + (None, None), + ))) +} diff --git a/crates/sqlbuild-core/src/build_command/_helpers/text.rs b/crates/sqlbuild-core/src/build_command/_helpers/text.rs new file mode 100644 index 0000000000..77e48b0f61 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/_helpers/text.rs @@ -0,0 +1,39 @@ +//! Safe display of resolved context values. + +use crate::build_command::constants::{ + C0_CONTROL_CODE_LIMIT, C1_CONTROL_CODE_LIMIT, C1_CONTROL_CODE_START, NOT_SET, +}; + +/// Escape C0 and C1 control characters as `\xNN`. +pub(crate) fn sanitize(value: &str) -> String { + let mut text: String = String::with_capacity(value.len()); + for character in value.chars() { + let code: u32 = u32::from(character); + if code < C0_CONTROL_CODE_LIMIT + || (C1_CONTROL_CODE_START..C1_CONTROL_CODE_LIMIT).contains(&code) + { + text.push_str(&format!("\\x{code:02x}")); + } else { + text.push(character); + } + } + text +} + +/// The sanitized value, or `not set`. +pub(crate) fn display_value(value: Option<&str>) -> String { + value.map_or_else(|| NOT_SET.to_owned(), sanitize) +} + +/// A count with `,` thousands separators. +pub(crate) fn grouped(value: usize) -> String { + let digits: String = value.to_string(); + let mut text: String = String::with_capacity(digits.len() + digits.len() / 3); + for (index, digit) in digits.chars().enumerate() { + if index > 0 && (digits.len() - index).is_multiple_of(3) { + text.push(','); + } + text.push(digit); + } + text +} diff --git a/crates/sqlbuild-core/src/build_command/constants.rs b/crates/sqlbuild-core/src/build_command/constants.rs new file mode 100644 index 0000000000..e4672a7a50 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/constants.rs @@ -0,0 +1,55 @@ +//! Build command codes, policy values and fixed text. + +pub const POLICY_ALLOW: &str = "allow"; +pub const POLICY_REQUIRE_CONFIRMATION: &str = "require_confirmation"; +pub const POLICY_DENY: &str = "deny"; + +pub(crate) const MIGRATION_ORIGIN_MISSING: &str = "origin_missing"; +pub(crate) const MIGRATION_CONFLICT: &str = "conflict"; +pub(crate) const MIGRATION_CODE: &str = "M102"; +pub(crate) const MIGRATION_CONFLICT_CODE: &str = "M103"; +pub(crate) const MIGRATION_INCOMPATIBLE_CODE: &str = "M104"; +pub(crate) const OLD_NAME_CODES: [&str; 2] = ["M114", "P008"]; +pub(crate) const OLD_NAME_DEFAULT_CODE: &str = "M114"; +pub(crate) const COLUMN_BLOCK_CODES: [(&str, &str); 4] = [ + ("source_missing", "M109"), + ("conflict", "M110"), + ("still_produced", "M111"), + ("unsupported", "M112"), +]; +pub(crate) const COLUMN_BLOCK_DEFAULT_CODE: &str = "M109"; +pub(crate) const SNAPSHOT_DENIED_CODE: &str = "C238"; +pub(crate) const SNAPSHOT_CONFIRMATION_CODE: &str = "C239"; +pub(crate) const SNAPSHOT_CANCELLED_CODE: &str = "C240"; +pub(crate) const MODEL_LIMIT_CODE: &str = "C413"; +pub(crate) const DURATION_UNSUPPORTED_CODE: &str = "C415"; +pub(crate) const DURATION_EXCEEDED_CODE: &str = "C416"; +pub(crate) const DEFAULT_TARGET_LABEL: &str = "default"; + +pub(crate) const HIDDEN_ORIGIN_REMEDY: &str = "the connection may lack permission to see it, or it was dropped outside SQLBuild; restore access to it, or remove migrate_from to build the model fresh deliberately (missing_migration_origin does not apply)"; +pub(crate) const MIGRATION_PLAN_HELP: &str = "Run sqb plan to see every model migration decision."; +pub(crate) const MIGRATION_CONFLICT_HELP: &str = "Set migrate_force true on the model to replace the destination; the replaced table remains recoverable through warehouse time travel."; +pub(crate) const MIGRATION_INCOMPATIBLE_HELP: &str = + "Run sqb plan to see the blocking schema differences."; +pub(crate) const COLUMN_MIGRATION_HELP: &str = + "Run sqb plan to see why each column rename cannot be applied."; +pub(crate) const OLD_NAME_HELP: &str = + "Run sqb plan to see every compatibility view and old-name reference."; +pub(crate) const SNAPSHOT_DENIED_HELP: &str = "Set a less strict project/model snapshot_full_refresh policy only if the snapshot history is recoverable."; +pub(crate) const RETENTION_DENIED_HELP: &str = "Keep the current retention, or set the target's time_travel_retention_decrease to 'require_confirmation' or 'allow'."; + +/// Label width of the verbose phase timing rows. +pub(crate) const PHASE_LABEL_WIDTH: usize = 24; +/// Label width of the verbose execution context rows. +pub(crate) const CONTEXT_LABEL_WIDTH: usize = 12; +pub(crate) const NOT_SET: &str = "not set"; +pub(crate) const C0_CONTROL_CODE_LIMIT: u32 = 0x20; +pub(crate) const C1_CONTROL_CODE_START: u32 = 0x7f; +pub(crate) const C1_CONTROL_CODE_LIMIT: u32 = 0xa0; + +pub(crate) const PYTHON_NODE_SUCCESS: &str = "success"; +pub(crate) const PYTHON_NODE_SKIPPED: &str = "skipped"; +pub(crate) const PYTHON_NODE_FAILED: &str = "failed"; +pub(crate) const PYTHON_STATUS_OK: &str = "OK"; +pub(crate) const PYTHON_STATUS_SKIP: &str = "SKIP"; +pub(crate) const PYTHON_STATUS_FAIL: &str = "FAIL"; diff --git a/crates/sqlbuild-core/src/build_command/errors.rs b/crates/sqlbuild-core/src/build_command/errors.rs new file mode 100644 index 0000000000..3068f7b076 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/errors.rs @@ -0,0 +1,21 @@ +//! Build command user errors. + +/// A user error the CLI reports with its optional code and help. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct BuildCommandError { + pub code: Option, + pub message: String, + pub help: Option, +} + +impl BuildCommandError { + /// An error with an optional code and help. + #[must_use] + pub fn new(code: Option<&str>, message: String, help: Option<&str>) -> Self { + Self { + code: code.map(str::to_owned), + message, + help: help.map(str::to_owned), + } + } +} diff --git a/crates/sqlbuild-core/src/build_command/main/build_exit_code.rs b/crates/sqlbuild-core/src/build_command/main/build_exit_code.rs new file mode 100644 index 0000000000..70cc560e14 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/build_exit_code.rs @@ -0,0 +1,7 @@ +//! The build exit code. + +/// `0` only when the build succeeded and no Python node or check failed. +#[must_use] +pub fn build_exit_code(succeeded: bool, (python_failed, checks_failed): (bool, bool)) -> i32 { + i32::from(!succeeded || python_failed || checks_failed) +} diff --git a/crates/sqlbuild-core/src/build_command/main/duration_limit_error.rs b/crates/sqlbuild-core/src/build_command/main/duration_limit_error.rs new file mode 100644 index 0000000000..6cfffa65a1 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/duration_limit_error.rs @@ -0,0 +1,35 @@ +//! The target duration limit against the adapter. + +use crate::build_command::constants::{ + DEFAULT_TARGET_LABEL, DURATION_EXCEEDED_CODE, DURATION_UNSUPPORTED_CODE, +}; +use crate::build_command::errors::BuildCommandError; +use crate::build_command::models::DurationLimit; + +/// The refusal when the adapter cannot enforce the target's `max_duration`. +#[must_use] +pub fn duration_limit_error(limit: &DurationLimit) -> Option { + let target: &str = limit.target_name.as_deref().unwrap_or(DEFAULT_TARGET_LABEL); + let adapter: &str = &limit.adapter_name; + let Some(maximum) = limit.adapter_limit_seconds else { + return limit.configures_max_duration.then(|| { + BuildCommandError::new( + Some(DURATION_UNSUPPORTED_CODE), + format!( + "Target '{target}' configures max_duration, but adapter '{adapter}' cannot cancel active statements safely" + ), + limit.remediation.as_deref(), + ) + }); + }; + let seconds: i64 = limit.max_duration_seconds?; + (seconds > maximum).then(|| { + BuildCommandError::new( + Some(DURATION_EXCEEDED_CODE), + format!( + "Target '{target}' max_duration exceeds the largest duration supported by adapter '{adapter}' ({maximum} seconds)" + ), + limit.remediation.as_deref(), + ) + }) +} diff --git a/crates/sqlbuild-core/src/build_command/main/mod.rs b/crates/sqlbuild-core/src/build_command/main/mod.rs new file mode 100644 index 0000000000..9256414af3 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/mod.rs @@ -0,0 +1,9 @@ +pub mod build_exit_code; +pub mod duration_limit_error; +pub mod model_limit_error; +pub mod plan_safety_steps; +pub mod python_node_failed; +pub mod render_confirmation_prompt; +pub mod render_phase_timings; +pub mod render_python_node_rows; +pub mod render_run_context; diff --git a/crates/sqlbuild-core/src/build_command/main/model_limit_error.rs b/crates/sqlbuild-core/src/build_command/main/model_limit_error.rs new file mode 100644 index 0000000000..dd54e37cb8 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/model_limit_error.rs @@ -0,0 +1,26 @@ +//! The target model-count limit. + +use crate::build_command::constants::{DEFAULT_TARGET_LABEL, MODEL_LIMIT_CODE}; +use crate::build_command::errors::BuildCommandError; +use crate::build_command::models::ModelLimit; + +/// The refusal when `model_count` exceeds the target's `max_models`. +#[must_use] +pub fn model_limit_error( + model_count: usize, + limit: &ModelLimit, + refusal_note: &str, +) -> Option { + let maximum: i64 = limit.max_models?; + if i64::try_from(model_count).is_ok_and(|count| count <= maximum) { + return None; + } + let target: &str = limit.target_name.as_deref().unwrap_or(DEFAULT_TARGET_LABEL); + Some(BuildCommandError::new( + Some(MODEL_LIMIT_CODE), + format!( + "Build plan exceeds the configured model limit\n\nTarget: {target}\nSelected models: {model_count}\nMaximum models: {maximum}\nConfiguration: targets.{target}.execution_limits.max_models\n\n{refusal_note}" + ), + limit.remediation.as_deref(), + )) +} diff --git a/crates/sqlbuild-core/src/build_command/main/plan_safety_steps.rs b/crates/sqlbuild-core/src/build_command/main/plan_safety_steps.rs new file mode 100644 index 0000000000..ccf4f53775 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/plan_safety_steps.rs @@ -0,0 +1,79 @@ +//! The plan safety gates in the order they apply. + +use crate::build_command::_helpers::migrations::{ + column_migration_refusal, missing_origin_confirmation, model_migration_refusal, + old_name_refusal, +}; +use crate::build_command::_helpers::storage::{retention_step, snapshot_steps, table_type_step}; +use crate::build_command::errors::BuildCommandError; +use crate::build_command::main::model_limit_error::model_limit_error; +use crate::build_command::models::{ + ModelLimit, PlanSafetyFacts, PlanSafetyGates, PolicyStep, SnapshotPolicies, +}; + +/// Migration, old-name, model-limit, snapshot, table-type and retention gates; a refusal ends the list. +pub fn plan_safety_steps( + facts: &PlanSafetyFacts, + gates: &PlanSafetyGates, + (limit, snapshot_policies): (&ModelLimit, &SnapshotPolicies), +) -> Result, BuildCommandError> { + let mut steps: Vec = Vec::new(); + let refusal: Option = model_migration_refusal(&facts.model_migrations) + .or_else(|| column_migration_refusal(&facts.column_migrations)); + if let Some(error) = refusal { + steps.push(PolicyStep::Refuse(error)); + return Ok(steps); + } + if !gates.allow_missing_migration_origin + && let Some(confirmation) = missing_origin_confirmation( + (&facts.model_migrations, &facts.column_migrations), + &gates.missing_origin_help, + ) + { + steps.push(PolicyStep::Confirm(confirmation)); + } + let refusal: Option = + old_name_refusal(&facts.error_warnings).or_else(|| { + model_limit_error( + facts.executable_model_count, + limit, + &gates.execution_limit_note, + ) + }); + if let Some(error) = refusal { + steps.push(PolicyStep::Refuse(error)); + return Ok(steps); + } + let storage: [Option; 3] = [ + snapshot_steps( + &facts.snapshot_refreshes, + snapshot_policies, + ( + gates.allow_snapshot_full_refresh, + &gates.snapshot_full_refresh_help, + ), + )?, + table_type_step( + &facts.table_type_downgrades, + ( + gates.allow_table_type_downgrade, + &gates.table_type_downgrade_help, + ), + ), + retention_step( + &facts.retention_decreases, + ( + gates.allow_retention_decrease, + &gates.retention_decrease_help, + ), + ), + ]; + for step in storage.into_iter().flatten() { + let refused: bool = matches!(step, PolicyStep::Refuse(_)); + steps.push(step); + if refused { + break; + } + } + Ok(steps) +} diff --git a/crates/sqlbuild-core/src/build_command/main/python_node_failed.rs b/crates/sqlbuild-core/src/build_command/main/python_node_failed.rs new file mode 100644 index 0000000000..7f1a1d9ef1 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/python_node_failed.rs @@ -0,0 +1,9 @@ +//! Whether a Python-node status is a failure. + +use crate::build_command::constants::PYTHON_NODE_FAILED; + +/// Only the `failed` status fails the build. +#[must_use] +pub fn python_node_failed(status: &str) -> bool { + status == PYTHON_NODE_FAILED +} diff --git a/crates/sqlbuild-core/src/build_command/main/render_confirmation_prompt.rs b/crates/sqlbuild-core/src/build_command/main/render_confirmation_prompt.rs new file mode 100644 index 0000000000..37e45921ac --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/render_confirmation_prompt.rs @@ -0,0 +1,12 @@ +//! The typed confirmation prompt. + +use crate::build_command::models::Confirmation; + +/// The warning, a blank line, then the prompt for the expected text. +#[must_use] +pub fn render_confirmation_prompt(confirmation: &Confirmation) -> String { + format!( + "{}\n\nType `{}` to continue: ", + confirmation.warning, confirmation.expected + ) +} diff --git a/crates/sqlbuild-core/src/build_command/main/render_phase_timings.rs b/crates/sqlbuild-core/src/build_command/main/render_phase_timings.rs new file mode 100644 index 0000000000..e77cd6e633 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/render_phase_timings.rs @@ -0,0 +1,33 @@ +//! Verbose build phase timings. + +use crate::build_command::constants::PHASE_LABEL_WIDTH; +use crate::build_command::models::PhaseTimings; +use crate::cli_text::models::{CliDocument, CliStyle}; + +/// A blank line, the `Phase timings` header and one row per measured phase. +#[must_use] +pub fn render_phase_timings(timings: &PhaseTimings, style: CliStyle) -> String { + let phases: [(&str, Option); 7] = [ + ("compile", timings.compile_seconds), + ("planning", timings.planning_seconds), + ( + "connection preparation", + timings.connection_preparation_seconds, + ), + ("schema preparation", timings.schema_preparation_seconds), + ("execution", timings.execution_seconds), + ("cost collection", timings.cost_collection_seconds), + ("total", timings.total_seconds), + ]; + let mut rows: Vec<(String, String)> = Vec::new(); + for (label, seconds) in phases { + if let Some(seconds) = seconds { + rows.push((label.to_owned(), format!("{seconds:.2}s"))); + } + } + let mut document: CliDocument = CliDocument::new(style); + document.lines.push(String::new()); + document.header("Phase timings", None); + document.fields(&rows, Some(PHASE_LABEL_WIDTH)); + document.render(true) +} diff --git a/crates/sqlbuild-core/src/build_command/main/render_python_node_rows.rs b/crates/sqlbuild-core/src/build_command/main/render_python_node_rows.rs new file mode 100644 index 0000000000..9f63a1ba17 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/render_python_node_rows.rs @@ -0,0 +1,39 @@ +//! Task and asset result rows. + +use crate::build_command::constants::{ + PYTHON_NODE_SKIPPED, PYTHON_NODE_SUCCESS, PYTHON_STATUS_FAIL, PYTHON_STATUS_OK, + PYTHON_STATUS_SKIP, +}; +use crate::build_command::models::PythonNodeRow; +use crate::cli_text::models::CliStyle; + +/// ` python `, then the error or skip reason. +#[must_use] +pub fn render_python_node_rows(rows: &[PythonNodeRow], style: CliStyle) -> String { + let mut text: String = String::new(); + for row in rows { + let status: &str = match row.status.as_str() { + PYTHON_NODE_SUCCESS => PYTHON_STATUS_OK, + PYTHON_NODE_SKIPPED => PYTHON_STATUS_SKIP, + _ => PYTHON_STATUS_FAIL, + }; + text.push_str(&format!( + " {:<10}{:<10}{:<50} {}", + "python", + row.kind, + row.node_name, + style.status(status) + )); + if let Some(message) = row + .error_message + .as_deref() + .filter(|value| !value.is_empty()) + { + text.push_str(&format!(" {message}")); + } else if let Some(reason) = row.skip_reason.as_deref().filter(|value| !value.is_empty()) { + text.push_str(&format!(" {reason}")); + } + text.push('\n'); + } + text +} diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs new file mode 100644 index 0000000000..6c902db1f2 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -0,0 +1,76 @@ +//! The verbose build execution context. + +use crate::build_command::_helpers::text::{display_value, grouped, sanitize}; +use crate::build_command::constants::{CONTEXT_LABEL_WIDTH, NOT_SET}; +use crate::build_command::models::RunContextFacts; +use crate::cli_text::models::{CliDocument, CliStyle}; + +fn date_vars(facts: &RunContextFacts) -> String { + match &facts.date_vars { + (None, None) => NOT_SET.to_owned(), + (None, Some(end)) => format!("end_date={}", sanitize(end)), + (Some(start), None) => format!("start_date={}", sanitize(start)), + (Some(start), Some(end)) => format!("{} to {}", sanitize(start), sanitize(end)), + } +} + +/// The resolved placement, mode and selection, then the selector files. +#[must_use] +pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { + let warehouse: String = facts.warehouse.as_ref().map_or_else( + || NOT_SET.to_owned(), + |(warehouse, source)| format!("{} ({source})", sanitize(warehouse)), + ); + let rows: Vec<(String, String)> = [ + ("command", sanitize(&facts.command)), + ("run_id", display_value(facts.run_id.as_deref())), + ("target", display_value(facts.target.as_deref())), + ("database", display_value(facts.database.as_deref())), + ("schema", display_value(facts.schema.as_deref())), + ("warehouse", warehouse), + ( + "concurrency", + format!("{} configured limit", facts.concurrency), + ), + ("full_refresh", facts.full_refresh.to_string()), + ( + "selected", + format!( + "{} of {} build resources", + grouped(facts.selected_count), + grouped(facts.total_count) + ), + ), + ("date vars", date_vars(facts)), + ] + .into_iter() + .map(|(label, value)| (label.to_owned(), value)) + .collect(); + let mut document: CliDocument = CliDocument::new(style); + document.header("Execution", None); + document.fields(&rows, Some(CONTEXT_LABEL_WIDTH)); + if !facts.selector_files.is_empty() { + document.lines.push(String::new()); + document.header("Selection files", None); + for file in &facts.selector_files { + let noun: &str = if file.selector_count == 1 { + "selector" + } else { + "selectors" + }; + document.fields( + &[( + "selector_file".to_owned(), + format!( + "{} ({} {noun})", + sanitize(&file.path), + grouped(file.selector_count) + ), + )], + Some(CONTEXT_LABEL_WIDTH), + ); + } + } + document.lines.push(String::new()); + document.render(true) +} diff --git a/crates/sqlbuild-core/src/build_command/mod.rs b/crates/sqlbuild-core/src/build_command/mod.rs new file mode 100644 index 0000000000..ba60d65dab --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/mod.rs @@ -0,0 +1,10 @@ +//! `sqb build`: plan safety gates, execution limits, run context, phase timings, Python-node rows and the exit code. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs new file mode 100644 index 0000000000..952e90d832 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -0,0 +1,198 @@ +//! Plain plan facts, gates and outputs of the build command. + +use crate::build_command::errors::BuildCommandError; + +/// One planned model migration, as the safety gates read it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelMigrationFact { + pub model_name: String, + /// The origin's qualified name, or its name. + pub origin_label: String, + pub decision: String, + pub blocks_build: bool, + pub origin_hidden: bool, + pub origin_missing: bool, + /// The target's missing-origin policy asks for confirmation. + pub confirm_missing_origin: bool, +} + +/// One planned column rename, as the safety gates read it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ColumnMigrationFact { + pub model_name: String, + pub destination_column: String, + pub origin_column: String, + pub decision: String, + pub decision_label: String, + pub blocks_build: bool, + pub origin_missing: bool, + pub confirm_missing_origin: bool, +} + +/// An error-severity plan warning. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PlanWarningFact { + pub code: Option, + pub message: String, +} + +/// A snapshot model planned for a full refresh. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SnapshotRefreshFact { + pub name: String, + /// The snapshot records an `observed_at` column. + pub historical: bool, + pub model_policy: Option, +} + +/// A planned permanent-to-transient table-type downgrade. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TableTypeFact { + pub model_name: String, + pub policy: String, +} + +/// A planned time-travel retention decrease. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RetentionFact { + pub model_names: Vec, + pub policy: String, +} + +/// Everything the plan safety gates read from one plan. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PlanSafetyFacts { + pub model_migrations: Vec, + pub column_migrations: Vec, + pub error_warnings: Vec, + pub executable_model_count: usize, + pub snapshot_refreshes: Vec, + pub table_type_downgrades: Vec, + pub retention_decreases: Vec, +} + +/// Pre-confirmed gates and the guidance shown when one needs a terminal. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PlanSafetyGates { + pub allow_missing_migration_origin: bool, + pub allow_snapshot_full_refresh: bool, + pub allow_table_type_downgrade: bool, + pub allow_retention_decrease: bool, + pub missing_origin_help: String, + pub snapshot_full_refresh_help: String, + pub table_type_downgrade_help: String, + pub retention_decrease_help: String, + pub execution_limit_note: String, +} + +/// Project snapshot full-refresh policies. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SnapshotPolicies { + pub historical: String, + pub current_state: String, +} + +/// A target's model-count limit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelLimit { + pub target_name: Option, + pub max_models: Option, + pub remediation: Option, +} + +/// A target's duration limit against what its adapter can enforce. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DurationLimit { + pub target_name: Option, + pub adapter_name: String, + pub configures_max_duration: bool, + pub max_duration_seconds: Option, + pub adapter_limit_seconds: Option, + pub remediation: Option, +} + +/// A typed confirmation one gate asks for. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Confirmation { + pub warning: String, + pub expected: String, + /// Raised when the run cannot ask. + pub refusal: BuildCommandError, + /// Raised when the answer does not match. + pub cancellation: BuildCommandError, +} + +/// One plan safety gate, in the order the gates apply. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum PolicyStep { + Refuse(BuildCommandError), + Confirm(Confirmation), +} + +impl PolicyStep { + /// The refusal this step raises, if it is one. + #[must_use] + pub fn refusal(&self) -> Option<&BuildCommandError> { + match self { + Self::Refuse(error) => Some(error), + Self::Confirm(_) => None, + } + } + + /// The confirmation this step asks for, if it is one. + #[must_use] + pub fn confirmation(&self) -> Option<&Confirmation> { + match self { + Self::Refuse(_) => None, + Self::Confirm(confirmation) => Some(confirmation), + } + } +} + +/// Final monotonic phase durations for verbose build output. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PhaseTimings { + pub compile_seconds: Option, + pub planning_seconds: Option, + pub connection_preparation_seconds: Option, + pub schema_preparation_seconds: Option, + pub execution_seconds: Option, + pub cost_collection_seconds: Option, + pub total_seconds: Option, +} + +/// One selector file the selection read. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SelectorFileFact { + pub path: String, + pub selector_count: usize, +} + +/// Resolved, non-secret build placement, mode and selection context. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RunContextFacts { + pub command: String, + pub run_id: Option, + pub target: Option, + pub database: Option, + pub schema: Option, + /// The warehouse and where it came from. + pub warehouse: Option<(String, String)>, + pub concurrency: i64, + pub full_refresh: bool, + pub selected_count: usize, + pub total_count: usize, + /// Valid ISO `start_date` and `end_date` variables. + pub date_vars: (Option, Option), + pub selector_files: Vec, +} + +/// One finished task or asset. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PythonNodeRow { + pub kind: String, + pub node_name: String, + pub status: String, + pub error_message: Option, + pub skip_reason: Option, +} diff --git a/crates/sqlbuild-core/src/build_command/tests.rs b/crates/sqlbuild-core/src/build_command/tests.rs new file mode 100644 index 0000000000..d8933807ae --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_build_command.rs"] +mod build_command; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_plan_safety.rs"] +mod plan_safety; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/build_command/tests/helpers.rs b/crates/sqlbuild-core/src/build_command/tests/helpers.rs new file mode 100644 index 0000000000..98efa3cb2a --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/tests/helpers.rs @@ -0,0 +1,92 @@ +use crate::build_command::main::plan_safety_steps::plan_safety_steps; +use crate::build_command::models::{ + ModelLimit, ModelMigrationFact, PlanSafetyFacts, PlanSafetyGates, PolicyStep, RetentionFact, + SnapshotPolicies, SnapshotRefreshFact, TableTypeFact, +}; + +/// Build-command gates with every allow flag off. +pub(super) fn build_gates() -> PlanSafetyGates { + PlanSafetyGates { + missing_origin_help: "origin help".to_owned(), + snapshot_full_refresh_help: "snapshot help".to_owned(), + table_type_downgrade_help: "table help".to_owned(), + retention_decrease_help: "retention help".to_owned(), + execution_limit_note: "No warehouse changes were made.".to_owned(), + ..PlanSafetyGates::default() + } +} + +/// Each step as one comparable line. +pub(super) fn described_steps( + facts: &PlanSafetyFacts, + gates: &PlanSafetyGates, + max_models: Option, +) -> Vec { + let limit: ModelLimit = ModelLimit { + target_name: Some("dev".to_owned()), + max_models, + remediation: None, + }; + let policies: SnapshotPolicies = SnapshotPolicies { + historical: "require_confirmation".to_owned(), + current_state: "deny".to_owned(), + }; + plan_safety_steps(facts, gates, (&limit, &policies)).map_or_else( + |error| vec![format!("error:{}", error.message)], + |steps| steps.iter().map(describe_step).collect(), + ) +} + +fn describe_step(step: &PolicyStep) -> String { + let refusal: Option = step.refusal().map(|error| { + format!( + "refuse:{}:{}", + error.code.clone().unwrap_or_default(), + error.message + ) + }); + let confirmation: Option = step.confirmation().map(|confirmation| { + format!( + "confirm:{}:{}:{}", + confirmation.expected, confirmation.refusal.message, confirmation.cancellation.message + ) + }); + refusal.or(confirmation).unwrap_or_default() +} + +pub(super) fn missing_origin(name: &str) -> ModelMigrationFact { + ModelMigrationFact { + model_name: name.to_owned(), + origin_label: format!("old_{name}"), + decision: "origin_missing".to_owned(), + origin_missing: true, + confirm_missing_origin: true, + ..ModelMigrationFact::default() + } +} + +pub(super) fn snapshot( + name: &str, + historical: bool, + model_policy: Option<&str>, +) -> SnapshotRefreshFact { + SnapshotRefreshFact { + name: name.to_owned(), + historical, + model_policy: model_policy.map(str::to_owned), + } +} + +pub(super) fn table_type(name: &str, policy: &str) -> TableTypeFact { + TableTypeFact { + model_name: name.to_owned(), + policy: policy.to_owned(), + } +} + +pub(super) fn retention(names: &[&str], policy: &str) -> RetentionFact { + RetentionFact { + model_names: names.iter().map(|name| (*name).to_owned()).collect(), + policy: policy.to_owned(), + } +} diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs new file mode 100644 index 0000000000..a2611194dd --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -0,0 +1,282 @@ +use crate::build_command::main::build_exit_code::build_exit_code; +use crate::build_command::main::duration_limit_error::duration_limit_error; +use crate::build_command::main::python_node_failed::python_node_failed; +use crate::build_command::main::render_confirmation_prompt::render_confirmation_prompt; +use crate::build_command::main::render_phase_timings::render_phase_timings; +use crate::build_command::main::render_python_node_rows::render_python_node_rows; +use crate::build_command::main::render_run_context::render_run_context; +use crate::build_command::models::{ + Confirmation, DurationLimit, PhaseTimings, PythonNodeRow, RunContextFacts, SelectorFileFact, +}; +use crate::build_command::tests::test_types::{ + ContextTestCase, DurationTestCase, ExitCodeTestCase, PromptTestCase, PythonRowsTestCase, + TimingsTestCase, +}; +use crate::cli_text::models::CliStyle; + +const PLAIN: CliStyle = CliStyle { use_color: false }; + +#[test] +fn given_target_duration_when_checking_adapter_then_refuses_unenforceable_limits() { + let test_cases = [ + DurationTestCase { + description: "an adapter without cancellation refuses max_duration", + limit: DurationLimit { + adapter_name: "duckdb".to_owned(), + configures_max_duration: true, + max_duration_seconds: Some(300), + ..DurationLimit::default() + }, + expected_error: Some(( + "C415", + "Target 'default' configures max_duration, but adapter 'duckdb' cannot cancel active statements safely", + )), + }, + DurationTestCase { + description: "a duration above the adapter maximum is refused", + limit: DurationLimit { + target_name: Some("prod".to_owned()), + adapter_name: "warehouse".to_owned(), + configures_max_duration: true, + max_duration_seconds: Some(700), + adapter_limit_seconds: Some(600), + ..DurationLimit::default() + }, + expected_error: Some(( + "C416", + "Target 'prod' max_duration exceeds the largest duration supported by adapter 'warehouse' (600 seconds)", + )), + }, + DurationTestCase { + description: "a duration within the adapter maximum passes", + limit: DurationLimit { + adapter_name: "warehouse".to_owned(), + configures_max_duration: true, + max_duration_seconds: Some(600), + adapter_limit_seconds: Some(600), + ..DurationLimit::default() + }, + expected_error: None, + }, + DurationTestCase { + description: "no duration passes on any adapter", + limit: DurationLimit { + adapter_name: "duckdb".to_owned(), + ..DurationLimit::default() + }, + expected_error: None, + }, + ]; + + for test_case in test_cases { + let actual: Option<(String, String)> = duration_limit_error(&test_case.limit) + .map(|error| (error.code.unwrap_or_default(), error.message)); + assert_eq!( + actual, + test_case + .expected_error + .map(|(code, message)| (code.to_owned(), message.to_owned())), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_phase_timings_when_rendering_then_lists_only_measured_phases() { + let test_cases = [ + TimingsTestCase { + description: "measured phases keep their order", + timings: PhaseTimings { + compile_seconds: Some(1.234), + execution_seconds: Some(0.005), + total_seconds: Some(12.0), + ..PhaseTimings::default() + }, + expected_text: "\nPhase timings\n compile 1.23s\n execution 0.01s\n total 12.00s\n", + }, + TimingsTestCase { + description: "no measured phase keeps the header", + timings: PhaseTimings::default(), + expected_text: "\nPhase timings\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_phase_timings(&test_case.timings, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_run_context_when_rendering_then_sanitizes_and_groups_counts() { + let test_cases = [ + ContextTestCase { + description: "unset values and both dates", + facts: RunContextFacts { + command: "sqb build".to_owned(), + run_id: Some("run\u{1b}1".to_owned()), + concurrency: 4, + selected_count: 1200, + total_count: 1_234_567, + date_vars: (Some("2026-01-01".to_owned()), Some("2026-01-31".to_owned())), + ..RunContextFacts::default() + }, + expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", + }, + ContextTestCase { + description: "a warehouse, one date and selector files", + facts: RunContextFacts { + command: "sqb build".to_owned(), + target: Some("dev".to_owned()), + warehouse: Some(("small".to_owned(), "connection warehouse".to_owned())), + full_refresh: true, + date_vars: (None, Some("2026-01-31".to_owned())), + selector_files: vec![ + SelectorFileFact { + path: "a.txt".to_owned(), + selector_count: 1, + }, + SelectorFileFact { + path: "b.txt".to_owned(), + selector_count: 2, + }, + ], + ..RunContextFacts::default() + }, + expected_text: "Execution\n command sqb build\n run_id not set\n target dev\n database not set\n schema not set\n warehouse small (connection warehouse)\n concurrency 0 configured limit\n full_refresh true\n selected 0 of 0 build resources\n date vars end_date=2026-01-31\n\nSelection files\n selector_file a.txt (1 selector)\n selector_file b.txt (2 selectors)\n\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_run_context(&test_case.facts, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_python_node_results_when_rendering_then_shows_status_and_detail() { + let test_cases = [ + PythonRowsTestCase { + description: "success, skip reason and failure message", + rows: vec![ + PythonNodeRow { + kind: "task".to_owned(), + node_name: "count_orders".to_owned(), + status: "success".to_owned(), + ..PythonNodeRow::default() + }, + PythonNodeRow { + kind: "asset".to_owned(), + node_name: "extract".to_owned(), + status: "skipped".to_owned(), + skip_reason: Some("upstream failed".to_owned()), + ..PythonNodeRow::default() + }, + PythonNodeRow { + kind: "task".to_owned(), + node_name: "boom".to_owned(), + status: "failed".to_owned(), + error_message: Some("boom".to_owned()), + skip_reason: Some("ignored".to_owned()), + }, + ], + expected_failed: true, + expected_text: concat!( + " python task count_orders OK\n", + " python asset extract SKIP upstream failed\n", + " python task boom FAIL boom\n", + ), + }, + PythonRowsTestCase { + description: "no results write nothing", + rows: Vec::new(), + expected_failed: false, + expected_text: "", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_python_node_rows(&test_case.rows, PLAIN), + test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!( + test_case + .rows + .iter() + .any(|row| python_node_failed(&row.status)), + test_case.expected_failed, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_build_outcome_when_resolving_exit_code_then_any_failure_is_one() { + let test_cases = [ + ExitCodeTestCase { + description: "a clean build exits 0", + outcome: (true, false, false), + expected_exit_code: 0, + }, + ExitCodeTestCase { + description: "a failed build exits 1", + outcome: (false, false, false), + expected_exit_code: 1, + }, + ExitCodeTestCase { + description: "a failed Python node exits 1", + outcome: (true, true, false), + expected_exit_code: 1, + }, + ExitCodeTestCase { + description: "a failed check exits 1", + outcome: (true, false, true), + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + let (succeeded, python_failed, checks_failed) = test_case.outcome; + assert_eq!( + build_exit_code(succeeded, (python_failed, checks_failed)), + test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_confirmation_when_rendering_prompt_then_asks_for_expected_text() { + let test_cases = [PromptTestCase { + description: "the warning precedes the typed prompt", + confirmation: Confirmation { + warning: "Careful.".to_owned(), + expected: "do it".to_owned(), + ..Confirmation::default() + }, + expected_text: "Careful.\n\nType `do it` to continue: ", + }]; + + for test_case in test_cases { + assert_eq!( + render_confirmation_prompt(&test_case.confirmation), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/build_command/tests/test_plan_safety.rs b/crates/sqlbuild-core/src/build_command/tests/test_plan_safety.rs new file mode 100644 index 0000000000..350e3bfcc4 --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/tests/test_plan_safety.rs @@ -0,0 +1,236 @@ +use crate::build_command::models::{ + ColumnMigrationFact, ModelMigrationFact, PlanSafetyFacts, PlanSafetyGates, PlanWarningFact, +}; +use crate::build_command::tests::helpers::{ + build_gates, described_steps, missing_origin, retention, snapshot, table_type, +}; +use crate::build_command::tests::test_types::PolicyTestCase; + +#[test] +fn given_plan_facts_when_applying_safety_gates_then_refuses_or_confirms_in_order() { + let test_cases = [ + PolicyTestCase { + description: "an empty plan has no gates", + facts: PlanSafetyFacts::default(), + gates: build_gates(), + max_models: None, + expected_steps: Vec::new(), + }, + PolicyTestCase { + description: "a hidden blocking origin refuses first", + facts: PlanSafetyFacts { + model_migrations: vec![ModelMigrationFact { + blocks_build: true, + origin_hidden: true, + decision: "conflict".to_owned(), + ..missing_origin("orders") + }], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec![ + "refuse:M102:model migration origin was built in this target but is not visible for 'orders' (migrate_from old_orders); the connection may lack permission to see it, or it was dropped outside SQLBuild; restore access to it, or remove migrate_from to build the model fresh deliberately (missing_migration_origin does not apply)".to_owned(), + ], + }, + PolicyTestCase { + description: "a conflict lists every blocked model", + facts: PlanSafetyFacts { + model_migrations: vec![ + ModelMigrationFact { + blocks_build: true, + decision: "conflict".to_owned(), + ..missing_origin("orders") + }, + ModelMigrationFact { + blocks_build: true, + decision: "migrate".to_owned(), + ..missing_origin("customers") + }, + ], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec![ + "refuse:M103:model migration conflict for 'orders', 'customers': the destination already exists with its own build history and no recorded migration".to_owned(), + ], + }, + PolicyTestCase { + description: "a blocked column rename uses its decision code", + facts: PlanSafetyFacts { + column_migrations: vec![ColumnMigrationFact { + model_name: "orders".to_owned(), + destination_column: "total".to_owned(), + origin_column: "amount".to_owned(), + decision: "still_produced".to_owned(), + decision_label: "still produced".to_owned(), + blocks_build: true, + ..ColumnMigrationFact::default() + }], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec![ + "refuse:M111:column migrations are blocked for 'orders' (total migrate_from amount: still produced)".to_owned(), + ], + }, + PolicyTestCase { + description: "missing origins confirm once with sorted names, then the limit refuses", + facts: PlanSafetyFacts { + model_migrations: vec![missing_origin("orders"), missing_origin("customers")], + executable_model_count: 3, + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: Some(2), + expected_steps: vec![ + "confirm:build 2 models without their migrations:building 'customers', 'orders' without its missing migrate_from origin requires confirmation:building 'customers', 'orders' without its missing migrate_from origin cancelled".to_owned(), + "refuse:C413:Build plan exceeds the configured model limit\n\nTarget: dev\nSelected models: 3\nMaximum models: 2\nConfiguration: targets.dev.execution_limits.max_models\n\nNo warehouse changes were made.".to_owned(), + ], + }, + PolicyTestCase { + description: "the allow flag skips the missing-origin confirmation", + facts: PlanSafetyFacts { + model_migrations: vec![missing_origin("orders")], + ..PlanSafetyFacts::default() + }, + gates: PlanSafetyGates { + allow_missing_migration_origin: true, + ..build_gates() + }, + max_models: None, + expected_steps: Vec::new(), + }, + PolicyTestCase { + description: "old-name errors join their messages", + facts: PlanSafetyFacts { + error_warnings: vec![ + PlanWarningFact { + code: Some("P008".to_owned()), + message: "first".to_owned(), + }, + PlanWarningFact { + code: Some("P001".to_owned()), + message: "unrelated".to_owned(), + }, + PlanWarningFact { + code: Some("M114".to_owned()), + message: "second".to_owned(), + }, + ], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec!["refuse:P008:first; second".to_owned()], + }, + PolicyTestCase { + description: "a current-state snapshot is denied by the project policy", + facts: PlanSafetyFacts { + snapshot_refreshes: vec![snapshot("current", false, None)], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec![ + "refuse:C238:full refresh is denied for snapshot model 'current' by snapshot_full_refresh policy".to_owned(), + ], + }, + PolicyTestCase { + description: "the stricter model policy wins and storage gates follow in order", + facts: PlanSafetyFacts { + snapshot_refreshes: vec![ + snapshot("history", true, Some("allow")), + snapshot("other", false, Some("require_confirmation")), + ], + table_type_downgrades: vec![table_type("orders", "require_confirmation")], + retention_decreases: vec![ + retention(&["orders", "customers"], "require_confirmation"), + retention(&["orders"], "require_confirmation"), + ], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec![ + "refuse:C238:full refresh is denied for snapshot model 'other' by snapshot_full_refresh policy".to_owned(), + ], + }, + PolicyTestCase { + description: "confirmations for snapshot, table type and retention", + facts: PlanSafetyFacts { + snapshot_refreshes: vec![snapshot("history", true, None)], + table_type_downgrades: vec![ + table_type("orders", "require_confirmation"), + table_type("customers", "allow"), + ], + retention_decreases: vec![ + retention(&["orders", "customers"], "require_confirmation"), + retention(&["orders"], "require_confirmation"), + ], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec![ + "confirm:discard snapshot history for history:snapshot full refresh requires confirmation:snapshot full refresh cancelled".to_owned(), + "confirm:downgrade table type for orders:table-type downgrade requires confirmation:table-type downgrade cancelled".to_owned(), + "confirm:decrease retention for 2 models:time travel retention decrease requires confirmation:time travel retention decrease cancelled".to_owned(), + ], + }, + PolicyTestCase { + description: "allow flags skip storage confirmations but not denials", + facts: PlanSafetyFacts { + snapshot_refreshes: vec![snapshot("history", true, None)], + table_type_downgrades: vec![table_type("orders", "require_confirmation")], + retention_decreases: vec![retention(&["orders"], "deny")], + ..PlanSafetyFacts::default() + }, + gates: PlanSafetyGates { + allow_snapshot_full_refresh: true, + allow_table_type_downgrade: true, + allow_retention_decrease: true, + ..build_gates() + }, + max_models: None, + expected_steps: vec![ + "refuse::time travel retention decrease is denied for 'orders'".to_owned(), + ], + }, + PolicyTestCase { + description: "a denied table type refuses without a code", + facts: PlanSafetyFacts { + table_type_downgrades: vec![table_type("orders", "deny"), table_type("customers", "deny")], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec![ + "refuse::table-type downgrade is denied for 'orders', 'customers' by table_type_downgrade policy".to_owned(), + ], + }, + PolicyTestCase { + description: "an unknown snapshot policy is an error", + facts: PlanSafetyFacts { + snapshot_refreshes: vec![snapshot("history", true, Some("sometimes"))], + ..PlanSafetyFacts::default() + }, + gates: build_gates(), + max_models: None, + expected_steps: vec![ + "error:'sometimes' is not a valid SnapshotFullRefreshPolicy".to_owned(), + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + described_steps(&test_case.facts, &test_case.gates, test_case.max_models), + test_case.expected_steps, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/build_command/tests/test_types.rs b/crates/sqlbuild-core/src/build_command/tests/test_types.rs new file mode 100644 index 0000000000..2f26bb4faf --- /dev/null +++ b/crates/sqlbuild-core/src/build_command/tests/test_types.rs @@ -0,0 +1,50 @@ +use crate::build_command::models::{ + Confirmation, DurationLimit, PhaseTimings, PlanSafetyFacts, PlanSafetyGates, PythonNodeRow, + RunContextFacts, +}; + +pub(super) struct PolicyTestCase { + pub(super) description: &'static str, + pub(super) facts: PlanSafetyFacts, + pub(super) gates: PlanSafetyGates, + pub(super) max_models: Option, + /// `refuse::` or `confirm:::`. + pub(super) expected_steps: Vec, +} + +pub(super) struct DurationTestCase { + pub(super) description: &'static str, + pub(super) limit: DurationLimit, + pub(super) expected_error: Option<(&'static str, &'static str)>, +} + +pub(super) struct TimingsTestCase { + pub(super) description: &'static str, + pub(super) timings: PhaseTimings, + pub(super) expected_text: &'static str, +} + +pub(super) struct ContextTestCase { + pub(super) description: &'static str, + pub(super) facts: RunContextFacts, + pub(super) expected_text: &'static str, +} + +pub(super) struct PythonRowsTestCase { + pub(super) description: &'static str, + pub(super) rows: Vec, + pub(super) expected_failed: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct ExitCodeTestCase { + pub(super) description: &'static str, + pub(super) outcome: (bool, bool, bool), + pub(super) expected_exit_code: i32, +} + +pub(super) struct PromptTestCase { + pub(super) description: &'static str, + pub(super) confirmation: Confirmation, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/contract_command/_helpers/json_document.rs b/crates/sqlbuild-core/src/contract_command/_helpers/json_document.rs new file mode 100644 index 0000000000..f6f53e3f12 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/_helpers/json_document.rs @@ -0,0 +1,40 @@ +//! JSON values of contract resources and findings. + +use crate::contract_command::models::{ContractFindingText, ContractResourceText}; +use crate::json::models::JsonValue; + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn optional(value: Option<&String>) -> JsonValue { + value.map_or(JsonValue::Null, |value| text(value)) +} + +fn finding(finding: &ContractFindingText) -> JsonValue { + JsonValue::Object(vec![ + ("kind".to_owned(), text(&finding.kind)), + ("column".to_owned(), optional(finding.column.as_ref())), + ( + "declared_type".to_owned(), + optional(finding.declared_type.as_ref()), + ), + ( + "physical_type".to_owned(), + optional(finding.physical_type.as_ref()), + ), + ("message".to_owned(), text(&finding.message)), + ]) +} + +pub(crate) fn resource(resource: &ContractResourceText) -> JsonValue { + JsonValue::Object(vec![ + ("resource_type".to_owned(), text(&resource.resource_type)), + ("resource_name".to_owned(), text(&resource.resource_name)), + ("relation".to_owned(), text(&resource.relation)), + ( + "findings".to_owned(), + JsonValue::Array(resource.findings.iter().map(finding).collect()), + ), + ]) +} diff --git a/crates/sqlbuild-core/src/contract_command/_helpers/mod.rs b/crates/sqlbuild-core/src/contract_command/_helpers/mod.rs new file mode 100644 index 0000000000..1007643e64 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod json_document; diff --git a/crates/sqlbuild-core/src/contract_command/main/contract_exit_code.rs b/crates/sqlbuild-core/src/contract_command/main/contract_exit_code.rs new file mode 100644 index 0000000000..71074aa350 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/main/contract_exit_code.rs @@ -0,0 +1,7 @@ +//! The exit code of `sqb contract`. + +/// 1 when any difference remains, else 0. +#[must_use] +pub fn contract_exit_code(differences: usize) -> i32 { + i32::from(differences > 0) +} diff --git a/crates/sqlbuild-core/src/contract_command/main/contract_progress_message.rs b/crates/sqlbuild-core/src/contract_command/main/contract_progress_message.rs new file mode 100644 index 0000000000..316326e0ae --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/main/contract_progress_message.rs @@ -0,0 +1,36 @@ +//! Lifecycle messages of contract inspection and repository updates. + +use crate::contract_command::models::ContractProgress; + +fn counted(count: usize, singular: &str) -> String { + format!("{count} {singular}{}", if count == 1 { "" } else { "s" }) +} + +/// The status line text of one contract lifecycle event. +#[must_use] +pub fn contract_progress_message(event: &ContractProgress) -> String { + match event { + ContractProgress::InspectionStart { count, target } => format!( + "Inspecting {} from target '{target}'...", + counted(*count, "contract") + ), + ContractProgress::InspectionComplete { + count, + target, + seconds, + } => format!( + "Inspected {} from target '{target}' ({seconds:.2}s)", + counted(*count, "contract") + ), + ContractProgress::InspectionFailed { seconds } => { + format!("Contract inspection failed after {seconds:.2}s") + } + ContractProgress::WriteComplete { count, seconds } => format!( + "Updated repository contracts ({}, {seconds:.2}s)", + counted(*count, "file") + ), + ContractProgress::WriteFailed { seconds } => { + format!("Repository contract update failed after {seconds:.2}s") + } + } +} diff --git a/crates/sqlbuild-core/src/contract_command/main/mod.rs b/crates/sqlbuild-core/src/contract_command/main/mod.rs new file mode 100644 index 0000000000..25c21a356a --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/main/mod.rs @@ -0,0 +1,4 @@ +pub mod contract_exit_code; +pub mod contract_progress_message; +pub mod render_contract_json; +pub mod render_contract_text; diff --git a/crates/sqlbuild-core/src/contract_command/main/render_contract_json.rs b/crates/sqlbuild-core/src/contract_command/main/render_contract_json.rs new file mode 100644 index 0000000000..606c9efc51 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/main/render_contract_json.rs @@ -0,0 +1,31 @@ +//! The contract comparison as sorted, indented JSON. + +use crate::contract_command::_helpers::json_document::{resource, text}; +use crate::contract_command::models::ContractReport; +use crate::json::errors::JsonEmitError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonInteger, JsonValue, StdlibJsonOptions}; + +/// `json.dumps(payload, indent=2, sort_keys=True)` of the report, and a newline. +pub fn render_contract_json(report: &ContractReport) -> Result { + let differences: JsonInteger = + JsonInteger::from(u64::try_from(report.difference_count()).unwrap_or(u64::MAX)); + let document: JsonValue = JsonValue::Object(vec![ + ("from_target".to_owned(), text(&report.from_target)), + ("differences".to_owned(), JsonValue::Integer(differences)), + ( + "written_paths".to_owned(), + JsonValue::Array(report.written_paths.iter().map(|path| text(path)).collect()), + ), + ( + "resources".to_owned(), + JsonValue::Array(report.resources.iter().map(resource).collect()), + ), + ]); + let mut options: StdlibJsonOptions = StdlibJsonOptions::new(Some(2)); + options.sort_keys = true; + Ok(format!( + "{}\n", + dumps(&document, &JsonDialect::Stdlib(options))? + )) +} diff --git a/crates/sqlbuild-core/src/contract_command/main/render_contract_text.rs b/crates/sqlbuild-core/src/contract_command/main/render_contract_text.rs new file mode 100644 index 0000000000..abde3efd43 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/main/render_contract_text.rs @@ -0,0 +1,39 @@ +//! The human contract comparison report. + +use crate::contract_command::models::{ContractReport, ContractResourceText}; + +fn resource_lines(resource: &ContractResourceText) -> Vec { + let label: String = format!("{}:{}", resource.resource_type, resource.resource_name); + if resource.findings.is_empty() { + return vec![format!(" OK {label}")]; + } + let mut lines: Vec = vec![format!(" DIFF {label}")]; + lines.extend( + resource + .findings + .iter() + .map(|finding| format!(" {}: {}", finding.kind, finding.message)), + ); + lines +} + +/// Each resource as OK or DIFF with its findings, written files, and the difference count. +#[must_use] +pub fn render_contract_text(report: &ContractReport) -> String { + let mut lines: Vec = vec![format!( + "Contract comparison from target '{}'", + report.from_target + )]; + for resource in &report.resources { + lines.extend(resource_lines(resource)); + } + if !report.written_paths.is_empty() { + lines.push("\nUpdated repository declarations:".to_owned()); + lines.extend(report.written_paths.iter().map(|path| format!(" {path}"))); + } + lines.push(format!( + "\n{} contract difference(s)", + report.difference_count() + )); + format!("{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-core/src/contract_command/mod.rs b/crates/sqlbuild-core/src/contract_command/mod.rs new file mode 100644 index 0000000000..fdd8c53a45 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/mod.rs @@ -0,0 +1,8 @@ +//! `sqb contract`: progress messages, the comparison report, its JSON, and the exit code. + +pub(crate) mod _helpers; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/contract_command/models.rs b/crates/sqlbuild-core/src/contract_command/models.rs new file mode 100644 index 0000000000..89216db2d4 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/models.rs @@ -0,0 +1,64 @@ +//! Contract comparison results as the command reports them. + +/// One difference between a declared and a physical contract. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ContractFindingText { + pub kind: String, + pub column: Option, + pub declared_type: Option, + pub physical_type: Option, + pub message: String, +} + +/// One compared model or source. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ContractResourceText { + pub resource_type: String, + pub resource_name: String, + /// The dotted relation of its non-empty database, schema and name. + pub relation: String, + pub findings: Vec, +} + +/// The whole comparison, with any repository files written. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ContractReport { + pub from_target: String, + pub resources: Vec, + pub written_paths: Vec, +} + +impl ContractReport { + /// How many differences all resources hold. + #[must_use] + pub fn difference_count(&self) -> usize { + self.resources + .iter() + .map(|resource| resource.findings.len()) + .sum() + } +} + +/// One contract lifecycle event and what its status line reports. +#[derive(Clone, Debug, PartialEq)] +pub enum ContractProgress { + InspectionStart { + count: usize, + target: String, + }, + InspectionComplete { + count: usize, + target: String, + seconds: f64, + }, + InspectionFailed { + seconds: f64, + }, + WriteComplete { + count: usize, + seconds: f64, + }, + WriteFailed { + seconds: f64, + }, +} diff --git a/crates/sqlbuild-core/src/contract_command/tests.rs b/crates/sqlbuild-core/src/contract_command/tests.rs new file mode 100644 index 0000000000..2e151816ae --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_contract_command.rs"] +mod contract_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/contract_command/tests/test_contract_command.rs b/crates/sqlbuild-core/src/contract_command/tests/test_contract_command.rs new file mode 100644 index 0000000000..19cc1e4357 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/tests/test_contract_command.rs @@ -0,0 +1,123 @@ +use crate::contract_command::main::contract_exit_code::contract_exit_code; +use crate::contract_command::main::contract_progress_message::contract_progress_message; +use crate::contract_command::main::render_contract_json::render_contract_json; +use crate::contract_command::main::render_contract_text::render_contract_text; +use crate::contract_command::models::{ + ContractFindingText, ContractProgress, ContractReport, ContractResourceText, +}; +use crate::contract_command::tests::test_types::{MessageTestCase, ReportTestCase}; + +#[test] +fn given_comparison_when_rendering_then_reports_text_json_and_exit_code() { + let test_cases = [ + ReportTestCase { + description: "a matching source after a write", + report: ContractReport { + from_target: "prod".to_owned(), + resources: vec![ContractResourceText { + resource_type: "source".to_owned(), + resource_name: "raw_orders".to_owned(), + relation: "raw.orders".to_owned(), + findings: Vec::new(), + }], + written_paths: vec!["sources/raw.yml".to_owned()], + }, + expected_text: "Contract comparison from target 'prod'\n OK source:raw_orders\n\nUpdated repository declarations:\n sources/raw.yml\n\n0 contract difference(s)\n", + expected_json: "{\n \"differences\": 0,\n \"from_target\": \"prod\",\n \"resources\": [\n {\n \"findings\": [],\n \"relation\": \"raw.orders\",\n \"resource_name\": \"raw_orders\",\n \"resource_type\": \"source\"\n }\n ],\n \"written_paths\": [\n \"sources/raw.yml\"\n ]\n}\n", + expected_exit_code: 0, + }, + ReportTestCase { + description: "a model with one type difference", + report: ContractReport { + from_target: "prod".to_owned(), + resources: vec![ContractResourceText { + resource_type: "model".to_owned(), + resource_name: "orders".to_owned(), + relation: "prod.orders".to_owned(), + findings: vec![ContractFindingText { + kind: "type_mismatch".to_owned(), + column: Some("id".to_owned()), + declared_type: Some("VARCHAR".to_owned()), + physical_type: Some("INTEGER".to_owned()), + message: "id differs".to_owned(), + }], + }], + written_paths: Vec::new(), + }, + expected_text: "Contract comparison from target 'prod'\n DIFF model:orders\n type_mismatch: id differs\n\n1 contract difference(s)\n", + expected_json: "{\n \"differences\": 1,\n \"from_target\": \"prod\",\n \"resources\": [\n {\n \"findings\": [\n {\n \"column\": \"id\",\n \"declared_type\": \"VARCHAR\",\n \"kind\": \"type_mismatch\",\n \"message\": \"id differs\",\n \"physical_type\": \"INTEGER\"\n }\n ],\n \"relation\": \"prod.orders\",\n \"resource_name\": \"orders\",\n \"resource_type\": \"model\"\n }\n ],\n \"written_paths\": []\n}\n", + expected_exit_code: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_contract_text(&test_case.report), + test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!( + render_contract_json(&test_case.report).ok().as_deref(), + Some(test_case.expected_json), + "{}", + test_case.description + ); + assert_eq!( + contract_exit_code(test_case.report.difference_count()), + test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_counts_when_reporting_progress_then_pluralizes_and_times() { + let test_cases = [ + MessageTestCase { + description: "one contract starts", + event: ContractProgress::InspectionStart { + count: 1, + target: "prod".to_owned(), + }, + expected_message: "Inspecting 1 contract from target 'prod'...", + }, + MessageTestCase { + description: "two contracts complete", + event: ContractProgress::InspectionComplete { + count: 2, + target: "prod".to_owned(), + seconds: 0.125, + }, + expected_message: "Inspected 2 contracts from target 'prod' (0.12s)", + }, + MessageTestCase { + description: "inspection fails", + event: ContractProgress::InspectionFailed { seconds: 1.5 }, + expected_message: "Contract inspection failed after 1.50s", + }, + MessageTestCase { + description: "one file written", + event: ContractProgress::WriteComplete { + count: 1, + seconds: 0.0, + }, + expected_message: "Updated repository contracts (1 file, 0.00s)", + }, + MessageTestCase { + description: "write fails", + event: ContractProgress::WriteFailed { seconds: 2.0 }, + expected_message: "Repository contract update failed after 2.00s", + }, + ]; + + for test_case in test_cases { + assert_eq!( + contract_progress_message(&test_case.event), + test_case.expected_message, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/contract_command/tests/test_types.rs b/crates/sqlbuild-core/src/contract_command/tests/test_types.rs new file mode 100644 index 0000000000..f2fb2674d2 --- /dev/null +++ b/crates/sqlbuild-core/src/contract_command/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::contract_command::models::{ContractProgress, ContractReport}; + +pub(super) struct ReportTestCase { + pub(super) description: &'static str, + pub(super) report: ContractReport, + pub(super) expected_text: &'static str, + pub(super) expected_json: &'static str, + pub(super) expected_exit_code: i32, +} + +pub(super) struct MessageTestCase { + pub(super) description: &'static str, + pub(super) event: ContractProgress, + pub(super) expected_message: &'static str, +} diff --git a/crates/sqlbuild-core/src/cost_command/constants.rs b/crates/sqlbuild-core/src/cost_command/constants.rs new file mode 100644 index 0000000000..83f1bbc19b --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/constants.rs @@ -0,0 +1,29 @@ +//! Cost command selectors, sort fields and defaults. + +pub const HISTORY_SELECTOR: &str = "history"; +pub const LATEST_SELECTOR: &str = "latest"; +pub const DESCENDING_ORDER: &str = "desc"; +pub const DEFAULT_HISTORY_SORT: &str = "completed"; +pub const DEFAULT_DETAIL_SORT: &str = "cost"; +pub const DEFAULT_HISTORY_LIMIT: i64 = 10; +pub const ISO_DATE_LENGTH: usize = 10; +/// History sort fields, sorted. +pub const HISTORY_SORT_FIELDS: [&str; 6] = [ + "completed", + "cost", + "credits", + "duration", + "models", + "status", +]; +/// Breakdown sort fields, sorted. +pub const BREAKDOWN_SORT_FIELDS: [&str; 6] = + ["busy", "cost", "credits", "model", "scanned", "warehouse"]; +/// History fields whose runs without metrics sort after the rest. +pub const HISTORY_METRIC_SORT_FIELDS: [&str; 2] = ["cost", "credits"]; +/// Statuses whose runs carry no complete metrics. +pub const MISSING_METRIC_STATUSES: [&str; 4] = + ["pending", "partial", "unavailable", "collection_failed"]; +/// Statuses a delayed refresh may still improve. +pub const REFRESHABLE_STATUSES: [&str; 2] = ["pending", "partial"]; +pub const LEDGER_PERSISTENCE_FAILURE_PREFIX: &str = "Statement-ledger persistence failed ("; diff --git a/crates/sqlbuild-core/src/cost_command/errors.rs b/crates/sqlbuild-core/src/cost_command/errors.rs new file mode 100644 index 0000000000..b33ab123cf --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/errors.rs @@ -0,0 +1,18 @@ +//! Expected cost command errors and their CLI codes. + +/// A refused cost request: its CLI error code and message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostCommandError { + pub code: &'static str, + pub message: String, +} + +impl CostCommandError { + #[must_use] + pub fn new(code: &'static str, message: impl Into) -> Self { + Self { + code, + message: message.into(), + } + } +} diff --git a/crates/sqlbuild-core/src/cost_command/main/accepts_cost_refresh.rs b/crates/sqlbuild-core/src/cost_command/main/accepts_cost_refresh.rs new file mode 100644 index 0000000000..c6e416abb6 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/main/accepts_cost_refresh.rs @@ -0,0 +1,40 @@ +//! Whether refreshed cost telemetry may replace the persisted summary. + +use crate::cost_command::constants::LEDGER_PERSISTENCE_FAILURE_PREFIX; +use crate::cost_command::models::CostCoverage; + +fn status_rank(status: &str) -> Option { + match status { + "collection_failed" | "unavailable" => Some(0), + "pending" => Some(1), + "partial" => Some(2), + "complete" => Some(3), + _ => None, + } +} + +fn coverage_gain(current: &CostCoverage, candidate: &CostCoverage) -> bool { + candidate.expected_statement_count > current.expected_statement_count + || candidate.observed_statement_count > current.observed_statement_count + || candidate.query_count > current.query_count +} + +/// Accept only when status and coverage cannot regress; `None` for an unknown status. +#[must_use] +pub fn accepts_cost_refresh(current: &CostCoverage, candidate: &CostCoverage) -> Option { + if status_rank(&candidate.status)? < status_rank(¤t.status)? { + return Some(false); + } + let ledger_failure: bool = current + .limitations + .iter() + .any(|limitation| limitation.starts_with(LEDGER_PERSISTENCE_FAILURE_PREFIX)); + if ledger_failure && !coverage_gain(current, candidate) { + return Some(false); + } + Some( + candidate.expected_statement_count >= current.expected_statement_count + && candidate.observed_statement_count >= current.observed_statement_count + && candidate.query_count >= current.query_count, + ) +} diff --git a/crates/sqlbuild-core/src/cost_command/main/cost_history_limit.rs b/crates/sqlbuild-core/src/cost_command/main/cost_history_limit.rs new file mode 100644 index 0000000000..4fa2b7c7ca --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/main/cost_history_limit.rs @@ -0,0 +1,12 @@ +//! How many history rows to show. + +use crate::cost_command::constants::DEFAULT_HISTORY_LIMIT; + +/// None for `--no-limit`, else the requested limit or the default. +#[must_use] +pub fn cost_history_limit(limit: Option, no_limit: bool) -> Option { + if no_limit { + return None; + } + Some(limit.unwrap_or(DEFAULT_HISTORY_LIMIT)) +} diff --git a/crates/sqlbuild-core/src/cost_command/main/is_zero_cost_duration.rs b/crates/sqlbuild-core/src/cost_command/main/is_zero_cost_duration.rs new file mode 100644 index 0000000000..31d25efaba --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/main/is_zero_cost_duration.rs @@ -0,0 +1,10 @@ +//! A `--since` of zero days, hours or minutes. + +/// Whether the value is one or more zeros followed by `d`, `h` or `m`. +#[must_use] +pub fn is_zero_cost_duration(value: &str) -> bool { + let Some(digits) = value.strip_suffix(['d', 'h', 'm']) else { + return false; + }; + !digits.is_empty() && digits.bytes().all(|byte| byte == b'0') +} diff --git a/crates/sqlbuild-core/src/cost_command/main/mod.rs b/crates/sqlbuild-core/src/cost_command/main/mod.rs new file mode 100644 index 0000000000..3f5f0cdb27 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/main/mod.rs @@ -0,0 +1,5 @@ +pub mod accepts_cost_refresh; +pub mod cost_history_limit; +pub mod is_zero_cost_duration; +pub mod select_cost_run; +pub mod validate_cost_request; diff --git a/crates/sqlbuild-core/src/cost_command/main/select_cost_run.rs b/crates/sqlbuild-core/src/cost_command/main/select_cost_run.rs new file mode 100644 index 0000000000..0d117e92ca --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/main/select_cost_run.rs @@ -0,0 +1,34 @@ +//! The persisted run a selector names. + +use crate::cost_command::errors::CostCommandError; + +/// The index of the exact run id, else of the one run id the selector prefixes. +pub fn select_cost_run(run_ids: &[String], selector: &str) -> Result { + if let Some(index) = run_ids.iter().position(|run_id| run_id == selector) { + return Ok(index); + } + let matches: Vec = run_ids + .iter() + .enumerate() + .filter(|(_, run_id)| run_id.starts_with(selector)) + .map(|(index, _)| index) + .collect(); + match matches.as_slice() { + [] => Err(CostCommandError::new( + "C705", + format!("No cost run matches '{selector}'"), + )), + [only] => Ok(*only), + _ => Err(CostCommandError::new( + "C706", + format!( + "Cost run prefix '{selector}' is ambiguous; matches: {}", + matches + .iter() + .map(|index| run_ids[*index].as_str()) + .collect::>() + .join(", ") + ), + )), + } +} diff --git a/crates/sqlbuild-core/src/cost_command/main/validate_cost_request.rs b/crates/sqlbuild-core/src/cost_command/main/validate_cost_request.rs new file mode 100644 index 0000000000..d160caf2e1 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/main/validate_cost_request.rs @@ -0,0 +1,55 @@ +//! The limit, sort and bound checks of a cost request. + +use crate::cost_command::constants::{ + BREAKDOWN_SORT_FIELDS, HISTORY_SELECTOR, HISTORY_SORT_FIELDS, +}; +use crate::cost_command::errors::CostCommandError; + +/// Refuse a bad limit, then an unknown sort field or `--since`/`--until` outside history. +pub fn validate_cost_request( + selector: &str, + (limit, no_limit): (Option, bool), + sort_field: &str, + has_bounds: bool, +) -> Result<(), CostCommandError> { + if limit.is_some_and(|limit| limit < 0) { + return Err(CostCommandError::new( + "C709", + "cost --limit must be non-negative", + )); + } + if no_limit && limit.is_some() { + return Err(CostCommandError::new( + "C710", + "cost --limit and --no-limit cannot be combined", + )); + } + if selector == HISTORY_SELECTOR { + if !HISTORY_SORT_FIELDS.contains(&sort_field) { + return Err(CostCommandError::new( + "C701", + format!( + "cost history --sort must be one of: {}", + HISTORY_SORT_FIELDS.join(", ") + ), + )); + } + return Ok(()); + } + if has_bounds { + return Err(CostCommandError::new( + "C702", + "--since and --until are only valid for cost history", + )); + } + if !BREAKDOWN_SORT_FIELDS.contains(&sort_field) { + return Err(CostCommandError::new( + "C703", + format!( + "cost --sort must be one of: {}", + BREAKDOWN_SORT_FIELDS.join(", ") + ), + )); + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/cost_command/mod.rs b/crates/sqlbuild-core/src/cost_command/mod.rs new file mode 100644 index 0000000000..93d93f21c4 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/mod.rs @@ -0,0 +1,9 @@ +//! `sqb cost`: request validation, run selection, and delayed-refresh acceptance. + +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/cost_command/models.rs b/crates/sqlbuild-core/src/cost_command/models.rs new file mode 100644 index 0000000000..e6d70de69c --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/models.rs @@ -0,0 +1,11 @@ +//! Cost summary facts a delayed refresh compares. + +/// The status and statement coverage of one persisted or refreshed cost summary. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CostCoverage { + pub status: String, + pub expected_statement_count: i64, + pub observed_statement_count: i64, + pub query_count: i64, + pub limitations: Vec, +} diff --git a/crates/sqlbuild-core/src/cost_command/tests.rs b/crates/sqlbuild-core/src/cost_command/tests.rs new file mode 100644 index 0000000000..7003564f27 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_cost_command.rs"] +mod cost_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/cost_command/tests/test_cost_command.rs b/crates/sqlbuild-core/src/cost_command/tests/test_cost_command.rs new file mode 100644 index 0000000000..02698797c9 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/tests/test_cost_command.rs @@ -0,0 +1,221 @@ +use crate::cost_command::main::accepts_cost_refresh::accepts_cost_refresh; +use crate::cost_command::main::cost_history_limit::cost_history_limit; +use crate::cost_command::main::is_zero_cost_duration::is_zero_cost_duration; +use crate::cost_command::main::select_cost_run::select_cost_run; +use crate::cost_command::main::validate_cost_request::validate_cost_request; +use crate::cost_command::models::CostCoverage; +use crate::cost_command::tests::test_types::{ + RefreshTestCase, ScalarTestCase, SelectionTestCase, ValidationTestCase, +}; + +#[test] +fn given_request_when_validating_then_refuses_with_the_expected_code() { + let test_cases = [ + ValidationTestCase { + description: "a negative limit", + selector: "history", + limits: (Some(-1), false), + sort_field: "completed", + has_bounds: false, + expected_code: Some("C709"), + }, + ValidationTestCase { + description: "a limit with no limit", + selector: "latest", + limits: (Some(1), true), + sort_field: "cost", + has_bounds: false, + expected_code: Some("C710"), + }, + ValidationTestCase { + description: "a breakdown field in history", + selector: "history", + limits: (None, false), + sort_field: "busy", + has_bounds: true, + expected_code: Some("C701"), + }, + ValidationTestCase { + description: "bounds on a run breakdown", + selector: "latest", + limits: (None, false), + sort_field: "nope", + has_bounds: true, + expected_code: Some("C702"), + }, + ValidationTestCase { + description: "an unknown breakdown field", + selector: "run-1", + limits: (None, false), + sort_field: "completed", + has_bounds: false, + expected_code: Some("C703"), + }, + ValidationTestCase { + description: "a valid breakdown", + selector: "run-1", + limits: (Some(0), false), + sort_field: "warehouse", + has_bounds: false, + expected_code: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + validate_cost_request( + test_case.selector, + test_case.limits, + test_case.sort_field, + test_case.has_bounds + ) + .err() + .map(|error| error.code), + test_case.expected_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selector_when_selecting_run_then_prefers_exact_then_unique_prefix() { + let run_ids: Vec = ["run-a1", "run-a2", "run-b1", "run-a"] + .iter() + .map(|run_id| (*run_id).to_owned()) + .collect(); + let test_cases = [ + SelectionTestCase { + description: "an exact id wins over prefixes", + selector: "run-a", + expected_index: Some(3), + expected_code: None, + }, + SelectionTestCase { + description: "a unique prefix", + selector: "run-b", + expected_index: Some(2), + expected_code: None, + }, + SelectionTestCase { + description: "no match", + selector: "nope", + expected_index: None, + expected_code: Some("C705"), + }, + SelectionTestCase { + description: "a shared prefix", + selector: "run", + expected_index: None, + expected_code: Some("C706"), + }, + ]; + + for test_case in test_cases { + let selected = select_cost_run(&run_ids, test_case.selector); + assert_eq!( + selected.clone().ok(), + test_case.expected_index, + "{}", + test_case.description + ); + assert_eq!( + selected.err().map(|error| error.code), + test_case.expected_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_refreshed_summary_when_comparing_then_never_regresses() { + let coverage = |status: &str, counts: (i64, i64, i64), limitations: &[&str]| CostCoverage { + status: status.to_owned(), + expected_statement_count: counts.0, + observed_statement_count: counts.1, + query_count: counts.2, + limitations: limitations.iter().map(|text| (*text).to_owned()).collect(), + }; + let test_cases = [ + RefreshTestCase { + description: "a complete refresh of a pending run", + current: coverage("pending", (1, 0, 0), &[]), + candidate: coverage("complete", (1, 1, 1), &[]), + expected_accepted: Some(true), + }, + RefreshTestCase { + description: "a failure never replaces a partial run", + current: coverage("partial", (17, 17, 2), &[]), + candidate: coverage("collection_failed", (17, 0, 0), &[]), + expected_accepted: Some(false), + }, + RefreshTestCase { + description: "a ledger failure needs a coverage gain", + current: coverage( + "partial", + (2, 2, 2), + &["Statement-ledger persistence failed (x)"], + ), + candidate: coverage("complete", (2, 2, 2), &[]), + expected_accepted: Some(false), + }, + RefreshTestCase { + description: "an unknown status", + current: coverage("partial", (0, 0, 0), &[]), + candidate: coverage("other", (0, 0, 0), &[]), + expected_accepted: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + accepts_cost_refresh(&test_case.current, &test_case.candidate), + test_case.expected_accepted, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_values_when_checking_zero_durations_and_limits_then_matches_cost_rules() { + let test_cases = [ + ScalarTestCase { + description: "zero days with the default limit", + value: "00d", + limits: (None, false), + expected_zero: true, + expected_limit: Some(10), + }, + ScalarTestCase { + description: "seconds are not a zero window and no limit is unbounded", + value: "0s", + limits: (None, true), + expected_zero: false, + expected_limit: None, + }, + ScalarTestCase { + description: "one day with an explicit limit", + value: "1d", + limits: (Some(3), false), + expected_zero: false, + expected_limit: Some(3), + }, + ]; + + for test_case in test_cases { + assert_eq!( + is_zero_cost_duration(test_case.value), + test_case.expected_zero, + "{}", + test_case.description + ); + assert_eq!( + cost_history_limit(test_case.limits.0, test_case.limits.1), + test_case.expected_limit, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cost_command/tests/test_types.rs b/crates/sqlbuild-core/src/cost_command/tests/test_types.rs new file mode 100644 index 0000000000..8fce355de2 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/tests/test_types.rs @@ -0,0 +1,32 @@ +use crate::cost_command::models::CostCoverage; + +pub(super) struct ValidationTestCase { + pub(super) description: &'static str, + pub(super) selector: &'static str, + pub(super) limits: (Option, bool), + pub(super) sort_field: &'static str, + pub(super) has_bounds: bool, + pub(super) expected_code: Option<&'static str>, +} + +pub(super) struct SelectionTestCase { + pub(super) description: &'static str, + pub(super) selector: &'static str, + pub(super) expected_index: Option, + pub(super) expected_code: Option<&'static str>, +} + +pub(super) struct RefreshTestCase { + pub(super) description: &'static str, + pub(super) current: CostCoverage, + pub(super) candidate: CostCoverage, + pub(super) expected_accepted: Option, +} + +pub(super) struct ScalarTestCase { + pub(super) description: &'static str, + pub(super) value: &'static str, + pub(super) limits: (Option, bool), + pub(super) expected_zero: bool, + pub(super) expected_limit: Option, +} diff --git a/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs b/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs new file mode 100644 index 0000000000..5a98a55555 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/_helpers/diagnostics.rs @@ -0,0 +1,102 @@ +//! One lint violation as a source-pointing diagnostic block. + +use crate::cli_text::models::CliStyle; +use crate::format_command::constants::{ + HELP_CONTINUATION, HELP_PREFIX, REPORT_LINE_WIDTH, TAB_CHARACTER, +}; +use crate::format_command::models::LintFinding; +use crate::text::main::cell_len::cell_len; +use crate::text::main::char_cells::char_cells; +use crate::text::main::text_wrap::text_wrap; + +fn strong(style: CliStyle, fault: bool, text: &str) -> String { + if fault { + style.error_strong(text) + } else { + style.warning_strong(text) + } +} + +fn excerpt_end(finding: &LintFinding, length: usize, start: usize) -> usize { + if finding.end_line == Some(finding.line) + && let Some(end_column) = finding.end_column + { + let end: i64 = (end_column - 1).max(start as i64 + 1); + return usize::try_from(end).unwrap_or_default().min(length); + } + if finding.end_line.is_some() && finding.end_line != Some(finding.line) { + return length; + } + (start + 1).min(length) +} + +fn source_excerpt(finding: &LintFinding, source_line: &str, style: CliStyle) -> [String; 4] { + let characters: Vec = source_line.chars().collect(); + let line_number: String = finding.line.to_string(); + let gutter: String = " ".repeat(line_number.len()); + let start: usize = usize::try_from((finding.column - 1).max(0)) + .unwrap_or_default() + .min(characters.len()); + let end: usize = excerpt_end(finding, characters.len(), start).max(start); + let padding: String = characters[..start] + .iter() + .map(|character| { + if *character == TAB_CHARACTER { + TAB_CHARACTER.to_string() + } else { + " ".repeat(char_cells(*character)) + } + }) + .collect(); + let marked: String = characters[start..end].iter().collect(); + let caret: String = "^".repeat(cell_len(&marked).max(1)); + [ + style.muted(&format!("{gutter} |")), + format!("{} {source_line}", style.muted(&format!("{line_number} |"))), + format!( + "{} {padding}{}", + style.muted(&format!("{gutter} |")), + strong(style, finding.fault, &caret) + ), + style.muted(&format!("{gutter} |")), + ] +} + +fn help_lines(remediation: &str, style: CliStyle) -> Vec { + let wrapped: Vec = text_wrap( + remediation, + REPORT_LINE_WIDTH, + (HELP_PREFIX, HELP_CONTINUATION), + ); + let Some((first, rest)) = wrapped.split_first() else { + return Vec::new(); + }; + let first: &str = first.strip_prefix(HELP_PREFIX).unwrap_or(first); + let mut lines: Vec = vec![format!(" {} {first}", style.muted("= help:"))]; + lines.extend(rest.iter().cloned()); + lines +} + +/// The diagnostic block of one violation. +pub(crate) fn violation_block(finding: &LintFinding, style: CliStyle) -> String { + let severity: &str = if finding.fault { "error" } else { "warning" }; + let label: String = format!("{severity}[{}]", finding.code); + let mut lines: Vec = vec![ + format!( + "{}: {}", + strong(style, finding.fault, &label), + finding.message + ), + style.muted(&format!( + " --> {}:{}:{}", + finding.display_path, finding.line, finding.column + )), + ]; + if let Some(source_line) = &finding.source_line { + lines.extend(source_excerpt(finding, source_line, style)); + } + if let Some(remediation) = &finding.remediation { + lines.extend(help_lines(remediation, style)); + } + lines.join("\n") +} diff --git a/crates/sqlbuild-core/src/format_command/_helpers/json_objects.rs b/crates/sqlbuild-core/src/format_command/_helpers/json_objects.rs new file mode 100644 index 0000000000..5c6010109d --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/_helpers/json_objects.rs @@ -0,0 +1,57 @@ +//! Format report entries as JSON values. + +use crate::format_command::constants::{SEVERITY_FAULT, SEVERITY_WARNING}; +use crate::format_command::models::{LintFinding, RuleFixLine}; +use crate::json::models::{JsonInteger, JsonValue}; + +pub(crate) fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn integer(value: i64) -> JsonValue { + JsonValue::Integer(JsonInteger::from(value)) +} + +fn optional_integer(value: Option) -> JsonValue { + value.map_or(JsonValue::Null, integer) +} + +pub(crate) fn count(value: usize) -> JsonValue { + JsonValue::Integer(JsonInteger::from(value as u64)) +} + +pub(crate) fn fix_object(fix: &RuleFixLine) -> JsonValue { + JsonValue::Object(vec![ + ("file".to_owned(), text(&fix.file)), + ("code".to_owned(), text(&fix.code)), + ("line".to_owned(), integer(fix.line)), + ("status".to_owned(), text(&fix.status)), + ("reason".to_owned(), text(&fix.reason)), + ]) +} + +pub(crate) fn finding_object(finding: &LintFinding) -> JsonValue { + let severity: &str = if finding.fault { + SEVERITY_FAULT + } else { + SEVERITY_WARNING + }; + JsonValue::Object(vec![ + ("code".to_owned(), text(&finding.code)), + ("column".to_owned(), integer(finding.column)), + ( + "end_column".to_owned(), + optional_integer(finding.end_column), + ), + ("end_line".to_owned(), optional_integer(finding.end_line)), + ("engine".to_owned(), text(&finding.engine)), + ("file".to_owned(), text(&finding.file)), + ("line".to_owned(), integer(finding.line)), + ("message".to_owned(), text(&finding.message)), + ( + "remediation".to_owned(), + finding.remediation.as_deref().map_or(JsonValue::Null, text), + ), + ("severity".to_owned(), text(severity)), + ]) +} diff --git a/crates/sqlbuild-core/src/format_command/_helpers/mod.rs b/crates/sqlbuild-core/src/format_command/_helpers/mod.rs new file mode 100644 index 0000000000..197bc720df --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod diagnostics; +pub(crate) mod json_objects; +pub(crate) mod selectors; +pub(crate) mod walk; diff --git a/crates/sqlbuild-core/src/format_command/_helpers/selectors.rs b/crates/sqlbuild-core/src/format_command/_helpers/selectors.rs new file mode 100644 index 0000000000..055f1b9207 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/_helpers/selectors.rs @@ -0,0 +1,57 @@ +//! Formatter path selector syntax shared by the selector entry points. + +use crate::format_command::constants::{ + LINT_DIRECTORY_NAMES, PARENT_PATH_SEGMENT, PATH_GRAPH_MARKERS, PATH_SELECTOR_PREFIX, + PATH_SEPARATOR, +}; + +/// `models/orders.sql`-style tokens are formatter paths without a `path:` prefix. +pub(crate) fn is_bare_lint_path(token: &str) -> bool { + let first_segment: &str = token.split(PATH_SEPARATOR).next().unwrap_or_default(); + token.contains(PATH_SEPARATOR) && LINT_DIRECTORY_NAMES.contains(&first_segment) +} + +/// Whether `relative_path` is one of `prefixes` or lies below one. +pub(crate) fn matches_path_prefix(relative_path: &str, prefixes: &[String]) -> bool { + prefixes.iter().any(|prefix| { + relative_path == prefix + || relative_path + .strip_prefix(prefix.as_str()) + .is_some_and(|rest| rest.starts_with(PATH_SEPARATOR)) + }) +} + +/// One prefix per selector token, or `None` when any token needs graph selection. +pub(crate) fn lint_path_prefixes(raw_selectors: &[String]) -> Option> { + let mut prefixes: Vec = Vec::new(); + for raw_selector in raw_selectors { + if raw_selector.trim().is_empty() { + return None; + } + for token in raw_selector.split_whitespace() { + let normalized: String = token.replace('\\', "/"); + if !normalized.starts_with(PATH_SELECTOR_PREFIX) && !is_bare_lint_path(&normalized) { + return None; + } + let prefix: &str = normalized + .strip_prefix(PATH_SELECTOR_PREFIX) + .unwrap_or(&normalized) + .trim_matches(PATH_SEPARATOR); + if prefix.contains(PATH_GRAPH_MARKERS) { + return None; + } + let parts: Vec<&str> = prefix + .split(PATH_SEPARATOR) + .filter(|part| !part.is_empty()) + .collect(); + let valid_root: bool = parts + .first() + .is_some_and(|root| LINT_DIRECTORY_NAMES.contains(root)); + if !valid_root || parts.contains(&PARENT_PATH_SEGMENT) { + return None; + } + prefixes.push(parts.join("/")); + } + } + Some(prefixes) +} diff --git a/crates/sqlbuild-core/src/format_command/_helpers/walk.rs b/crates/sqlbuild-core/src/format_command/_helpers/walk.rs new file mode 100644 index 0000000000..9843fc0d86 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/_helpers/walk.rs @@ -0,0 +1,48 @@ +//! `Path.rglob("*.sql")` and `Path.resolve()` over formatter-owned folders. + +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::format_command::constants::SQL_FILE_SUFFIX; + +/// `path.resolve()`: the canonical path, or the path itself when it cannot be resolved. +pub(crate) fn resolved(path: &Path) -> PathBuf { + fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +/// Every `*.sql` file below `root`, without following directory links, sorted as Python sorts paths. +pub(crate) fn sql_files_below(root: &Path) -> Vec { + let mut found: Vec = Vec::new(); + let mut pending: Vec = vec![root.to_path_buf()]; + while let Some(directory) = pending.pop() { + let Ok(entries) = fs::read_dir(&directory) else { + continue; + }; + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + let Ok(file_type) = entry.file_type() else { + continue; + }; + if file_type.is_dir() { + pending.push(path); + } else if entry + .file_name() + .to_string_lossy() + .ends_with(SQL_FILE_SUFFIX) + && path.is_file() + { + found.push(path); + } + } + } + found.sort(); + found +} + +/// `path.as_posix()`. +pub(crate) fn as_posix(path: &Path) -> String { + path.components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/") +} diff --git a/crates/sqlbuild-core/src/format_command/constants.rs b/crates/sqlbuild-core/src/format_command/constants.rs new file mode 100644 index 0000000000..856d9ee00a --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/constants.rs @@ -0,0 +1,29 @@ +//! Formatter-owned folders, selector syntax and report layout. + +/// The project folders whose SQL files the formatter owns, in collection order. +pub const LINT_DIRECTORY_NAMES: [&str; 8] = [ + "models", + "tests", + "audits", + "functions", + "enums", + "constants", + "hooks", + "schemas", +]; +pub(crate) const PATH_SELECTOR_PREFIX: &str = "path:"; +pub(crate) const PATH_SEPARATOR: char = '/'; +pub(crate) const PARENT_PATH_SEGMENT: &str = ".."; +pub(crate) const SQL_FILE_SUFFIX: &str = ".sql"; +/// Characters that make a path selector a graph selector instead. +pub(crate) const PATH_GRAPH_MARKERS: [char; 3] = ['+', '~', ',']; +/// Characters that make a model selector need canonical graph resolution. +pub(crate) const GRAPH_SELECTOR_MARKERS: [char; 6] = ['+', ':', '~', '/', '\\', ',']; +pub(crate) const UNKNOWN_ROOT_CODE: &str = "S012"; +pub(crate) const REPORT_LINE_WIDTH: usize = 100; +pub(crate) const HELP_PREFIX: &str = " = help: "; +pub(crate) const HELP_CONTINUATION: &str = " "; +pub(crate) const TAB_CHARACTER: char = '\t'; +pub(crate) const SEVERITY_FAULT: &str = "fault"; +pub(crate) const SEVERITY_WARNING: &str = "warning"; +pub(crate) const FORMAT_PATH_CODE: &str = "C113"; diff --git a/crates/sqlbuild-core/src/format_command/errors.rs b/crates/sqlbuild-core/src/format_command/errors.rs new file mode 100644 index 0000000000..98b4d759db --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/errors.rs @@ -0,0 +1,19 @@ +//! Formatter selector errors. + +use std::fmt; + +/// A selector or positional path names a path outside every formatter-owned folder. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FormatSelectorError { + pub message: String, + pub code: &'static str, + pub help: Option, +} + +impl fmt::Display for FormatSelectorError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for FormatSelectorError {} diff --git a/crates/sqlbuild-core/src/format_command/main/collect_lint_files.rs b/crates/sqlbuild-core/src/format_command/main/collect_lint_files.rs new file mode 100644 index 0000000000..2c434bd216 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/collect_lint_files.rs @@ -0,0 +1,31 @@ +//! Authored SQL files of formatter-owned folders, read without newline translation. + +use std::collections::BTreeSet; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::format_command::_helpers::walk::{resolved, sql_files_below}; +use crate::format_command::constants::LINT_DIRECTORY_NAMES; + +/// Each folder's SQL files in path order, keeping only `selected` resolved paths when given. +pub fn collect_lint_files( + project_dir: &Path, + selected: Option<&BTreeSet>, +) -> io::Result> { + let mut files: Vec<(PathBuf, String)> = Vec::new(); + for directory_name in LINT_DIRECTORY_NAMES { + let root: PathBuf = project_dir.join(directory_name); + if !root.is_dir() { + continue; + } + for file_path in sql_files_below(&root) { + if selected.is_some_and(|paths| !paths.contains(&resolved(&file_path))) { + continue; + } + let contents: String = fs::read_to_string(&file_path)?; + files.push((file_path, contents)); + } + } + Ok(files) +} diff --git a/crates/sqlbuild-core/src/format_command/main/exact_selector_names.rs b/crates/sqlbuild-core/src/format_command/main/exact_selector_names.rs new file mode 100644 index 0000000000..d30fb42863 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/exact_selector_names.rs @@ -0,0 +1,23 @@ +//! Plain model names of selectors that need no graph resolution. + +use std::collections::BTreeSet; + +use crate::format_command::constants::GRAPH_SELECTOR_MARKERS; + +/// Every selector token as a model name, or `None` when canonical graph resolution is required. +#[must_use] +pub fn exact_selector_names(raw_selectors: &[String]) -> Option> { + let mut names: BTreeSet = BTreeSet::new(); + for raw_selector in raw_selectors { + if raw_selector.trim().is_empty() { + return None; + } + for token in raw_selector.split_whitespace() { + if token.contains(GRAPH_SELECTOR_MARKERS) { + return None; + } + names.insert(token.to_owned()); + } + } + Some(names) +} diff --git a/crates/sqlbuild-core/src/format_command/main/format_path_selectors.rs b/crates/sqlbuild-core/src/format_command/main/format_path_selectors.rs new file mode 100644 index 0000000000..1cdb773642 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/format_path_selectors.rs @@ -0,0 +1,59 @@ +//! Translate `sqb format` positional paths into formatter path selectors. + +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::format_command::_helpers::walk::{as_posix, resolved}; +use crate::format_command::constants::{ + FORMAT_PATH_CODE, LINT_DIRECTORY_NAMES, PATH_SELECTOR_PREFIX, +}; +use crate::format_command::errors::FormatSelectorError; + +fn path_error(message: String, help: Option) -> FormatSelectorError { + FormatSelectorError { + message, + code: FORMAT_PATH_CODE, + help, + } +} + +/// One `path:` selector per positional path, relative to the project root. +pub fn format_path_selectors( + paths: &[String], + project_dir: Option<&Path>, + cwd: &Path, +) -> Result, FormatSelectorError> { + let project_root: PathBuf = resolved(project_dir.unwrap_or(cwd)); + let mut selectors: Vec = Vec::new(); + for raw_path in paths { + let Ok(path) = fs::canonicalize(cwd.join(raw_path)) else { + return Err(path_error( + format!("format path '{raw_path}' does not exist"), + None, + )); + }; + let Ok(relative) = path.strip_prefix(&project_root) else { + return Err(path_error( + format!( + "format path '{raw_path}' is outside the project '{}'", + project_root.display() + ), + None, + )); + }; + let relative: String = as_posix(relative); + let root: &str = relative.split('/').next().unwrap_or_default(); + if !LINT_DIRECTORY_NAMES.contains(&root) { + let folders: Vec = LINT_DIRECTORY_NAMES + .iter() + .map(|name| format!("{name}/")) + .collect(); + return Err(path_error( + format!("format path '{raw_path}' is not under a formatted folder"), + Some(format!("use a path under {}", folders.join(", "))), + )); + } + selectors.push(format!("{PATH_SELECTOR_PREFIX}{relative}")); + } + Ok(selectors) +} diff --git a/crates/sqlbuild-core/src/format_command/main/lint_sql_paths.rs b/crates/sqlbuild-core/src/format_command/main/lint_sql_paths.rs new file mode 100644 index 0000000000..94ec70a614 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/lint_sql_paths.rs @@ -0,0 +1,35 @@ +//! Resolved SQL files under formatter path prefixes. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use crate::format_command::_helpers::walk::{resolved, sql_files_below}; +use crate::format_command::constants::{LINT_DIRECTORY_NAMES, SQL_FILE_SUFFIX}; + +/// Resolved SQL files named by or below `prefixes`, or below every formatter folder when empty. +#[must_use] +pub fn lint_sql_paths(project_dir: &Path, prefixes: &[String]) -> BTreeSet { + let roots: Vec = if prefixes.is_empty() { + LINT_DIRECTORY_NAMES + .iter() + .map(|name| project_dir.join(name)) + .collect() + } else { + prefixes + .iter() + .map(|prefix| project_dir.join(prefix)) + .collect() + }; + let mut paths: BTreeSet = BTreeSet::new(); + for root in roots { + let is_sql: bool = root.extension().is_some_and(|suffix| { + format!(".{}", suffix.to_string_lossy()).to_lowercase() == SQL_FILE_SUFFIX + }); + if root.is_file() && is_sql { + paths.insert(resolved(&root)); + } else if root.is_dir() { + paths.extend(sql_files_below(&root).iter().map(|path| resolved(path))); + } + } + paths +} diff --git a/crates/sqlbuild-core/src/format_command/main/mod.rs b/crates/sqlbuild-core/src/format_command/main/mod.rs new file mode 100644 index 0000000000..93bbaa9828 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/mod.rs @@ -0,0 +1,9 @@ +pub mod collect_lint_files; +pub mod exact_selector_names; +pub mod format_path_selectors; +pub mod lint_sql_paths; +pub mod partition_lint_selectors; +pub mod render_format_status; +pub mod render_lint_json; +pub mod render_lint_text; +pub mod resolve_lint_paths; diff --git a/crates/sqlbuild-core/src/format_command/main/partition_lint_selectors.rs b/crates/sqlbuild-core/src/format_command/main/partition_lint_selectors.rs new file mode 100644 index 0000000000..03f4075221 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/partition_lint_selectors.rs @@ -0,0 +1,52 @@ +//! Separate formatter-root paths from model selectors. + +use crate::format_command::_helpers::selectors::lint_path_prefixes; +use crate::format_command::constants::{ + LINT_DIRECTORY_NAMES, PATH_SELECTOR_PREFIX, PATH_SEPARATOR, UNKNOWN_ROOT_CODE, +}; +use crate::format_command::errors::FormatSelectorError; + +fn reject_unknown_root(token: &str) -> Result<(), FormatSelectorError> { + let normalized: String = token.replace('\\', "/"); + let Some(path) = normalized.strip_prefix(PATH_SELECTOR_PREFIX) else { + return Ok(()); + }; + let root: &str = path + .trim_matches(PATH_SEPARATOR) + .split(PATH_SEPARATOR) + .next() + .unwrap_or_default(); + if LINT_DIRECTORY_NAMES.contains(&root) { + return Ok(()); + } + let roots: Vec = LINT_DIRECTORY_NAMES + .iter() + .map(|name| format!("'{name}/'")) + .collect(); + Err(FormatSelectorError { + message: format!( + "format path selectors must start with one of: {}", + roots.join(", ") + ), + code: UNKNOWN_ROOT_CODE, + help: None, + }) +} + +/// The path prefixes and the model selector tokens, rejecting `path:` outside formatter roots. +pub fn partition_lint_selectors( + raw_selectors: &[String], +) -> Result<(Vec, Vec), FormatSelectorError> { + let mut path_tokens: Vec = Vec::new(); + let mut model_tokens: Vec = Vec::new(); + for raw_selector in raw_selectors { + for token in raw_selector.split_whitespace() { + reject_unknown_root(token)?; + match lint_path_prefixes(&[token.to_owned()]) { + Some(prefixes) => path_tokens.extend(prefixes), + None => model_tokens.push(token.to_owned()), + } + } + } + Ok((path_tokens, model_tokens)) +} diff --git a/crates/sqlbuild-core/src/format_command/main/render_format_status.rs b/crates/sqlbuild-core/src/format_command/main/render_format_status.rs new file mode 100644 index 0000000000..47328729a6 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/render_format_status.rs @@ -0,0 +1,23 @@ +//! The completion text of the transient `Formatting SQL` status line. + +use crate::format_command::models::LintReport; + +/// Whether the run failed, and its status message after `elapsed` seconds. +#[must_use] +pub fn render_format_status(report: &LintReport, elapsed: f64) -> (bool, String) { + let faults: usize = report.fault_count(); + if faults > 0 { + return ( + true, + format!("Formatting SQL ERROR ({elapsed:.2}s; {faults} faults)"), + ); + } + ( + false, + format!( + "Formatting SQL OK ({elapsed:.2}s; {} files checked, {} changed)", + report.files_checked, + report.formatted_files.len() + ), + ) +} diff --git a/crates/sqlbuild-core/src/format_command/main/render_lint_json.rs b/crates/sqlbuild-core/src/format_command/main/render_lint_json.rs new file mode 100644 index 0000000000..defc52829e --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/render_lint_json.rs @@ -0,0 +1,42 @@ +//! A format run as stable machine-readable JSON. + +use crate::format_command::_helpers::json_objects::{count, finding_object, fix_object, text}; +use crate::format_command::models::LintReport; +use crate::json::errors::JsonEmitError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +/// The report with sorted keys and compact separators, followed by a newline. +pub fn render_lint_json(report: &LintReport) -> Result { + let document: JsonValue = JsonValue::Object(vec![ + ("faults".to_owned(), count(report.fault_count())), + ("files_checked".to_owned(), count(report.files_checked)), + ( + "formatted_files".to_owned(), + JsonValue::Array( + report + .formatted_files + .iter() + .map(|path| text(path)) + .collect(), + ), + ), + ( + "rule_fixes".to_owned(), + JsonValue::Array(report.rule_fixes.iter().map(fix_object).collect()), + ), + ( + "violations".to_owned(), + JsonValue::Array(report.findings.iter().map(finding_object).collect()), + ), + ("warnings".to_owned(), count(report.warning_count())), + ]); + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }; + Ok(format!( + "{}\n", + dumps(&document, &JsonDialect::Stdlib(options))? + )) +} diff --git a/crates/sqlbuild-core/src/format_command/main/render_lint_text.rs b/crates/sqlbuild-core/src/format_command/main/render_lint_text.rs new file mode 100644 index 0000000000..de6ba80fbd --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/render_lint_text.rs @@ -0,0 +1,54 @@ +//! A format run's human report. + +use crate::cli_text::models::CliStyle; +use crate::format_command::_helpers::diagnostics::violation_block; +use crate::format_command::models::LintReport; + +/// Changed files under `heading`, diagnostics, Rule fixes and the styled summary line. +#[must_use] +pub fn render_lint_text(report: &LintReport, style: CliStyle, heading: &str) -> String { + let mut output: String = String::new(); + if !report.formatted_files.is_empty() { + output.push_str(heading); + output.push('\n'); + for path in &report.formatted_files { + output.push_str(&format!(" {path}\n")); + } + output.push('\n'); + } + if !report.findings.is_empty() { + let blocks: Vec = report + .findings + .iter() + .map(|finding| violation_block(finding, style)) + .collect(); + output.push_str(&blocks.join("\n\n")); + output.push_str("\n\n"); + } + for fix in &report.rule_fixes { + output.push_str(&format!( + "{} {} {}:{}: {}\n", + fix.status.to_uppercase(), + fix.code, + fix.file, + fix.line, + fix.reason + )); + } + let faults: usize = report.fault_count(); + let warnings: usize = report.warning_count(); + let summary: String = format!( + "Completed. FAULT={faults} WARN={warnings} FILES={}", + report.files_checked + ); + let styled: String = if faults > 0 { + style.error_strong(&summary) + } else if warnings > 0 { + style.warning_strong(&summary) + } else { + style.success_strong(&summary) + }; + output.push_str(&styled); + output.push('\n'); + output +} diff --git a/crates/sqlbuild-core/src/format_command/main/resolve_lint_paths.rs b/crates/sqlbuild-core/src/format_command/main/resolve_lint_paths.rs new file mode 100644 index 0000000000..1c2235c5ad --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/main/resolve_lint_paths.rs @@ -0,0 +1,30 @@ +//! Resolve simple path selectors across every formatter-owned SQL folder. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use crate::format_command::_helpers::selectors::lint_path_prefixes; +use crate::format_command::_helpers::selectors::matches_path_prefix; +use crate::format_command::_helpers::walk::{as_posix, resolved}; +use crate::format_command::main::lint_sql_paths::lint_sql_paths; + +/// The selected files, or `None` when a selector needs graph resolution. +#[must_use] +pub fn resolve_lint_paths( + project_dir: &Path, + (select, exclude): (&[String], &[String]), +) -> Option> { + let selected_prefixes: Vec = lint_path_prefixes(select)?; + let excluded_prefixes: Vec = lint_path_prefixes(exclude)?; + let root: PathBuf = resolved(project_dir); + Some( + lint_sql_paths(project_dir, &selected_prefixes) + .into_iter() + .filter(|path| { + path.strip_prefix(&root).is_ok_and(|relative| { + !matches_path_prefix(&as_posix(relative), &excluded_prefixes) + }) + }) + .collect(), + ) +} diff --git a/crates/sqlbuild-core/src/format_command/mod.rs b/crates/sqlbuild-core/src/format_command/mod.rs new file mode 100644 index 0000000000..c39fa447df --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/mod.rs @@ -0,0 +1,10 @@ +//! `sqb format`: formatter path selectors, authored SQL file collection and result rendering. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/format_command/models.rs b/crates/sqlbuild-core/src/format_command/models.rs new file mode 100644 index 0000000000..9ad8954388 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/models.rs @@ -0,0 +1,55 @@ +//! A format run's outcome as its text and JSON reports show it. + +/// One lint violation with the authored source line it points at. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LintFinding { + /// `str(file_path)`. + pub file: String, + /// The file relative to the project root as a POSIX path, or the whole POSIX path outside it. + pub display_path: String, + pub line: i64, + pub column: i64, + pub end_line: Option, + pub end_column: Option, + pub code: String, + pub message: String, + /// Fault severity; every other violation is a warning. + pub fault: bool, + pub engine: String, + pub remediation: Option, + /// The authored text of `line`, when the run holds the file and the line exists. + pub source_line: Option, +} + +/// One applied, refused or unavailable Rule fix. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RuleFixLine { + pub file: String, + pub code: String, + pub line: i64, + pub status: String, + pub reason: String, +} + +/// Everything a format run reports. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LintReport { + pub files_checked: usize, + pub formatted_files: Vec, + pub rule_fixes: Vec, + pub findings: Vec, +} + +impl LintReport { + /// How many violations are faults. + #[must_use] + pub fn fault_count(&self) -> usize { + self.findings.iter().filter(|finding| finding.fault).count() + } + + /// How many violations are warnings. + #[must_use] + pub fn warning_count(&self) -> usize { + self.findings.len() - self.fault_count() + } +} diff --git a/crates/sqlbuild-core/src/format_command/tests.rs b/crates/sqlbuild-core/src/format_command/tests.rs new file mode 100644 index 0000000000..545bf17288 --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_format_command.rs"] +mod format_command; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/format_command/tests/helpers.rs b/crates/sqlbuild-core/src/format_command/tests/helpers.rs new file mode 100644 index 0000000000..cbcf23ea6f --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/tests/helpers.rs @@ -0,0 +1,47 @@ +use crate::format_command::models::{LintFinding, LintReport, RuleFixLine}; + +/// Owned copies of string literals. +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +/// A report with one excerpted fault, one location-only warning and one Rule fix. +pub(super) fn sample_report() -> LintReport { + LintReport { + files_checked: 3, + formatted_files: texts(&["/p/models/orders.sql"]), + rule_fixes: vec![RuleFixLine { + file: "/p/models/orders.sql".to_owned(), + code: "SQBRSQL002".to_owned(), + line: 4, + status: "applied".to_owned(), + reason: "ok".to_owned(), + }], + findings: vec![ + LintFinding { + file: "/p/models/orders.sql".to_owned(), + display_path: "models/orders.sql".to_owned(), + line: 2, + column: 8, + end_line: Some(2), + end_column: Some(15), + code: "SQBRSQL004".to_owned(), + message: "Avoid LIMIT".to_owned(), + fault: true, + engine: "native".to_owned(), + remediation: Some("Remove the LIMIT clause from the model body because the model must return every row it is expected to produce downstream.".to_owned()), + source_line: Some("SELECT\t1 LIMIT 5".to_owned()), + }, + LintFinding { + file: "/other/x.sql".to_owned(), + display_path: "/other/x.sql".to_owned(), + line: 1, + column: 1, + code: "description-present".to_owned(), + message: "Missing".to_owned(), + engine: "sqlbuild".to_owned(), + ..LintFinding::default() + }, + ], + } +} diff --git a/crates/sqlbuild-core/src/format_command/tests/test_format_command.rs b/crates/sqlbuild-core/src/format_command/tests/test_format_command.rs new file mode 100644 index 0000000000..8c6d97144b --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/tests/test_format_command.rs @@ -0,0 +1,89 @@ +use std::collections::BTreeSet; + +use crate::cli_text::models::CliStyle; +use crate::format_command::_helpers::selectors::lint_path_prefixes; +use crate::format_command::main::exact_selector_names::exact_selector_names; +use crate::format_command::main::partition_lint_selectors::partition_lint_selectors; +use crate::format_command::main::render_lint_json::render_lint_json; +use crate::format_command::main::render_lint_text::render_lint_text; +use crate::format_command::tests::helpers::{sample_report, texts}; +use crate::format_command::tests::test_types::{ReportTestCase, SelectorTestCase}; + +#[test] +fn given_format_report_when_rendering_then_text_and_json_match_python() { + let test_cases = [ReportTestCase { + description: "excerpt, wrapped help, location-only warning and a Rule fix", + report: sample_report(), + expected_text: "Formatted files:\n /p/models/orders.sql\n\nerror[SQBRSQL004]: Avoid LIMIT\n --> models/orders.sql:2:8\n |\n2 | SELECT\t1 LIMIT 5\n | \t^^^^^^^\n |\n = help: Remove the LIMIT clause from the model body because the model must return every row it is\n expected to produce downstream.\n\nwarning[description-present]: Missing\n --> /other/x.sql:1:1\n\nAPPLIED SQBRSQL002 /p/models/orders.sql:4: ok\nCompleted. FAULT=1 WARN=1 FILES=3\n", + expected_json: "{\"faults\":1,\"files_checked\":3,\"formatted_files\":[\"/p/models/orders.sql\"],\"rule_fixes\":[{\"code\":\"SQBRSQL002\",\"file\":\"/p/models/orders.sql\",\"line\":4,\"reason\":\"ok\",\"status\":\"applied\"}],\"violations\":[{\"code\":\"SQBRSQL004\",\"column\":8,\"end_column\":15,\"end_line\":2,\"engine\":\"native\",\"file\":\"/p/models/orders.sql\",\"line\":2,\"message\":\"Avoid LIMIT\",\"remediation\":\"Remove the LIMIT clause from the model body because the model must return every row it is expected to produce downstream.\",\"severity\":\"fault\"},{\"code\":\"description-present\",\"column\":1,\"end_column\":null,\"end_line\":null,\"engine\":\"sqlbuild\",\"file\":\"/other/x.sql\",\"line\":1,\"message\":\"Missing\",\"remediation\":null,\"severity\":\"warning\"}],\"warnings\":1}\n", + }]; + + for test_case in test_cases { + let style: CliStyle = CliStyle { use_color: false }; + assert_eq!( + render_lint_text(&test_case.report, style, "Formatted files:"), + test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!( + render_lint_json(&test_case.report).ok().as_deref(), + Some(test_case.expected_json), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_selectors_when_classifying_then_paths_and_models_match_python() { + let test_cases = [ + SelectorTestCase { + description: "path selectors and backslashes become prefixes", + selectors: texts(&["path:models/staging/ tests\\unit"]), + expected_prefixes: Some(texts(&["models/staging", "tests/unit"])), + expected_partition: (texts(&["models/staging", "tests/unit"]), Vec::new()), + expected_names: None, + }, + SelectorTestCase { + description: "model names are exact names, not paths", + selectors: texts(&["orders customers"]), + expected_prefixes: None, + expected_partition: (Vec::new(), texts(&["orders", "customers"])), + expected_names: Some(texts(&["customers", "orders"])), + }, + SelectorTestCase { + description: "graph operators keep a token a model selector", + selectors: texts(&["path:models/staging +orders", "models/a.sql"]), + expected_prefixes: None, + expected_partition: ( + texts(&["models/staging", "models/a.sql"]), + texts(&["+orders"]), + ), + expected_names: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + lint_path_prefixes(&test_case.selectors), + test_case.expected_prefixes, + "{}", + test_case.description + ); + assert_eq!( + partition_lint_selectors(&test_case.selectors).ok(), + Some(test_case.expected_partition), + "{}", + test_case.description + ); + assert_eq!( + exact_selector_names(&test_case.selectors), + test_case + .expected_names + .map(|names| names.into_iter().collect::>()), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/format_command/tests/test_types.rs b/crates/sqlbuild-core/src/format_command/tests/test_types.rs new file mode 100644 index 0000000000..87af7ed59a --- /dev/null +++ b/crates/sqlbuild-core/src/format_command/tests/test_types.rs @@ -0,0 +1,16 @@ +use crate::format_command::models::LintReport; + +pub(super) struct ReportTestCase { + pub(super) description: &'static str, + pub(super) report: LintReport, + pub(super) expected_text: &'static str, + pub(super) expected_json: &'static str, +} + +pub(super) struct SelectorTestCase { + pub(super) description: &'static str, + pub(super) selectors: Vec, + pub(super) expected_prefixes: Option>, + pub(super) expected_partition: (Vec, Vec), + pub(super) expected_names: Option>, +} diff --git a/crates/sqlbuild-core/src/init_command/constants.rs b/crates/sqlbuild-core/src/init_command/constants.rs new file mode 100644 index 0000000000..efd035371f --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/constants.rs @@ -0,0 +1,36 @@ +//! The directories a blank project starts with. + +/// Each resource directory, in creation order. +pub const SCAFFOLD_DIRECTORIES: [&str; 15] = [ + "models/staging", + "models/marts", + "schemas", + "sources", + "seeds", + "python", + "hooks/sql", + "hooks/python", + "tests/unit", + "tests/scenarios", + "functions/sql", + "functions/python", + "macros", + "audits/generic", + "audits/singular", +]; + +/// The project configuration file name. +pub const PROJECT_FILE: &str = "sqlbuild_project.toml"; + +/// The placeholder file kept in otherwise empty directories. +pub const GITKEEP_FILE: &str = ".gitkeep"; + +/// The numbered next steps before the compile and build commands. +pub const NEXT_STEPS: [&str; 6] = [ + " 1. Add sources to sources/", + " 2. Add seeds to seeds/", + " 3. Add Python loaders, tasks, assets, checks, and factories to python/", + " 4. Add SQL hooks to hooks/sql/, Python hooks to hooks/python/, functions to functions/, or macros to macros/", + " 5. Add models to models/staging/ and models/marts/", + " 6. Add tests to tests/unit/ or tests/scenarios/, reusable audits to audits/generic/, or standalone audits to audits/singular/", +]; diff --git a/crates/sqlbuild-core/src/init_command/main/init_project_name.rs b/crates/sqlbuild-core/src/init_command/main/init_project_name.rs new file mode 100644 index 0000000000..da45a6947a --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/main/init_project_name.rs @@ -0,0 +1,7 @@ +//! The project name derived from its directory. + +/// The directory name with dashes and spaces as underscores, lowercased. +#[must_use] +pub fn init_project_name(directory_name: &str) -> String { + directory_name.replace(['-', ' '], "_").to_lowercase() +} diff --git a/crates/sqlbuild-core/src/init_command/main/init_project_toml.rs b/crates/sqlbuild-core/src/init_command/main/init_project_toml.rs new file mode 100644 index 0000000000..655ee1e1b7 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/main/init_project_toml.rs @@ -0,0 +1,14 @@ +//! The configuration of a blank DuckDB project. + +/// `sqlbuild_project.toml` with dev and prod targets on one local connection. +#[must_use] +pub fn init_project_toml(project_name: &str) -> String { + format!( + "name = \"{project_name}\"\nadapter = \"duckdb\"\ndefault_target = \"dev\"\n\n\ + [connections.developer]\ndatabase = \"{project_name}.duckdb\"\n\n\ + [settings]\ndefault_audit_severity = \"warn\"\n\n\ + [defaults]\nmaterialized = \"table\"\n\n\ + [targets.prod]\nconnection = \"developer\"\nschema = \"prod\"\n\n\ + [targets.dev]\nconnection = \"developer\"\nschema = \"dev\"\n" + ) +} diff --git a/crates/sqlbuild-core/src/init_command/main/mod.rs b/crates/sqlbuild-core/src/init_command/main/mod.rs new file mode 100644 index 0000000000..b8eae0db00 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/main/mod.rs @@ -0,0 +1,3 @@ +pub mod init_project_name; +pub mod init_project_toml; +pub mod render_init_summary; diff --git a/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs b/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs new file mode 100644 index 0000000000..300bf21589 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/main/render_init_summary.rs @@ -0,0 +1,22 @@ +//! The summary printed after a project is created. + +use crate::cli_text::models::{CliDocument, CliStyle}; +use crate::init_command::constants::NEXT_STEPS; + +/// The created header, project and config fields, and the numbered next steps. +#[must_use] +pub fn render_init_summary(project_name: &str, style: CliStyle) -> String { + let mut document: CliDocument = CliDocument::new(style); + document.header("SQLBuild project created", None); + document.lines.push(String::new()); + document.field("Project", (project_name, " ")); + document.field("Config", ("sqlbuild_project.toml", " ")); + document.lines.push(String::new()); + document.section("Next steps"); + document + .lines + .extend(NEXT_STEPS.iter().map(|step| (*step).to_owned())); + document.command_line(" 7. ", ("sqb compile", true)); + document.command_line(" 8. ", ("sqb build", true)); + document.render(true) +} diff --git a/crates/sqlbuild-core/src/init_command/mod.rs b/crates/sqlbuild-core/src/init_command/mod.rs new file mode 100644 index 0000000000..d9f45c6513 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/mod.rs @@ -0,0 +1,7 @@ +//! `sqb init`: the blank project layout, its configuration, and the created summary. + +pub mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/init_command/tests.rs b/crates/sqlbuild-core/src/init_command/tests.rs new file mode 100644 index 0000000000..0073ff9014 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_init_command.rs"] +mod init_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/init_command/tests/test_init_command.rs b/crates/sqlbuild-core/src/init_command/tests/test_init_command.rs new file mode 100644 index 0000000000..fb7d093669 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/tests/test_init_command.rs @@ -0,0 +1,54 @@ +use crate::cli_text::models::CliStyle; +use crate::init_command::main::init_project_name::init_project_name; +use crate::init_command::main::init_project_toml::init_project_toml; +use crate::init_command::main::render_init_summary::render_init_summary; +use crate::init_command::tests::test_types::{NameTestCase, SummaryTestCase}; + +#[test] +fn given_directory_name_when_naming_project_then_normalizes_separators_and_case() { + let test_cases = [ + NameTestCase { + description: "dashes and spaces become underscores", + directory_name: "Demo-Project Name", + expected_name: "demo_project_name", + }, + NameTestCase { + description: "non-ASCII letters lowercase", + directory_name: "UPPER_Ünïcode", + expected_name: "upper_ünïcode", + }, + ]; + + for test_case in test_cases { + assert_eq!( + init_project_name(test_case.directory_name), + test_case.expected_name, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_project_name_when_scaffolding_then_writes_config_and_summary() { + let test_cases = [SummaryTestCase { + description: "a plain project", + project_name: "demo", + expected_toml_start: "name = \"demo\"\nadapter = \"duckdb\"\ndefault_target = \"dev\"\n\n[connections.developer]\ndatabase = \"demo.duckdb\"\n", + expected_summary_start: "SQLBuild project created\n\n Project: demo\n Config: sqlbuild_project.toml\n\nNext steps:\n 1. Add sources to sources/\n", + }]; + + for test_case in test_cases { + assert!( + init_project_toml(test_case.project_name).starts_with(test_case.expected_toml_start), + "{}", + test_case.description + ); + assert!( + render_init_summary(test_case.project_name, CliStyle { use_color: false }) + .starts_with(test_case.expected_summary_start), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/init_command/tests/test_types.rs b/crates/sqlbuild-core/src/init_command/tests/test_types.rs new file mode 100644 index 0000000000..d564c403ac --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/tests/test_types.rs @@ -0,0 +1,12 @@ +pub(super) struct NameTestCase { + pub(super) description: &'static str, + pub(super) directory_name: &'static str, + pub(super) expected_name: &'static str, +} + +pub(super) struct SummaryTestCase { + pub(super) description: &'static str, + pub(super) project_name: &'static str, + pub(super) expected_toml_start: &'static str, + pub(super) expected_summary_start: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 6b08424e07..8e0852bd87 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -2,13 +2,19 @@ #![forbid(unsafe_code)] +pub mod audit_command; +pub mod build_command; pub mod check_command; pub mod cli_text; pub mod clone_text; pub mod constants; +pub mod contract_command; +pub mod cost_command; pub mod cost_report; pub mod debug_report; +pub mod format_command; pub mod freshness_text; +pub mod init_command; pub mod integration_results; pub mod janitor_text; pub mod json; @@ -17,7 +23,9 @@ pub mod models; pub mod panics; pub mod posix_path; pub mod progress; +pub mod query_command; pub mod redaction; +pub mod scenario_command; pub mod seed_command; pub mod terminal_events; pub mod test_command; diff --git a/crates/sqlbuild-core/src/query_command/_helpers/footer.rs b/crates/sqlbuild-core/src/query_command/_helpers/footer.rs new file mode 100644 index 0000000000..e145c57f3f --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/_helpers/footer.rs @@ -0,0 +1,20 @@ +//! The row count and truncation lines under long and table output. + +use crate::query_command::models::QueryDisplay; + +/// A blank line, ` row(s)`, and the truncation hint when the limit cut rows off. +pub(crate) fn footer(display: &QueryDisplay) -> Vec { + let count: usize = display.rows.len(); + let mut lines: Vec = vec![ + String::new(), + format!("{count} {}", if count == 1 { "row" } else { "rows" }), + ]; + if display.truncated { + lines.push(display.limit.map_or_else(String::new, |limit| { + format!( + "Showing {limit} rows. Use --limit to show more or --no-limit to disable the limit." + ) + })); + } + lines +} diff --git a/crates/sqlbuild-core/src/query_command/_helpers/mod.rs b/crates/sqlbuild-core/src/query_command/_helpers/mod.rs new file mode 100644 index 0000000000..c8f10e5269 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod footer; diff --git a/crates/sqlbuild-core/src/query_command/constants.rs b/crates/sqlbuild-core/src/query_command/constants.rs new file mode 100644 index 0000000000..7d9aac4965 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/constants.rs @@ -0,0 +1,6 @@ +//! Query output layout. + +pub(crate) const RECORD_RULE_WIDTH: usize = 40; +pub(crate) const TABLE_SEPARATOR: &str = " | "; +pub(crate) const CSV_DELIMITER: char = ','; +pub(crate) const CSV_QUOTE: char = '"'; diff --git a/crates/sqlbuild-core/src/query_command/main/mod.rs b/crates/sqlbuild-core/src/query_command/main/mod.rs new file mode 100644 index 0000000000..7496e3a434 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/main/mod.rs @@ -0,0 +1,5 @@ +pub mod render_dag_summary; +pub mod render_query_csv; +pub mod render_query_json; +pub mod render_query_long; +pub mod render_query_table; diff --git a/crates/sqlbuild-core/src/query_command/main/render_dag_summary.rs b/crates/sqlbuild-core/src/query_command/main/render_dag_summary.rs new file mode 100644 index 0000000000..ccc4a13e78 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/main/render_dag_summary.rs @@ -0,0 +1,7 @@ +//! The one-line `sqb dag` summary. + +/// `DAG ready ( nodes, edges, checks)` and a newline. +#[must_use] +pub fn render_dag_summary((nodes, edges, checks): (usize, usize, usize)) -> String { + format!("DAG ready ({nodes} nodes, {edges} edges, {checks} checks)\n") +} diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs b/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs new file mode 100644 index 0000000000..1247dfa793 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs @@ -0,0 +1,34 @@ +//! Query rows as CSV, as Python's default `csv.writer` writes them. + +use crate::query_command::constants::{CSV_DELIMITER, CSV_QUOTE}; + +fn field(value: &str) -> String { + if value + .chars() + .any(|character| matches!(character, CSV_DELIMITER | CSV_QUOTE | '\n' | '\r')) + { + format!("\"{}\"", value.replace('"', "\"\"")) + } else { + value.to_owned() + } +} + +fn record(values: &[String]) -> String { + if let [only] = values + && only.is_empty() + { + return "\"\"\n".to_owned(); + } + let fields: Vec = values.iter().map(|value| field(value)).collect(); + format!("{}\n", fields.join(",")) +} + +/// The header record then one record per row; fields quoted only when they must be. +#[must_use] +pub fn render_query_csv(columns: &[String], rows: &[Vec]) -> String { + let mut text: String = record(columns); + for row in rows { + text.push_str(&record(row)); + } + text +} diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_json.rs b/crates/sqlbuild-core/src/query_command/main/render_query_json.rs new file mode 100644 index 0000000000..e2d6e0b02a --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/main/render_query_json.rs @@ -0,0 +1,32 @@ +//! Query rows as a JSON array of objects. + +use crate::json::errors::JsonEmitError; +use crate::json::main::dumps::dumps; +use crate::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +fn row_object(columns: &[String], row: Vec) -> JsonValue { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (column, value) in columns.iter().zip(row) { + match entries.iter_mut().find(|(name, _)| name == column) { + Some(entry) => entry.1 = value, + None => entries.push((column.clone(), value)), + } + } + JsonValue::Object(entries) +} + +/// One object per row; a repeated column keeps its first position and its last value. +pub fn render_query_json( + columns: &[String], + rows: Vec>, +) -> Result { + let objects: Vec = rows + .into_iter() + .map(|row| row_object(columns, row)) + .collect(); + let rendered: String = dumps( + &JsonValue::Array(objects), + &JsonDialect::Stdlib(StdlibJsonOptions::new(None)), + )?; + Ok(format!("{rendered}\n")) +} diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_long.rs b/crates/sqlbuild-core/src/query_command/main/render_query_long.rs new file mode 100644 index 0000000000..d56e24dab7 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/main/render_query_long.rs @@ -0,0 +1,30 @@ +//! Query rows as one expanded record per row. + +use crate::query_command::_helpers::footer::footer; +use crate::query_command::constants::RECORD_RULE_WIDTH; +use crate::query_command::models::QueryDisplay; + +/// `-[ RECORD n ]---+` then `column | value` lines per row, then the row count. +#[must_use] +pub fn render_query_long(display: &QueryDisplay) -> String { + let width: usize = display + .columns + .iter() + .map(|column| column.chars().count()) + .max() + .unwrap_or(0); + let mut lines: Vec = Vec::new(); + for (index, row) in display.rows.iter().enumerate() { + if !lines.is_empty() { + lines.push(String::new()); + } + let header: String = format!("-[ RECORD {} ]", index + 1); + let rule: usize = RECORD_RULE_WIDTH.saturating_sub(header.chars().count()); + lines.push(format!("{header}{}+", "-".repeat(rule))); + for (column, value) in display.columns.iter().zip(row) { + lines.push(format!("{column: String { + format!("{value: String { + let mut widths: Vec = display + .columns + .iter() + .map(|column| column.chars().count()) + .collect(); + for row in &display.rows { + for (index, value) in row.iter().enumerate() { + if let Some(width) = widths.get_mut(index) { + *width = (*width).max(value.chars().count()); + } + } + } + let width_at = |index: usize| widths.get(index).copied().unwrap_or(0); + let mut lines: Vec = vec![ + display + .columns + .iter() + .enumerate() + .map(|(index, column)| padded(column, width_at(index))) + .collect::>() + .join(TABLE_SEPARATOR), + widths + .iter() + .map(|width| "-".repeat(*width)) + .collect::>() + .join(TABLE_SEPARATOR), + ]; + for row in &display.rows { + let line: String = row + .iter() + .enumerate() + .map(|(index, value)| padded(value, width_at(index))) + .collect::>() + .join(TABLE_SEPARATOR); + lines.push(line.trim_end_matches(python_isspace).to_owned()); + } + lines.extend(footer(display)); + format!("{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-core/src/query_command/mod.rs b/crates/sqlbuild-core/src/query_command/mod.rs new file mode 100644 index 0000000000..7115fbb5d8 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/mod.rs @@ -0,0 +1,9 @@ +//! `sqb query` result rendering and `sqb dag` summary text. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/query_command/models.rs b/crates/sqlbuild-core/src/query_command/models.rs new file mode 100644 index 0000000000..724b1349f8 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/models.rs @@ -0,0 +1,10 @@ +//! A query result as its text formats display it. + +/// Column names, each row's cells as display text, and whether the limit cut rows off. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct QueryDisplay { + pub columns: Vec, + pub rows: Vec>, + pub truncated: bool, + pub limit: Option, +} diff --git a/crates/sqlbuild-core/src/query_command/tests.rs b/crates/sqlbuild-core/src/query_command/tests.rs new file mode 100644 index 0000000000..d3561b270d --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_query_command.rs"] +mod query_command; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/query_command/tests/helpers.rs b/crates/sqlbuild-core/src/query_command/tests/helpers.rs new file mode 100644 index 0000000000..e611510a34 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/tests/helpers.rs @@ -0,0 +1,4 @@ +/// Owned copies of string literals. +pub(super) fn texts(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} diff --git a/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs b/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs new file mode 100644 index 0000000000..0cdd792d93 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs @@ -0,0 +1,119 @@ +use crate::json::models::JsonValue; +use crate::query_command::main::render_dag_summary::render_dag_summary; +use crate::query_command::main::render_query_csv::render_query_csv; +use crate::query_command::main::render_query_json::render_query_json; +use crate::query_command::main::render_query_long::render_query_long; +use crate::query_command::main::render_query_table::render_query_table; +use crate::query_command::models::QueryDisplay; +use crate::query_command::tests::helpers::texts; +use crate::query_command::tests::test_types::{ + CsvTestCase, DagTestCase, DisplayTestCase, JsonTestCase, +}; + +#[test] +fn given_rows_when_rendering_text_then_aligns_and_counts() { + let test_cases = [ + DisplayTestCase { + description: "two rows with unicode and a truncation hint", + display: QueryDisplay { + columns: texts(&["id", "name"]), + rows: vec![texts(&["1", "héllo"]), texts(&["22", ""])], + truncated: true, + limit: Some(2), + }, + expected_long: "-[ RECORD 1 ]---------------------------+\nid | 1\nname | héllo\n\n-[ RECORD 2 ]---------------------------+\nid | 22\nname | \n\n2 rows\nShowing 2 rows. Use --limit to show more or --no-limit to disable the limit.\n", + expected_table: "id | name \n-- | -----\n1 | héllo\n22 |\n\n2 rows\nShowing 2 rows. Use --limit to show more or --no-limit to disable the limit.\n", + }, + DisplayTestCase { + description: "no rows and an unlimited truncation leaves an empty line", + display: QueryDisplay { + columns: texts(&["x"]), + rows: Vec::new(), + truncated: true, + limit: None, + }, + expected_long: "\n0 rows\n\n", + expected_table: "x\n-\n\n0 rows\n\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_query_long(&test_case.display), + test_case.expected_long, + "{}", + test_case.description + ); + assert_eq!( + render_query_table(&test_case.display), + test_case.expected_table, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rows_when_rendering_csv_then_quotes_minimally() { + let test_cases = [CsvTestCase { + description: "delimiters, quotes, newlines and a lone empty field", + columns: texts(&["a", "b"]), + rows: vec![ + texts(&["x,y", "q\"z"]), + texts(&["l\nm", ""]), + texts(&[""]), + Vec::new(), + ], + expected_csv: "a,b\n\"x,y\",\"q\"\"z\"\n\"l\nm\",\n\"\"\n\n", + }]; + + for test_case in test_cases { + assert_eq!( + render_query_csv(&test_case.columns, &test_case.rows), + test_case.expected_csv, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_rows_when_rendering_json_then_keeps_first_position_and_last_value() { + let test_cases = [JsonTestCase { + description: "a repeated column and non-ASCII text", + columns: texts(&["a", "b", "a"]), + rows: vec![vec![ + JsonValue::Bool(true), + JsonValue::String("é".to_owned()), + JsonValue::Null, + ]], + expected_json: "[{\"a\": null, \"b\": \"\\u00e9\"}]\n", + }]; + + for test_case in test_cases { + assert_eq!( + render_query_json(&test_case.columns, test_case.rows).ok(), + Some(test_case.expected_json.to_owned()), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_graph_counts_when_summarizing_dag_then_reports_them() { + let test_cases = [DagTestCase { + description: "nodes, edges and checks", + counts: (20, 24, 36), + expected_text: "DAG ready (20 nodes, 24 edges, 36 checks)\n", + }]; + + for test_case in test_cases { + assert_eq!( + render_dag_summary(test_case.counts), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/query_command/tests/test_types.rs b/crates/sqlbuild-core/src/query_command/tests/test_types.rs new file mode 100644 index 0000000000..99d7d0b43a --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/tests/test_types.rs @@ -0,0 +1,29 @@ +use crate::json::models::JsonValue; +use crate::query_command::models::QueryDisplay; + +pub(super) struct DisplayTestCase { + pub(super) description: &'static str, + pub(super) display: QueryDisplay, + pub(super) expected_long: &'static str, + pub(super) expected_table: &'static str, +} + +pub(super) struct CsvTestCase { + pub(super) description: &'static str, + pub(super) columns: Vec, + pub(super) rows: Vec>, + pub(super) expected_csv: &'static str, +} + +pub(super) struct JsonTestCase { + pub(super) description: &'static str, + pub(super) columns: Vec, + pub(super) rows: Vec>, + pub(super) expected_json: &'static str, +} + +pub(super) struct DagTestCase { + pub(super) description: &'static str, + pub(super) counts: (usize, usize, usize), + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/scenario_command/_helpers/expectations.rs b/crates/sqlbuild-core/src/scenario_command/_helpers/expectations.rs new file mode 100644 index 0000000000..b35248300c --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/_helpers/expectations.rs @@ -0,0 +1,89 @@ +//! The expectation rows and failure details under a scenario result. + +use crate::cli_text::models::CliStyle; +use crate::scenario_command::_helpers::rows::{result_error, rows_label}; +use crate::scenario_command::constants::{ + EXPECTATION_ERROR_INDENT, EXPECTATION_LABEL_WIDTH, EXPECTATION_NAME_WIDTH, FAIL_STATUS, + PASS_STATUS, +}; +use crate::scenario_command::models::{CodedText, ScenarioResultRow}; + +/// One `expect` row. +fn expectation_line(item: &str, (failed, detail): (bool, String), style: CliStyle) -> String { + let status: String = style.status(if failed { FAIL_STATUS } else { PASS_STATUS }); + format!( + " {:, style: CliStyle) -> String { + error.map_or_else(String::new, |error| { + format!( + "{:>EXPECTATION_ERROR_INDENT$}{}\n", + "", + result_error(error, style) + ) + }) +} + +/// Each expected comparison and assertion row, with its error. +pub(crate) fn expectation_rows(row: &ScenarioResultRow, style: CliStyle) -> String { + let mut text: String = String::new(); + for expected in &row.expected { + let detail: String = if expected.failed { + format!(" {} mismatched", expected.mismatched_row_count) + } else { + String::new() + }; + text.push_str(&expectation_line( + &format!("expected {}", expected.model_name), + (expected.failed, detail), + style, + )); + text.push_str(&expectation_error(expected.error.as_ref(), style)); + } + for assertion in &row.assertions { + let detail: String = if assertion.failed { + format!( + " {} {}", + assertion.failing_row_count, + rows_label(assertion.failing_row_count) + ) + } else { + String::new() + }; + text.push_str(&expectation_line( + &format!("assertion {}", assertion.name), + (assertion.failed, detail), + style, + )); + text.push_str(&expectation_error(assertion.error.as_ref(), style)); + } + text +} + +/// The counts of each failed expectation, and the first failing assertion sample. +pub(crate) fn expectation_failures(row: &ScenarioResultRow) -> String { + let mut text: String = String::new(); + for expected in row.expected.iter().filter(|expected| expected.failed) { + text.push_str(&format!( + " expected {}: actual={} expected={} mismatched={}\n", + expected.model_name, + expected.actual_row_count, + expected.expected_row_count, + expected.mismatched_row_count + )); + } + for assertion in row.assertions.iter().filter(|assertion| assertion.failed) { + text.push_str(&format!( + " assertion {}: failing_rows={}\n", + assertion.name, assertion.failing_row_count + )); + if let Some(sample) = &assertion.first_sample { + text.push_str(&format!(" sample: {sample}\n")); + } + } + text +} diff --git a/crates/sqlbuild-core/src/scenario_command/_helpers/mod.rs b/crates/sqlbuild-core/src/scenario_command/_helpers/mod.rs new file mode 100644 index 0000000000..64e033b9d7 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod expectations; +pub(crate) mod paths; +pub(crate) mod rows; +pub(crate) mod selection; +pub(crate) mod statements; diff --git a/crates/sqlbuild-core/src/scenario_command/_helpers/paths.rs b/crates/sqlbuild-core/src/scenario_command/_helpers/paths.rs new file mode 100644 index 0000000000..64645fb473 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/_helpers/paths.rs @@ -0,0 +1,63 @@ +//! Path comparisons with `pathlib.PurePath` semantics, and runtime artifact paths. + +use std::path::{Component, Path, PathBuf}; + +use crate::scenario_command::constants::{FUNCTIONS_DIR, MODELS_DIR, SQL_EXTENSION}; + +/// `Path(text)`: the path without `.` components or repeated separators. +pub(crate) fn normalized(path: &Path) -> PathBuf { + path.components() + .filter(|component| !matches!(component, Component::CurDir)) + .collect() +} + +/// `path == prefix or prefix in path.parents`. +pub(crate) fn is_under(path: &Path, prefix: &Path) -> bool { + if prefix.as_os_str().is_empty() { + return !path.has_root(); + } + path.starts_with(prefix) +} + +/// `Path(selector).suffix == ".sql"`. +pub(crate) fn has_sql_suffix(path: &Path) -> bool { + path.extension() + .is_some_and(|extension| extension == SQL_EXTENSION) +} + +/// A logical name safe as one file stem: path separators become `__`. +pub(crate) fn safe_file_stem(name: &str) -> String { + name.replace(['/', '\\'], "__") +} + +/// A model's artifact path, always under `models/`. +pub(crate) fn model_output_path(relative_path: &Path) -> PathBuf { + let path: PathBuf = normalized(relative_path); + if path + .components() + .next() + .is_some_and(|first| first.as_os_str() == MODELS_DIR) + { + return path; + } + Path::new(MODELS_DIR).join(path) +} + +/// A function's artifact path under `functions//`, with a `.sql` suffix. +pub(crate) fn function_output_path(relative_path: &Path, language: &str) -> PathBuf { + let path: PathBuf = normalized(relative_path); + let mut parts = path.components(); + let under_language: bool = parts + .next() + .is_some_and(|first| first.as_os_str() == FUNCTIONS_DIR) + && parts + .next() + .is_some_and(|second| second.as_os_str() == language); + let mut output: PathBuf = if under_language { + path + } else { + Path::new(FUNCTIONS_DIR).join(language).join(path) + }; + output.set_extension(SQL_EXTENSION); + output +} diff --git a/crates/sqlbuild-core/src/scenario_command/_helpers/rows.rs b/crates/sqlbuild-core/src/scenario_command/_helpers/rows.rs new file mode 100644 index 0000000000..8a0ec1aa5e --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/_helpers/rows.rs @@ -0,0 +1,93 @@ +//! Shared pieces of scenario and capture result rows. + +use crate::cli_text::main::format_coded_error::format_coded_error; +use crate::cli_text::models::CliStyle; +use crate::scenario_command::constants::{ + BINARY_UNIT_SIZE, CAPTURE_RELATION_KIND_WIDTH, CAPTURE_RELATION_NAME_WIDTH, FAIL_STATUS, + PASS_STATUS, SCENARIO_NAME_WIDTH, +}; +use crate::scenario_command::models::{CaptureRelationRow, CodedText}; +use crate::text::main::python_splitlines::python_splitlines; + +/// `render_result_error`: the coded error when it has a code, else its message. +pub(crate) fn result_error(error: &CodedText, style: CliStyle) -> String { + match &error.code { + Some(code) => { + format_coded_error((code, &error.message, error.help.as_deref()), true, style) + } + None => error.message.clone(), + } +} + +/// Each line of a rendered result error, indented by four spaces. +pub(crate) fn indented_error_lines(error: &CodedText, style: CliStyle) -> String { + python_splitlines(&result_error(error, style)) + .into_iter() + .map(|line| format!(" {line}\n")) + .collect() +} + +/// `row` for one, `rows` otherwise. +pub(crate) fn rows_label(count: i64) -> &'static str { + if count == 1 { "row" } else { "rows" } +} + +/// The result row heading: the padded name, the styled status and its detail. +pub(crate) fn heading(name: &str, status: &str, detail: &str) -> String { + format!("{name: String { + let bytes: f64 = byte_count as f64; + if bytes < BINARY_UNIT_SIZE { + return format!("{byte_count} B"); + } + let kibibytes: f64 = bytes / BINARY_UNIT_SIZE; + if kibibytes < BINARY_UNIT_SIZE { + return format!("{kibibytes:.1} KB"); + } + format!("{:.1} MB", kibibytes / BINARY_UNIT_SIZE) +} + +/// ` relation(s), row(s)`: passed relations and every captured row. +pub(crate) fn capture_detail(relations: &[CaptureRelationRow]) -> String { + let relation_count: usize = relations.iter().filter(|relation| relation.passed).count(); + let row_count: i64 = relations.iter().map(|relation| relation.row_count).sum(); + let relation_label: &str = if relation_count == 1 { + "relation" + } else { + "relations" + }; + format!( + " {relation_count} {relation_label}, {row_count} {}", + rows_label(row_count) + ) +} + +/// One captured relation row under its scenario. +pub(crate) fn capture_relation_rows(relations: &[CaptureRelationRow], style: CliStyle) -> String { + relations + .iter() + .map(|relation| { + let status: String = style.status(if relation.passed { + PASS_STATUS + } else { + FAIL_STATUS + }); + format!( + " {: String { + format!(" {: Self { + let relative_path: PathBuf = normalized(&scenario.relative_path); + let root: PathBuf = SCENARIO_ROOT.iter().collect(); + let root_relative_path: PathBuf = relative_path + .strip_prefix(&root) + .map_or_else(|_| relative_path.clone(), Path::to_path_buf); + Self { + file_path: normalized(&scenario.file_path), + relative_path, + root_relative_path, + } + } +} + +/// Reject graph expansion and path operators, which scenarios do not support. +fn validate_selector(selector: &str) -> Result<(), ScenarioCommandError> { + if selector.starts_with(GRAPH_SELECTOR_EXPANSION_MARKER) + || selector.ends_with(GRAPH_SELECTOR_EXPANSION_MARKER) + || selector.contains(GRAPH_SELECTOR_PATH_MARKER) + { + return Err(ScenarioCommandError { + code: UNSUPPORTED_GRAPH_SELECTOR_CODE, + message: format!( + "Scenario selector '{selector}' uses graph operators, which are not supported" + ), + help: Some( + "Use scenario names, scenario file paths, folders, --select-file, and --exclude. \ + Graph expansion operators such as '+model', 'model+', and 'a~b' are only \ + supported by graph resource commands." + .to_owned(), + ), + }); + } + Ok(()) +} + +/// Whether a name-less selector path names the scenario's file or one of its folders. +fn matches(paths: &ScenarioPaths, selector_path: &Path, project_dir: &Path) -> bool { + let project_selector: PathBuf = normalized(&project_dir.join(selector_path)); + if has_sql_suffix(selector_path) { + return paths.file_path == project_selector + || [ + &paths.file_path, + &paths.relative_path, + &paths.root_relative_path, + ] + .contains(&&selector_path.to_path_buf()); + } + is_under(&paths.file_path, selector_path) + || is_under(&paths.file_path, &project_selector) + || is_under(&paths.relative_path, selector_path) + || is_under(&paths.root_relative_path, selector_path) +} + +/// Indices of the scenarios one selector names, in project order. +pub(crate) fn selector_matches( + scenarios: &[ScenarioFile], + paths: &[ScenarioPaths], + (selector, project_dir): (&str, &Path), +) -> Result, ScenarioCommandError> { + validate_selector(selector)?; + let selector_path: PathBuf = normalized(Path::new(selector)); + let found: Vec = scenarios + .iter() + .zip(paths) + .enumerate() + .filter(|(_, (scenario, scenario_paths))| { + scenario.name == selector || matches(scenario_paths, &selector_path, project_dir) + }) + .map(|(index, _)| index) + .collect(); + if found.is_empty() { + return Err(ScenarioCommandError { + code: UNKNOWN_SELECTOR_CODE, + message: format!("Unknown scenario selector '{selector}'"), + help: None, + }); + } + Ok(found) +} diff --git a/crates/sqlbuild-core/src/scenario_command/_helpers/statements.rs b/crates/sqlbuild-core/src/scenario_command/_helpers/statements.rs new file mode 100644 index 0000000000..98c454e46d --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/_helpers/statements.rs @@ -0,0 +1,65 @@ +//! Executed SQL statements as runtime artifact text. + +use std::path::Path; + +use crate::scenario_command::constants::EXPECTATIONS_DIR; +use crate::scenario_command::models::{ExpectationSql, RuntimeArtifact}; +use crate::text::main::python_isspace::python_isspace; + +/// One statement ending in `;`, or unchanged when it is blank. +pub(crate) fn terminated(statement: &str) -> String { + let stripped: &str = statement.trim_end_matches(python_isspace); + if stripped.is_empty() { + return statement.to_owned(); + } + if stripped.ends_with(';') { + return stripped.to_owned(); + } + format!("{stripped};") +} + +/// Statements joined by blank lines, each terminated. +pub(crate) fn statements_text(statements: &[String]) -> String { + statements + .iter() + .map(|statement| terminated(statement)) + .collect::>() + .join("\n\n") +} + +/// The artifact, unless its statements render to no text. +pub(crate) fn statements_artifact( + relative_path: &Path, + statements: &[String], +) -> Option { + let sql: String = statements_text(statements); + if sql.is_empty() { + return None; + } + Some(RuntimeArtifact { + relative_path: relative_path.to_path_buf(), + sql, + }) +} + +/// The comparison SQL of each expected model, then the count and sample SQL of each assertion. +pub(crate) fn expectation_artifacts(expectations: &ExpectationSql) -> Vec { + let root: &Path = Path::new(EXPECTATIONS_DIR); + let mut artifacts: Vec = Vec::new(); + for (model_name, sql) in &expectations.expected { + artifacts.push(RuntimeArtifact { + relative_path: root.join(format!("expected__{model_name}.sql")), + sql: sql.clone(), + }); + } + for (name, sql) in &expectations.assertions { + artifacts.push(RuntimeArtifact { + relative_path: root.join(format!("assertion__{name}.sql")), + sql: format!( + "SELECT COUNT(*) FROM ({sql}) AS __scenario_assertion_failures;\n\n\ + SELECT * FROM ({sql}) AS __scenario_assertion_failures\nLIMIT 10;" + ), + }); + } + artifacts +} diff --git a/crates/sqlbuild-core/src/scenario_command/constants.rs b/crates/sqlbuild-core/src/scenario_command/constants.rs new file mode 100644 index 0000000000..5a5540fe8d --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/constants.rs @@ -0,0 +1,59 @@ +//! Scenario command labels, widths, defaults and error codes. + +/// The environment variable that sets the scenario run namespace. +pub const SCENARIO_NAMESPACE_ENV: &str = "SQLBUILD_SCENARIO_NAMESPACE"; +/// The longest accepted scenario run namespace. +pub(crate) const MAX_NAMESPACE_LENGTH: usize = 128; +pub(crate) const UNSET_NAMESPACE_LABEL: &str = "(unset)"; +pub(crate) const UNSET_NAMESPACE_SOURCE: &str = "unset"; +pub(crate) const CLI_SOURCE: &str = "cli"; +pub(crate) const ENV_SOURCE: &str = "env"; +pub(crate) const LOCAL_CONFIG_SOURCE: &str = "local config"; +pub(crate) const PROJECT_CONFIG_SOURCE: &str = "project config"; + +pub(crate) const SCENARIO_NAME_WIDTH: usize = 64; +pub(crate) const EXPECTATION_LABEL_WIDTH: usize = 10; +pub(crate) const EXPECTATION_NAME_WIDTH: usize = 50; +pub(crate) const EXPECTATION_ERROR_INDENT: usize = 14; +pub(crate) const RETAINED_KIND_WIDTH: usize = 6; +pub(crate) const CAPTURE_RELATION_KIND_WIDTH: usize = 8; +pub(crate) const CAPTURE_RELATION_NAME_WIDTH: usize = + SCENARIO_NAME_WIDTH - 4 - CAPTURE_RELATION_KIND_WIDTH - 1; +pub(crate) const BINARY_UNIT_SIZE: f64 = 1024.0; + +pub(crate) const PASS_STATUS: &str = "PASS"; +pub(crate) const FAIL_STATUS: &str = "FAIL"; + +pub(crate) const DEFAULT_MAX_SNAPSHOT_ROWS_PER_RELATION: i64 = 10_000; +pub(crate) const DEFAULT_MAX_SNAPSHOT_TOTAL_ROWS: i64 = 50_000; +pub(crate) const DEFAULT_MAX_SNAPSHOT_BYTES_PER_RELATION: i64 = 5_000_000; +pub(crate) const DEFAULT_MAX_SNAPSHOT_TOTAL_BYTES: i64 = 25_000_000; + +pub(crate) const NONE_DISCOVERED_CODE: &str = "C451"; +pub(crate) const LOCAL_RETAIN_UNSUPPORTED_CODE: &str = "C452"; +pub(crate) const UNKNOWN_SELECTOR_CODE: &str = "C453"; +pub(crate) const LOCAL_SNAPSHOT_FLAG_REQUIRED_CODE: &str = "C454"; +pub(crate) const SQL_VALIDATION_REQUIRED_CODE: &str = "C455"; +pub(crate) const CAPTURE_DIALECT_REQUIRED_CODE: &str = "C456"; +pub(crate) const UNSUPPORTED_GRAPH_SELECTOR_CODE: &str = "C457"; +pub(crate) const INVALID_NAMESPACE_CODE: &str = "C458"; + +pub(crate) const GRAPH_SELECTOR_EXPANSION_MARKER: char = '+'; +pub(crate) const GRAPH_SELECTOR_PATH_MARKER: char = '~'; +pub(crate) const SQL_EXTENSION: &str = "sql"; +pub(crate) const SCENARIO_ROOT: [&str; 2] = ["tests", "scenarios"]; + +pub(crate) const MODELS_DIR: &str = "models"; +pub(crate) const FUNCTIONS_DIR: &str = "functions"; +pub(crate) const FIXTURES_DIR: &str = "fixtures"; +pub(crate) const SEEDS_DIR: &str = "seeds"; +pub(crate) const EXPECTATIONS_DIR: &str = "expectations"; +pub(crate) const CLEANUP_DIR: &str = "cleanup"; + +pub(crate) const SECONDS_PER_DAY: i64 = 86_400; +pub(crate) const DAYS_PER_ERA: i64 = 146_097; +pub(crate) const EPOCH_DAY_OFFSET: i64 = 719_468; +pub(crate) const YEARS_PER_ERA: i64 = 400; +pub(crate) const FEBRUARY: i64 = 2; +/// The March-based month index of January. +pub(crate) const MARCH_BASED_JANUARY: i64 = 10; diff --git a/crates/sqlbuild-core/src/scenario_command/errors.rs b/crates/sqlbuild-core/src/scenario_command/errors.rs new file mode 100644 index 0000000000..5c29997d27 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/errors.rs @@ -0,0 +1,9 @@ +//! Scenario command usage errors. + +/// A user error the CLI reports with its code and optional help. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScenarioCommandError { + pub code: &'static str, + pub message: String, + pub help: Option, +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/build_local_runtime_artifacts.rs b/crates/sqlbuild-core/src/scenario_command/main/build_local_runtime_artifacts.rs new file mode 100644 index 0000000000..db7086ae41 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/build_local_runtime_artifacts.rs @@ -0,0 +1,45 @@ +//! The SQL a local scenario replay writes under `target/run/scenarios//local/`. + +use std::path::Path; + +use crate::scenario_command::_helpers::paths::{ + function_output_path, model_output_path, safe_file_stem, +}; +use crate::scenario_command::_helpers::statements::{expectation_artifacts, statements_text}; +use crate::scenario_command::constants::FIXTURES_DIR; +use crate::scenario_command::models::{LocalRuntimeInputs, RuntimeArtifact}; + +/// Loaded snapshot tables, created functions, model SQL, then expectations. +#[must_use] +pub fn build_local_runtime_artifacts(inputs: &LocalRuntimeInputs) -> Vec { + let mut artifacts: Vec = Vec::new(); + for relation in &inputs.loaded_relations { + artifacts.push(RuntimeArtifact { + relative_path: Path::new(FIXTURES_DIR).join(format!( + "{}__{}.sql", + relation.kind, + safe_file_stem(&relation.logical_name) + )), + sql: format!( + "-- loaded from {path}\n-- rows: {rows}\nCREATE TABLE \"{table}\" AS SELECT * FROM '{path}';", + path = relation.file_path, + rows = relation.row_count, + table = relation.table_name + ), + }); + } + for function in &inputs.functions { + artifacts.push(RuntimeArtifact { + relative_path: function_output_path(&function.relative_path, &function.language), + sql: statements_text(&function.statements), + }); + } + for (relative_path, sql) in &inputs.models { + artifacts.push(RuntimeArtifact { + relative_path: model_output_path(relative_path), + sql: sql.clone(), + }); + } + artifacts.extend(expectation_artifacts(&inputs.expectations)); + artifacts +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/build_runtime_artifacts.rs b/crates/sqlbuild-core/src/scenario_command/main/build_runtime_artifacts.rs new file mode 100644 index 0000000000..6ef9b8f736 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/build_runtime_artifacts.rs @@ -0,0 +1,56 @@ +//! The SQL a warehouse scenario run writes under `target/run/scenarios//`. + +use std::path::Path; + +use crate::scenario_command::_helpers::paths::{model_output_path, safe_file_stem}; +use crate::scenario_command::_helpers::statements::{expectation_artifacts, statements_artifact}; +use crate::scenario_command::constants::{CLEANUP_DIR, FIXTURES_DIR, SEEDS_DIR}; +use crate::scenario_command::models::{RuntimeArtifact, WarehouseRuntimeInputs}; + +/// Prepare cleanup, fixtures, seeds, models, expectations, then the final cleanup. +#[must_use] +pub fn build_runtime_artifacts(inputs: &WarehouseRuntimeInputs) -> Vec { + let cleanup = |file_name: &str, statements: &Option>| { + statements.as_ref().and_then(|statements| { + statements_artifact(&Path::new(CLEANUP_DIR).join(file_name), statements) + }) + }; + let fixtures = inputs.fixtures.iter().filter_map(|fixture| { + statements_artifact( + &Path::new(FIXTURES_DIR).join(format!( + "{}__{}.sql", + fixture.kind, + safe_file_stem(&fixture.logical_name) + )), + &fixture.statements, + ) + }); + let seeds = inputs.seeds.iter().filter_map(|seed| { + statements_artifact( + &Path::new(SEEDS_DIR).join(format!("{}.sql", seed.name)), + &seed.statements, + ) + }); + let mut models: Vec = Vec::new(); + for model in &inputs.models { + let Some(entry) = inputs + .model_entries + .iter() + .find(|entry| entry.name == model.name) + else { + continue; + }; + models.extend(statements_artifact( + &model_output_path(&entry.relative_path), + &model.statements, + )); + } + cleanup("prepare.sql", &inputs.prepare_cleanup) + .into_iter() + .chain(fixtures) + .chain(seeds) + .chain(models) + .chain(expectation_artifacts(&inputs.expectations)) + .chain(cleanup("final.sql", &inputs.final_cleanup)) + .collect() +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/capture_dialect_required_error.rs b/crates/sqlbuild-core/src/scenario_command/main/capture_dialect_required_error.rs new file mode 100644 index 0000000000..bc720758d8 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/capture_dialect_required_error.rs @@ -0,0 +1,20 @@ +//! The error when an adapter has no SQL analysis dialect for scenario snapshots. + +use crate::scenario_command::constants::CAPTURE_DIALECT_REQUIRED_CODE; +use crate::scenario_command::errors::ScenarioCommandError; + +/// Snapshots need the adapter's SQL analysis dialect to be validated and replayed. +#[must_use] +pub fn capture_dialect_required_error(adapter_name: &str) -> ScenarioCommandError { + ScenarioCommandError { + code: CAPTURE_DIALECT_REQUIRED_CODE, + message: format!( + "Adapter '{adapter_name}' must define a SQL analysis dialect for scenario snapshots" + ), + help: Some( + "Set sql_analysis_dialect_name on the adapter so captured snapshots can be validated \ + and replayed locally." + .to_owned(), + ), + } +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/format_captured_at.rs b/crates/sqlbuild-core/src/scenario_command/main/format_captured_at.rs new file mode 100644 index 0000000000..e120c37853 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/format_captured_at.rs @@ -0,0 +1,40 @@ +//! The capture time of a snapshot, as `YYYY-MM-DDTHH:MM:SSZ`. + +use crate::scenario_command::constants::{ + DAYS_PER_ERA, EPOCH_DAY_OFFSET, FEBRUARY, MARCH_BASED_JANUARY, SECONDS_PER_DAY, YEARS_PER_ERA, +}; + +/// The proleptic Gregorian `(year, month, day)` of days since 1970-01-01 (Hinnant's algorithm). +fn civil_date(days: i64) -> (i64, i64, i64) { + let shifted: i64 = days + EPOCH_DAY_OFFSET; + let era: i64 = shifted.div_euclid(DAYS_PER_ERA); + let day_of_era: i64 = shifted.rem_euclid(DAYS_PER_ERA); + let year_of_era: i64 = + (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year: i64 = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index: i64 = (5 * day_of_year + 2) / 153; + let day: i64 = day_of_year - (153 * month_index + 2) / 5 + 1; + let month: i64 = if month_index < MARCH_BASED_JANUARY { + month_index + 3 + } else { + month_index - 9 + }; + ( + year_of_era + era * YEARS_PER_ERA + i64::from(month <= FEBRUARY), + month, + day, + ) +} + +/// `datetime.now(UTC).replace(microsecond=0).isoformat()` with `+00:00` written as `Z`. +#[must_use] +pub fn format_captured_at(epoch_seconds: i64) -> String { + let (year, month, day) = civil_date(epoch_seconds.div_euclid(SECONDS_PER_DAY)); + let of_day: i64 = epoch_seconds.rem_euclid(SECONDS_PER_DAY); + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z", + of_day / 3_600, + of_day % 3_600 / 60, + of_day % 60 + ) +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/mod.rs b/crates/sqlbuild-core/src/scenario_command/main/mod.rs new file mode 100644 index 0000000000..905a2d5b48 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/mod.rs @@ -0,0 +1,17 @@ +pub mod build_local_runtime_artifacts; +pub mod build_runtime_artifacts; +pub mod capture_dialect_required_error; +pub mod format_captured_at; +pub mod render_capture_result; +pub mod render_namespace_header; +pub mod render_scenario_result; +pub mod render_snapshot_sync_result; +pub mod resolve_scenario_namespace; +pub mod resolve_snapshot_limits; +pub mod scenario_activity_message; +pub mod select_scenarios; +pub mod snapshot_capture_warning; +pub mod sql_analysis_purpose; +pub mod sql_analysis_required_error; +pub mod validate_scenario_test_flags; +pub mod write_runtime_artifacts; diff --git a/crates/sqlbuild-core/src/scenario_command/main/render_capture_result.rs b/crates/sqlbuild-core/src/scenario_command/main/render_capture_result.rs new file mode 100644 index 0000000000..92de827343 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/render_capture_result.rs @@ -0,0 +1,45 @@ +//! The result row of one finished scenario snapshot capture. + +use crate::cli_text::models::CliStyle; +use crate::scenario_command::_helpers::rows::{ + capture_detail, capture_relation_rows, heading, indented_error_lines, snapshot_line, +}; +use crate::scenario_command::constants::{FAIL_STATUS, PASS_STATUS, RETAINED_KIND_WIDTH}; +use crate::scenario_command::models::CaptureRow; + +/// The status row, its error, captured relations, snapshot path and retained inputs. +#[must_use] +pub fn render_capture_result(row: &CaptureRow, style: CliStyle) -> String { + let status: String = style.status(if row.passed { PASS_STATUS } else { FAIL_STATUS }); + let detail: String = row + .relations + .as_deref() + .map(capture_detail) + .unwrap_or_default(); + let mut text: String = heading(&row.name, &status, &detail); + if let Some(error) = row.error.as_ref().filter(|error| !error.message.is_empty()) { + text.push_str(&indented_error_lines(error, style)); + if !row.retained { + text.push_str(" Rerun with --retain to inspect scenario-owned artifacts.\n"); + } + } + if let (Some(relations), Some(path)) = (&row.relations, &row.snapshot_path) { + text.push_str(&capture_relation_rows(relations, style)); + text.push_str(&snapshot_line(path)); + } + if row.retained + && let Some(inputs) = &row.retained_inputs + { + text.push_str(" Retained relations:\n"); + for fixture in &inputs.fixtures { + text.push_str(&format!( + " {: {}\n", + fixture.kind, fixture.logical_name, fixture.physical_name + )); + } + for seed in &inputs.seeds { + text.push_str(&format!(" seed {seed}\n")); + } + } + text +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/render_namespace_header.rs b/crates/sqlbuild-core/src/scenario_command/main/render_namespace_header.rs new file mode 100644 index 0000000000..5b7cb2b32c --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/render_namespace_header.rs @@ -0,0 +1,14 @@ +//! The namespace line every scenario command prints first. + +use crate::scenario_command::constants::UNSET_NAMESPACE_LABEL; +use crate::scenario_command::models::ScenarioNamespace; + +/// `Scenario namespace: (source: )` and a newline. +#[must_use] +pub fn render_namespace_header(namespace: &ScenarioNamespace) -> String { + format!( + "Scenario namespace: {} (source: {})\n", + namespace.value.as_deref().unwrap_or(UNSET_NAMESPACE_LABEL), + namespace.source + ) +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/render_scenario_result.rs b/crates/sqlbuild-core/src/scenario_command/main/render_scenario_result.rs new file mode 100644 index 0000000000..608f70ade9 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/render_scenario_result.rs @@ -0,0 +1,40 @@ +//! The result row of one finished scenario run. + +use crate::cli_text::models::CliStyle; +use crate::scenario_command::_helpers::expectations::{expectation_failures, expectation_rows}; +use crate::scenario_command::_helpers::rows::{heading, indented_error_lines}; +use crate::scenario_command::constants::RETAINED_KIND_WIDTH; +use crate::scenario_command::models::ScenarioResultRow; + +/// The status row, its error, expectations, retained artifacts and failure details. +#[must_use] +pub fn render_scenario_result(row: &ScenarioResultRow, style: CliStyle) -> String { + let duration: String = row + .duration_ms + .map(|milliseconds| format!(" {:.2}s", milliseconds / 1000.0)) + .unwrap_or_default(); + let mut text: String = heading(&row.name, &style.status(&row.status), &duration); + if let Some(error) = row.error.as_ref().filter(|error| !error.message.is_empty()) { + text.push_str(&indented_error_lines(error, style)); + if !row.retained && !row.local { + text.push_str(" Rerun with --retain to inspect scenario-owned artifacts.\n"); + } + } + text.push_str(&expectation_rows(row, style)); + if row.retained { + if let Some(relations) = &row.retained_relations { + text.push_str(" Retained relations:\n"); + for relation in relations { + text.push_str(&format!( + " {: {}\n", + relation.kind, relation.logical_name, relation.physical_name + )); + } + } + if let Some(path) = &row.local_duckdb_path { + text.push_str(&format!(" Retained local DuckDB: {path}\n")); + } + } + text.push_str(&expectation_failures(row)); + text +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/render_snapshot_sync_result.rs b/crates/sqlbuild-core/src/scenario_command/main/render_snapshot_sync_result.rs new file mode 100644 index 0000000000..a48f6195ac --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/render_snapshot_sync_result.rs @@ -0,0 +1,31 @@ +//! The result row of one scenario captured by `--sync-snapshots` or `--refresh`. + +use crate::cli_text::models::CliStyle; +use crate::scenario_command::_helpers::rows::{ + capture_detail, capture_relation_rows, heading, indented_error_lines, snapshot_line, +}; +use crate::scenario_command::constants::FAIL_STATUS; +use crate::scenario_command::models::CaptureRow; + +/// `CAPTURED`, `REFRESHED` or `FAIL`, its error, captured relations and snapshot path. +#[must_use] +pub fn render_snapshot_sync_result(row: &CaptureRow, refresh: bool, style: CliStyle) -> String { + let success: &str = if refresh { "REFRESHED" } else { "CAPTURED" }; + let status: String = style.status(if row.passed { success } else { FAIL_STATUS }); + let detail: String = row + .relations + .as_deref() + .map(capture_detail) + .unwrap_or_default(); + let mut text: String = heading(&row.name, &status, &detail); + if let Some(error) = row.error.as_ref().filter(|error| !error.message.is_empty()) { + text.push_str(&indented_error_lines(error, style)); + } + if let Some(relations) = &row.relations { + text.push_str(&capture_relation_rows(relations, style)); + if let Some(path) = &row.snapshot_path { + text.push_str(&snapshot_line(path)); + } + } + text +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/resolve_scenario_namespace.rs b/crates/sqlbuild-core/src/scenario_command/main/resolve_scenario_namespace.rs new file mode 100644 index 0000000000..fd3d3021e0 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/resolve_scenario_namespace.rs @@ -0,0 +1,49 @@ +//! The effective scenario run namespace. + +use crate::scenario_command::constants::{ + CLI_SOURCE, ENV_SOURCE, INVALID_NAMESPACE_CODE, LOCAL_CONFIG_SOURCE, MAX_NAMESPACE_LENGTH, + PROJECT_CONFIG_SOURCE, UNSET_NAMESPACE_SOURCE, +}; +use crate::scenario_command::errors::ScenarioCommandError; +use crate::scenario_command::models::{NamespaceCandidates, ScenarioNamespace}; + +fn is_valid(value: &str) -> bool { + (1..=MAX_NAMESPACE_LENGTH).contains(&value.len()) + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-')) +} + +/// CLI, then environment, then local config, then project config; the first set value must be valid. +pub fn resolve_scenario_namespace( + candidates: NamespaceCandidates, +) -> Result { + let ordered: [(Option, &str); 4] = [ + (candidates.cli, CLI_SOURCE), + (candidates.env, ENV_SOURCE), + (candidates.local_config, LOCAL_CONFIG_SOURCE), + (candidates.project_config, PROJECT_CONFIG_SOURCE), + ]; + for (value, source) in ordered { + let Some(value) = value else { + continue; + }; + if !is_valid(&value) { + return Err(ScenarioCommandError { + code: INVALID_NAMESPACE_CODE, + message: "scenario.run_namespace must be 1–128 characters: ASCII letters, \ + digits, '-', '_' or '.'" + .to_owned(), + help: None, + }); + } + return Ok(ScenarioNamespace { + value: Some(value), + source: source.to_owned(), + }); + } + Ok(ScenarioNamespace { + value: None, + source: UNSET_NAMESPACE_SOURCE.to_owned(), + }) +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/resolve_snapshot_limits.rs b/crates/sqlbuild-core/src/scenario_command/main/resolve_snapshot_limits.rs new file mode 100644 index 0000000000..20cc330efc --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/resolve_snapshot_limits.rs @@ -0,0 +1,34 @@ +//! Effective snapshot capture limits. + +use crate::scenario_command::constants::{ + DEFAULT_MAX_SNAPSHOT_BYTES_PER_RELATION, DEFAULT_MAX_SNAPSHOT_ROWS_PER_RELATION, + DEFAULT_MAX_SNAPSHOT_TOTAL_BYTES, DEFAULT_MAX_SNAPSHOT_TOTAL_ROWS, +}; +use crate::scenario_command::models::{SnapshotLimitConfig, SnapshotLimitInputs, SnapshotLimits}; + +/// Each limit from the command line, else the configuration, else its default. +#[must_use] +pub fn resolve_snapshot_limits( + inputs: SnapshotLimitInputs, + config: SnapshotLimitConfig, +) -> SnapshotLimits { + SnapshotLimits { + max_rows_per_relation: inputs + .max_rows_per_relation + .or(config.max_rows_per_relation) + .unwrap_or(DEFAULT_MAX_SNAPSHOT_ROWS_PER_RELATION), + max_total_rows: inputs + .max_total_rows + .or(config.max_total_rows) + .unwrap_or(DEFAULT_MAX_SNAPSHOT_TOTAL_ROWS), + max_bytes_per_relation: inputs + .max_bytes_per_relation + .or(config.max_bytes_per_relation) + .unwrap_or(DEFAULT_MAX_SNAPSHOT_BYTES_PER_RELATION), + max_total_bytes: inputs + .max_total_bytes + .or(config.max_total_bytes) + .unwrap_or(DEFAULT_MAX_SNAPSHOT_TOTAL_BYTES), + force: inputs.force, + } +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/scenario_activity_message.rs b/crates/sqlbuild-core/src/scenario_command/main/scenario_activity_message.rs new file mode 100644 index 0000000000..3036695aab --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/scenario_activity_message.rs @@ -0,0 +1,14 @@ +//! Scenario lifecycle progress labelled with the effective namespace. + +use crate::scenario_command::constants::UNSET_NAMESPACE_LABEL; +use crate::scenario_command::models::ScenarioNamespace; + +/// ` (namespace: ; source: )`. +#[must_use] +pub fn scenario_activity_message(activity: &str, namespace: &ScenarioNamespace) -> String { + format!( + "{activity} (namespace: {}; source: {})", + namespace.value.as_deref().unwrap_or(UNSET_NAMESPACE_LABEL), + namespace.source + ) +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/select_scenarios.rs b/crates/sqlbuild-core/src/scenario_command/main/select_scenarios.rs new file mode 100644 index 0000000000..10826941c0 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/select_scenarios.rs @@ -0,0 +1,50 @@ +//! Scenario names, files and folders resolved to the scenarios they select. + +use std::collections::BTreeSet; + +use crate::scenario_command::_helpers::selection::{ScenarioPaths, selector_matches}; +use crate::scenario_command::constants::NONE_DISCOVERED_CODE; +use crate::scenario_command::errors::ScenarioCommandError; +use crate::scenario_command::models::{ScenarioFile, ScenarioSelection}; + +/// Indices of the selected scenarios: every scenario without selectors, less the excluded ones. +pub fn select_scenarios( + scenarios: &[ScenarioFile], + selection: ScenarioSelection<'_>, +) -> Result, ScenarioCommandError> { + let paths: Vec = scenarios.iter().map(ScenarioPaths::of).collect(); + let selected: Vec = if selection.selectors.is_empty() { + if scenarios.is_empty() { + return Err(ScenarioCommandError { + code: NONE_DISCOVERED_CODE, + message: "No SQL scenarios were discovered under tests/scenarios".to_owned(), + help: None, + }); + } + (0..scenarios.len()).collect() + } else { + let mut selected: Vec = Vec::new(); + let mut names: BTreeSet<&str> = BTreeSet::new(); + for selector in selection.selectors { + for index in selector_matches(scenarios, &paths, (selector, selection.project_dir))? { + if names.insert(scenarios[index].name.as_str()) { + selected.push(index); + } + } + } + selected + }; + if selection.exclude.is_empty() { + return Ok(selected); + } + let mut excluded: BTreeSet<&str> = BTreeSet::new(); + for selector in selection.exclude { + for index in selector_matches(scenarios, &paths, (selector, selection.project_dir))? { + excluded.insert(scenarios[index].name.as_str()); + } + } + Ok(selected + .into_iter() + .filter(|index| !excluded.contains(scenarios[*index].name.as_str())) + .collect()) +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/snapshot_capture_warning.rs b/crates/sqlbuild-core/src/scenario_command/main/snapshot_capture_warning.rs new file mode 100644 index 0000000000..00c2f8b818 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/snapshot_capture_warning.rs @@ -0,0 +1,15 @@ +//! The warning shown before committable snapshot files are written. + +/// Review guidance for captured snapshots, noting when size limits are bypassed. +#[must_use] +pub fn snapshot_capture_warning(force: bool) -> String { + let suffix: &str = if force { + " Size limits are bypassed." + } else { + "" + }; + format!( + "Review captured scenario snapshots before committing; they may contain sensitive \ + warehouse data.{suffix}" + ) +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/sql_analysis_purpose.rs b/crates/sqlbuild-core/src/scenario_command/main/sql_analysis_purpose.rs new file mode 100644 index 0000000000..52e117196d --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/sql_analysis_purpose.rs @@ -0,0 +1,14 @@ +//! The purpose SQL analysis guidance names for a scenario command. + +use crate::scenario_command::models::SqlAnalysisCommand; + +/// The purpose a config file's SQL analysis guidance names for `command`. +#[must_use] +pub fn sql_analysis_purpose(command: SqlAnalysisCommand) -> &'static str { + match command { + SqlAnalysisCommand::LocalTest => { + "to run local scenario replay, snapshot sync, or snapshot refresh" + } + SqlAnalysisCommand::Capture => "to capture snapshots for local scenario replay", + } +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/sql_analysis_required_error.rs b/crates/sqlbuild-core/src/scenario_command/main/sql_analysis_required_error.rs new file mode 100644 index 0000000000..89175dca2e --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/sql_analysis_required_error.rs @@ -0,0 +1,37 @@ +//! The error when a scenario command needs SQL analysis that is turned off. + +use crate::scenario_command::constants::SQL_VALIDATION_REQUIRED_CODE; +use crate::scenario_command::errors::ScenarioCommandError; +use crate::scenario_command::models::SqlAnalysisCommand; + +/// Turned off by `--no-sql-analysis`, or by the config file `guidance` names as `(note, help)`. +#[must_use] +pub fn sql_analysis_required_error( + command: SqlAnalysisCommand, + disabled_by_flag: bool, + guidance: Option<(String, String)>, +) -> Option { + let subject: &str = match command { + SqlAnalysisCommand::LocalTest => "scenario test --local", + SqlAnalysisCommand::Capture => "scenario capture", + }; + if disabled_by_flag { + let help: &str = match command { + SqlAnalysisCommand::LocalTest => "run the command without `--no-sql-analysis`", + SqlAnalysisCommand::Capture => "run the capture without `--no-sql-analysis`", + }; + return Some(ScenarioCommandError { + code: SQL_VALIDATION_REQUIRED_CODE, + message: format!( + "{subject} requires SQL analysis, but `--no-sql-analysis` turns it off" + ), + help: Some(help.to_owned()), + }); + } + let (note, help) = guidance?; + Some(ScenarioCommandError { + code: SQL_VALIDATION_REQUIRED_CODE, + message: format!("{subject} requires SQL analysis; {note}"), + help: Some(help), + }) +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/validate_scenario_test_flags.rs b/crates/sqlbuild-core/src/scenario_command/main/validate_scenario_test_flags.rs new file mode 100644 index 0000000000..0534bb3432 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/validate_scenario_test_flags.rs @@ -0,0 +1,33 @@ +//! `sqb scenario test` flag combinations rejected before the project is read. + +use crate::scenario_command::constants::{ + LOCAL_RETAIN_UNSUPPORTED_CODE, LOCAL_SNAPSHOT_FLAG_REQUIRED_CODE, +}; +use crate::scenario_command::errors::ScenarioCommandError; +use crate::scenario_command::models::ScenarioTestFlags; + +/// `--local` cannot retain, and snapshot sync flags need `--local`. +pub fn validate_scenario_test_flags(flags: ScenarioTestFlags) -> Result<(), ScenarioCommandError> { + if flags.local && flags.retain { + return Err(ScenarioCommandError { + code: LOCAL_RETAIN_UNSUPPORTED_CODE, + message: "scenario test --local does not support --retain".to_owned(), + help: Some( + "Local scenario DuckDB files are always kept under target/run/scenarios/." + .to_owned(), + ), + }); + } + if !flags.local && (flags.sync_snapshots || flags.refresh) { + return Err(ScenarioCommandError { + code: LOCAL_SNAPSHOT_FLAG_REQUIRED_CODE, + message: "scenario snapshot sync flags require --local".to_owned(), + help: Some( + "Use sqb scenario test --local --sync-snapshots or sqb scenario test --local \ + --refresh." + .to_owned(), + ), + }); + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/scenario_command/main/write_runtime_artifacts.rs b/crates/sqlbuild-core/src/scenario_command/main/write_runtime_artifacts.rs new file mode 100644 index 0000000000..e9593d4778 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/main/write_runtime_artifacts.rs @@ -0,0 +1,26 @@ +//! Replace a scenario run directory with its runtime SQL artifacts. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use crate::scenario_command::models::RuntimeArtifact; +use crate::text::main::python_isspace::python_isspace; + +/// Remove `root`, then write each artifact's SQL without trailing whitespace and with one newline. +pub fn write_runtime_artifacts(root: &Path, artifacts: &[RuntimeArtifact]) -> io::Result<()> { + if root.exists() { + fs::remove_dir_all(root)?; + } + for artifact in artifacts { + let path: PathBuf = root.join(&artifact.relative_path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write( + &path, + format!("{}\n", artifact.sql.trim_end_matches(python_isspace)), + )?; + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/scenario_command/mod.rs b/crates/sqlbuild-core/src/scenario_command/mod.rs new file mode 100644 index 0000000000..ca807ba945 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/mod.rs @@ -0,0 +1,10 @@ +//! `sqb scenario`: namespaces, selection, capture limits, result rows and runtime SQL artifacts. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/scenario_command/models.rs b/crates/sqlbuild-core/src/scenario_command/models.rs new file mode 100644 index 0000000000..f9255c724e --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/models.rs @@ -0,0 +1,248 @@ +//! Plain data of scenario command inputs, results and runtime artifacts. + +use std::path::PathBuf; + +/// The effective scenario run namespace and the configuration source it came from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScenarioNamespace { + pub value: Option, + pub source: String, +} + +/// Every place a scenario run namespace can be set, from highest precedence to lowest. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NamespaceCandidates { + pub cli: Option, + pub env: Option, + pub local_config: Option, + pub project_config: Option, +} + +/// The `sqb scenario test` flags validated before the project is read. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ScenarioTestFlags { + pub local: bool, + pub retain: bool, + pub sync_snapshots: bool, + pub refresh: bool, +} + +/// Which scenario command needs SQL analysis, as its error messages name it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SqlAnalysisCommand { + LocalTest, + Capture, +} + +/// A coded result error: its message with optional code and help. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CodedText { + pub code: Option, + pub message: String, + pub help: Option, +} + +/// One expected-relation comparison of a scenario result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExpectedRow { + pub model_name: String, + pub failed: bool, + pub actual_row_count: i64, + pub expected_row_count: i64, + pub mismatched_row_count: i64, + pub error: Option, +} + +/// One zero-row assertion of a scenario result. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AssertionRow { + pub name: String, + pub failed: bool, + pub failing_row_count: i64, + /// `str()` of the first failing sample row, when there is one. + pub first_sample: Option, + pub error: Option, +} + +/// A retained scenario relation: its artifact kind, logical name and physical relation. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RetainedRelation { + pub kind: String, + pub logical_name: String, + pub physical_name: String, +} + +/// One finished scenario run as its result row shows it. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ScenarioResultRow { + pub name: String, + /// `PASS`/`FAIL`, or the local replay status. + pub status: String, + /// The run replayed snapshots locally. + pub local: bool, + pub duration_ms: Option, + pub error: Option, + pub retained: bool, + /// The retained relation map, when the run has one. + pub retained_relations: Option>, + pub local_duckdb_path: Option, + pub expected: Vec, + pub assertions: Vec, +} + +/// One captured scenario input relation. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CaptureRelationRow { + pub kind: String, + pub logical_name: String, + pub passed: bool, + pub row_count: i64, + pub byte_count: i64, +} + +/// The scenario-owned inputs a retained capture kept. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RetainedInputs { + pub fixtures: Vec, + pub seeds: Vec, +} + +/// One finished scenario snapshot capture as its result row shows it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CaptureRow { + pub name: String, + pub passed: bool, + pub error: Option, + pub retained: bool, + /// The captured relations, when the capture wrote a snapshot result. + pub relations: Option>, + /// The snapshot manifest path shown to the user, when there is one. + pub snapshot_path: Option, + /// The retained inputs, when the capture was retained and planned. + pub retained_inputs: Option, +} + +/// Snapshot limit overrides from the command line. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SnapshotLimitInputs { + pub max_rows_per_relation: Option, + pub max_total_rows: Option, + pub max_bytes_per_relation: Option, + pub max_total_bytes: Option, + pub force: bool, +} + +/// Snapshot limits configured under `[scenario.snapshot_limits]`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct SnapshotLimitConfig { + pub max_rows_per_relation: Option, + pub max_total_rows: Option, + pub max_bytes_per_relation: Option, + pub max_total_bytes: Option, +} + +/// The effective snapshot capture limits. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SnapshotLimits { + pub max_rows_per_relation: i64, + pub max_total_rows: i64, + pub max_bytes_per_relation: i64, + pub max_total_bytes: i64, + pub force: bool, +} + +/// One compiled scenario as selectors see it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ScenarioFile { + pub name: String, + pub file_path: PathBuf, + /// The scenario file path relative to the project. + pub relative_path: PathBuf, +} + +/// Scenario selectors and exclusions with the project they are relative to. +#[derive(Clone, Copy, Debug)] +pub struct ScenarioSelection<'a> { + pub selectors: &'a [String], + pub exclude: &'a [String], + pub project_dir: &'a std::path::Path, +} + +/// One runtime SQL artifact: its path under the scenario run directory and its SQL. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RuntimeArtifact { + pub relative_path: PathBuf, + pub sql: String, +} + +/// SQL statements a fixture executed, with its artifact kind and logical name. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FixtureStatements { + pub kind: String, + pub logical_name: String, + pub statements: Vec, +} + +/// SQL statements a seed or model executed, with its path or name. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NamedStatements { + pub name: String, + pub statements: Vec, +} + +/// A planned model whose executed SQL is written under `models/`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelArtifactEntry { + pub name: String, + pub relative_path: PathBuf, +} + +/// The planned expected comparisons and assertions of a scenario. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExpectationSql { + /// Each expected model with its rendered comparison SQL. + pub expected: Vec<(String, String)>, + /// Each assertion with its authored SQL. + pub assertions: Vec<(String, String)>, +} + +/// Everything a warehouse scenario run writes under `target/run/scenarios//`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct WarehouseRuntimeInputs { + pub prepare_cleanup: Option>, + pub fixtures: Vec, + pub seeds: Vec, + pub models: Vec, + pub model_entries: Vec, + pub expectations: ExpectationSql, + pub final_cleanup: Option>, +} + +/// A snapshot relation a local replay loaded into DuckDB. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LoadedRelation { + pub kind: String, + pub logical_name: String, + /// The snapshot file, as a POSIX path. + pub file_path: String, + pub table_name: String, + pub row_count: i64, +} + +/// A function a local replay created, with its rendered statements. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FunctionStatements { + pub relative_path: PathBuf, + pub language: String, + pub statements: Vec, +} + +/// Everything a local scenario replay writes under `target/run/scenarios//local/`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LocalRuntimeInputs { + pub loaded_relations: Vec, + pub functions: Vec, + /// Each planned model's relative path and resolved SQL. + pub models: Vec<(PathBuf, String)>, + pub expectations: ExpectationSql, +} diff --git a/crates/sqlbuild-core/src/scenario_command/tests.rs b/crates/sqlbuild-core/src/scenario_command/tests.rs new file mode 100644 index 0000000000..26751b47e3 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_scenario_command.rs"] +mod scenario_command; +#[path = "tests/test_scenario_rows.rs"] +mod scenario_rows; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/scenario_command/tests/helpers.rs b/crates/sqlbuild-core/src/scenario_command/tests/helpers.rs new file mode 100644 index 0000000000..7571418f09 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/tests/helpers.rs @@ -0,0 +1,189 @@ +use std::path::{Path, PathBuf}; + +use crate::scenario_command::errors::ScenarioCommandError; +use crate::scenario_command::models::{ + AssertionRow, CaptureRelationRow, CodedText, ExpectationSql, ExpectedRow, NamespaceCandidates, + RetainedRelation, RuntimeArtifact, ScenarioFile, ScenarioNamespace, ScenarioResultRow, +}; + +pub(super) const PROJECT_DIR: &str = "/work/orders_project"; + +pub(super) fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_owned()).collect() +} + +pub(super) fn scenario_files() -> Vec { + [ + ("orders_paid", "tests/scenarios/orders/paid.sql"), + ("orders_refund", "tests/scenarios/orders/refund.sql"), + ("customers_new", "tests/scenarios/customers_new.sql"), + ] + .iter() + .map(|(name, relative_path)| ScenarioFile { + name: (*name).to_owned(), + file_path: Path::new(PROJECT_DIR).join(relative_path), + relative_path: PathBuf::from(relative_path), + }) + .collect() +} + +pub(super) fn coded(code: Option<&str>, message: &str, help: Option<&str>) -> Option { + Some(CodedText { + code: code.map(str::to_owned), + message: message.to_owned(), + help: help.map(str::to_owned), + }) +} + +pub(super) fn artifact_pairs(artifacts: Vec) -> Vec<(PathBuf, String)> { + artifacts + .into_iter() + .map(|artifact| (artifact.relative_path, artifact.sql)) + .collect() +} + +pub(super) fn expected_pairs(expected: &[(PathBuf, &str)]) -> Vec<(PathBuf, String)> { + expected + .iter() + .map(|(path, sql)| (path.clone(), (*sql).to_owned())) + .collect() +} + +pub(super) fn retained(kind: &str, logical_name: &str, physical_name: &str) -> RetainedRelation { + RetainedRelation { + kind: kind.to_owned(), + logical_name: logical_name.to_owned(), + physical_name: physical_name.to_owned(), + } +} + +pub(super) fn failing_result_row() -> ScenarioResultRow { + ScenarioResultRow { + name: "orders_paid".to_owned(), + status: "FAIL".to_owned(), + duration_ms: Some(1234.5), + error: coded( + Some("X506"), + "expected comparison failed\nsecond line", + Some("Compare the CTE."), + ), + retained: true, + retained_relations: Some(vec![ + retained("source", "raw_orders", "main.__sqb_abc__source__raw_orders"), + retained("model", "fact_orders", "main.__sqb_abc__model__fact_orders"), + ]), + expected: vec![ + ExpectedRow { + model_name: "fact_orders".to_owned(), + failed: true, + actual_row_count: 3, + expected_row_count: 2, + mismatched_row_count: 2, + error: coded(Some("X506"), "mismatch", Some("look")), + }, + ExpectedRow { + model_name: "dim_customers".to_owned(), + actual_row_count: 1, + expected_row_count: 1, + ..ExpectedRow::default() + }, + ], + assertions: vec![ + AssertionRow { + name: "no_negative".to_owned(), + failed: true, + failing_row_count: 1, + first_sample: Some("(1, 'a', None)".to_owned()), + error: None, + }, + AssertionRow { + name: "few".to_owned(), + failed: true, + failing_row_count: 3, + first_sample: None, + error: coded(None, "plain error", None), + }, + AssertionRow { + name: "ok".to_owned(), + ..AssertionRow::default() + }, + ], + ..ScenarioResultRow::default() + } +} + +pub(super) fn relation( + kind: &str, + logical_name: &str, + (passed, row_count, byte_count): (bool, i64, i64), +) -> CaptureRelationRow { + CaptureRelationRow { + kind: kind.to_owned(), + logical_name: logical_name.to_owned(), + passed, + row_count, + byte_count, + } +} + +pub(super) fn statements(values: &[&str]) -> Vec { + strings(values) +} + +pub(super) fn expectation_sql() -> ExpectationSql { + ExpectationSql { + expected: vec![( + "order_totals".to_owned(), + "SELECT 1 EXCEPT SELECT 2".to_owned(), + )], + assertions: vec![( + "no_negative".to_owned(), + "SELECT * FROM t WHERE x < 0".to_owned(), + )], + } +} + +pub(super) const INVALID_NAMESPACE: &str = + "scenario.run_namespace must be 1–128 characters: ASCII letters, digits, '-', '_' or '.'"; + +pub(super) fn namespace( + value: &str, + source: &str, +) -> Result { + Ok(ScenarioNamespace { + value: Some(value.to_owned()), + source: source.to_owned(), + }) +} + +pub(super) fn invalid_namespace() -> Result { + Err(ScenarioCommandError { + code: "C458", + message: INVALID_NAMESPACE.to_owned(), + help: None, + }) +} + +pub(super) fn candidates(values: [Option<&str>; 4]) -> NamespaceCandidates { + let [cli, env, local_config, project_config] = values.map(|value| value.map(str::to_owned)); + NamespaceCandidates { + cli, + env, + local_config, + project_config, + } +} + +pub(super) fn selected_names( + files: &[ScenarioFile], + selected: Result, ScenarioCommandError>, +) -> Result, (&'static str, String)> { + selected + .map(|indices| { + indices + .into_iter() + .map(|index| files[index].name.clone()) + .collect() + }) + .map_err(|error| (error.code, error.message)) +} diff --git a/crates/sqlbuild-core/src/scenario_command/tests/test_scenario_command.rs b/crates/sqlbuild-core/src/scenario_command/tests/test_scenario_command.rs new file mode 100644 index 0000000000..5bdb8de153 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/tests/test_scenario_command.rs @@ -0,0 +1,515 @@ +use std::path::Path; + +use crate::scenario_command::main::format_captured_at::format_captured_at; +use crate::scenario_command::main::render_namespace_header::render_namespace_header; +use crate::scenario_command::main::resolve_scenario_namespace::resolve_scenario_namespace; +use crate::scenario_command::main::resolve_snapshot_limits::resolve_snapshot_limits; +use crate::scenario_command::main::scenario_activity_message::scenario_activity_message; +use crate::scenario_command::main::select_scenarios::select_scenarios; +use crate::scenario_command::main::snapshot_capture_warning::snapshot_capture_warning; +use crate::scenario_command::main::sql_analysis_required_error::sql_analysis_required_error; +use crate::scenario_command::main::validate_scenario_test_flags::validate_scenario_test_flags; +use crate::scenario_command::models::{ + ScenarioNamespace, ScenarioSelection, ScenarioTestFlags, SnapshotLimitConfig, + SnapshotLimitInputs, SnapshotLimits, SqlAnalysisCommand, +}; +use crate::scenario_command::tests::helpers::{ + PROJECT_DIR, candidates, invalid_namespace, namespace, scenario_files, selected_names, strings, +}; +use crate::scenario_command::tests::test_types::{ + CapturedAtTestCase, FlagsTestCase, LimitsTestCase, NamespaceTestCase, NamespaceTextTestCase, + SelectionTestCase, SqlAnalysisTestCase, WarningTestCase, +}; + +#[test] +fn given_namespace_sources_when_resolving_then_highest_precedence_wins() { + let long: String = "x".repeat(128); + let too_long: String = "x".repeat(129); + let test_cases = [ + NamespaceTestCase { + description: "CLI wins", + candidates: candidates([Some("job-1"), Some("job-2"), Some("local"), Some("project")]), + expected_namespace: namespace("job-1", "cli"), + }, + NamespaceTestCase { + description: "environment wins", + candidates: candidates([None, Some("job-2"), Some("local"), Some("project")]), + expected_namespace: namespace("job-2", "env"), + }, + NamespaceTestCase { + description: "local wins", + candidates: candidates([None, None, Some("local"), Some("project")]), + expected_namespace: namespace("local", "local config"), + }, + NamespaceTestCase { + description: "project fallback", + candidates: candidates([None, None, None, Some("project")]), + expected_namespace: namespace("project", "project config"), + }, + NamespaceTestCase { + description: "unset preserves default", + candidates: candidates([None; 4]), + expected_namespace: Ok(ScenarioNamespace { + value: None, + source: "unset".to_owned(), + }), + }, + NamespaceTestCase { + description: "CLI overrides invalid environment", + candidates: candidates([Some("job-1"), Some(""), None, None]), + expected_namespace: namespace("job-1", "cli"), + }, + NamespaceTestCase { + description: "empty environment does not fall back", + candidates: candidates([None, Some(""), None, Some("project")]), + expected_namespace: invalid_namespace(), + }, + NamespaceTestCase { + description: "all allowed characters", + candidates: candidates([Some("AZaz09-_."), None, None, None]), + expected_namespace: namespace("AZaz09-_.", "cli"), + }, + NamespaceTestCase { + description: "maximum length", + candidates: candidates([Some(&long), None, None, None]), + expected_namespace: namespace(&long, "cli"), + }, + NamespaceTestCase { + description: "too long", + candidates: candidates([Some(&too_long), None, None, None]), + expected_namespace: invalid_namespace(), + }, + NamespaceTestCase { + description: "whitespace", + candidates: candidates([Some(" "), None, None, None]), + expected_namespace: invalid_namespace(), + }, + NamespaceTestCase { + description: "slash", + candidates: candidates([Some("job/1"), None, None, None]), + expected_namespace: invalid_namespace(), + }, + NamespaceTestCase { + description: "colon", + candidates: candidates([Some("job:1"), None, None, None]), + expected_namespace: invalid_namespace(), + }, + NamespaceTestCase { + description: "trailing newline", + candidates: candidates([Some("job\n"), None, None, None]), + expected_namespace: invalid_namespace(), + }, + NamespaceTestCase { + description: "non ASCII", + candidates: candidates([Some("é"), None, None, None]), + expected_namespace: invalid_namespace(), + }, + ]; + + for test_case in test_cases { + assert_eq!( + resolve_scenario_namespace(test_case.candidates), + test_case.expected_namespace, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_namespace_when_rendering_progress_then_names_value_and_source() { + let test_cases = [ + NamespaceTextTestCase { + description: "a set namespace", + namespace: ScenarioNamespace { + value: Some("job-1".to_owned()), + source: "cli".to_owned(), + }, + expected_header: "Scenario namespace: job-1 (source: cli)\n", + expected_activity: "Running scenarios... (namespace: job-1; source: cli)", + }, + NamespaceTextTestCase { + description: "no namespace", + namespace: ScenarioNamespace { + value: None, + source: "unset".to_owned(), + }, + expected_header: "Scenario namespace: (unset) (source: unset)\n", + expected_activity: "Running scenarios... (namespace: (unset); source: unset)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + render_namespace_header(&test_case.namespace), + scenario_activity_message("Running scenarios...", &test_case.namespace) + ), + ( + test_case.expected_header.to_owned(), + test_case.expected_activity.to_owned() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_scenario_selectors_when_selecting_then_returns_matching_scenarios() { + let absolute_file: String = format!("{PROJECT_DIR}/tests/scenarios/orders/paid.sql"); + let absolute_folder: String = format!("{PROJECT_DIR}/tests/scenarios/orders"); + let test_cases = [ + SelectionTestCase { + scenario_files: scenario_files(), + description: "no selectors select every scenario", + selectors: Vec::new(), + exclude: Vec::new(), + expected_names: Ok(vec!["orders_paid", "orders_refund", "customers_new"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "excludes scenarios from explicit selection", + selectors: strings(&["tests/scenarios"]), + exclude: strings(&["orders_refund"]), + expected_names: Ok(vec!["orders_paid", "customers_new"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "selector order wins and duplicates are dropped", + selectors: strings(&["customers_new", "orders_paid", "customers_new"]), + exclude: Vec::new(), + expected_names: Ok(vec!["customers_new", "orders_paid"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "a folder relative to the scenario root", + selectors: strings(&["orders/"]), + exclude: Vec::new(), + expected_names: Ok(vec!["orders_paid", "orders_refund"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "an absolute folder", + selectors: vec![absolute_folder], + exclude: Vec::new(), + expected_names: Ok(vec!["orders_paid", "orders_refund"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "a file relative to the project", + selectors: strings(&["./tests/scenarios/orders/refund.sql"]), + exclude: Vec::new(), + expected_names: Ok(vec!["orders_refund"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "a file relative to the scenario root", + selectors: strings(&["customers_new.sql"]), + exclude: Vec::new(), + expected_names: Ok(vec!["customers_new"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "an absolute file selects only that scenario", + selectors: vec![absolute_file], + exclude: Vec::new(), + expected_names: Ok(vec!["orders_paid"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "excluding a folder", + selectors: Vec::new(), + exclude: strings(&["orders"]), + expected_names: Ok(vec!["customers_new"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "the project folder selects every scenario", + selectors: strings(&["."]), + exclude: Vec::new(), + expected_names: Ok(vec!["orders_paid", "orders_refund", "customers_new"]), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "unknown selector", + selectors: strings(&["missing"]), + exclude: Vec::new(), + expected_names: Err(("C453", "Unknown scenario selector 'missing'")), + }, + SelectionTestCase { + scenario_files: Vec::new(), + description: "no discovered scenarios", + selectors: Vec::new(), + exclude: Vec::new(), + expected_names: Err(( + "C451", + "No SQL scenarios were discovered under tests/scenarios", + )), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "unknown exclusion", + selectors: Vec::new(), + exclude: strings(&["missing.sql"]), + expected_names: Err(("C453", "Unknown scenario selector 'missing.sql'")), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "rejects upstream graph selector", + selectors: strings(&["+orders_paid"]), + exclude: Vec::new(), + expected_names: Err(( + "C457", + "Scenario selector '+orders_paid' uses graph operators, which are not supported", + )), + }, + SelectionTestCase { + scenario_files: scenario_files(), + description: "rejects path graph selector", + selectors: strings(&["orders_paid~orders_refund"]), + exclude: Vec::new(), + expected_names: Err(( + "C457", + "Scenario selector 'orders_paid~orders_refund' uses graph operators, which are \ + not supported", + )), + }, + ]; + for test_case in test_cases { + let selected = select_scenarios( + &test_case.scenario_files, + ScenarioSelection { + selectors: &test_case.selectors, + exclude: &test_case.exclude, + project_dir: Path::new(PROJECT_DIR), + }, + ); + assert_eq!( + selected_names(&test_case.scenario_files, selected), + test_case + .expected_names + .map(|names| names.into_iter().map(str::to_owned).collect()) + .map_err(|(code, message)| (code, message.to_owned())), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_scenario_test_flags_when_validating_then_rejects_unsupported_combinations() { + let test_cases = [ + FlagsTestCase { + description: "local with retain", + flags: ScenarioTestFlags { + local: true, + retain: true, + ..ScenarioTestFlags::default() + }, + expected_code: Some("C452"), + }, + FlagsTestCase { + description: "sync without local", + flags: ScenarioTestFlags { + sync_snapshots: true, + ..ScenarioTestFlags::default() + }, + expected_code: Some("C454"), + }, + FlagsTestCase { + description: "refresh without local", + flags: ScenarioTestFlags { + refresh: true, + ..ScenarioTestFlags::default() + }, + expected_code: Some("C454"), + }, + FlagsTestCase { + description: "local refresh", + flags: ScenarioTestFlags { + local: true, + refresh: true, + ..ScenarioTestFlags::default() + }, + expected_code: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + validate_scenario_test_flags(test_case.flags) + .err() + .map(|error| error.code), + test_case.expected_code, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_analysis_off_when_checking_scenario_command_then_explains_how_to_turn_it_on() { + let test_cases = [ + SqlAnalysisTestCase { + description: "local replay turned off by flag", + command: SqlAnalysisCommand::LocalTest, + disabled_by_flag: true, + guidance: None, + expected_error: Some(( + "scenario test --local requires SQL analysis, but `--no-sql-analysis` turns it off", + "run the command without `--no-sql-analysis`", + )), + }, + SqlAnalysisTestCase { + description: "capture turned off by flag", + command: SqlAnalysisCommand::Capture, + disabled_by_flag: true, + guidance: None, + expected_error: Some(( + "scenario capture requires SQL analysis, but `--no-sql-analysis` turns it off", + "run the capture without `--no-sql-analysis`", + )), + }, + SqlAnalysisTestCase { + description: "capture turned off by config", + command: SqlAnalysisCommand::Capture, + disabled_by_flag: false, + guidance: Some(( + "the project turns it off".to_owned(), + "turn it on".to_owned(), + )), + expected_error: Some(( + "scenario capture requires SQL analysis; the project turns it off", + "turn it on", + )), + }, + SqlAnalysisTestCase { + description: "analysis on", + command: SqlAnalysisCommand::LocalTest, + disabled_by_flag: false, + guidance: None, + expected_error: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + sql_analysis_required_error( + test_case.command, + test_case.disabled_by_flag, + test_case.guidance + ) + .map(|error| (error.code, error.message, error.help)), + test_case.expected_error.map(|(message, help)| ( + "C455", + message.to_owned(), + Some(help.to_owned()) + )), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_limit_sources_when_resolving_then_cli_beats_config_beats_default() { + let test_cases = [ + LimitsTestCase { + description: "defaults", + inputs: SnapshotLimitInputs::default(), + config: SnapshotLimitConfig::default(), + expected_limits: SnapshotLimits { + max_rows_per_relation: 10_000, + max_total_rows: 50_000, + max_bytes_per_relation: 5_000_000, + max_total_bytes: 25_000_000, + force: false, + }, + }, + LimitsTestCase { + description: "cli then config", + inputs: SnapshotLimitInputs { + max_rows_per_relation: Some(5), + force: true, + ..SnapshotLimitInputs::default() + }, + config: SnapshotLimitConfig { + max_rows_per_relation: Some(1), + max_total_bytes: Some(10), + ..SnapshotLimitConfig::default() + }, + expected_limits: SnapshotLimits { + max_rows_per_relation: 5, + max_total_rows: 50_000, + max_bytes_per_relation: 5_000_000, + max_total_bytes: 10, + force: true, + }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + resolve_snapshot_limits(test_case.inputs, test_case.config), + test_case.expected_limits, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_epoch_seconds_when_formatting_capture_time_then_writes_utc_seconds() { + let test_cases = [ + CapturedAtTestCase { + description: "epoch", + epoch_seconds: 0, + expected_text: "1970-01-01T00:00:00Z", + }, + CapturedAtTestCase { + description: "leap day", + epoch_seconds: 1_709_210_096, + expected_text: "2024-02-29T12:34:56Z", + }, + CapturedAtTestCase { + description: "end of year", + epoch_seconds: 1_798_761_599, + expected_text: "2026-12-31T23:59:59Z", + }, + ]; + + for test_case in test_cases { + assert_eq!( + format_captured_at(test_case.epoch_seconds), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_force_when_warning_about_snapshots_then_mentions_bypassed_limits() { + let test_cases = [ + WarningTestCase { + description: "limits enforced", + force: false, + expected_text: "Review captured scenario snapshots before committing; they may contain sensitive warehouse data.", + }, + WarningTestCase { + description: "limits bypassed", + force: true, + expected_text: "Review captured scenario snapshots before committing; they may contain sensitive warehouse data. Size limits are bypassed.", + }, + ]; + + for test_case in test_cases { + assert_eq!( + snapshot_capture_warning(test_case.force), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/scenario_command/tests/test_scenario_rows.rs b/crates/sqlbuild-core/src/scenario_command/tests/test_scenario_rows.rs new file mode 100644 index 0000000000..60c710811b --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/tests/test_scenario_rows.rs @@ -0,0 +1,339 @@ +use std::path::PathBuf; + +use crate::cli_text::models::CliStyle; +use crate::scenario_command::main::build_local_runtime_artifacts::build_local_runtime_artifacts; +use crate::scenario_command::main::build_runtime_artifacts::build_runtime_artifacts; +use crate::scenario_command::main::render_capture_result::render_capture_result; +use crate::scenario_command::main::render_scenario_result::render_scenario_result; +use crate::scenario_command::main::render_snapshot_sync_result::render_snapshot_sync_result; +use crate::scenario_command::models::{ + CaptureRow, ExpectedRow, FixtureStatements, FunctionStatements, LoadedRelation, + LocalRuntimeInputs, ModelArtifactEntry, NamedStatements, RetainedInputs, ScenarioResultRow, + WarehouseRuntimeInputs, +}; +use crate::scenario_command::tests::helpers::{ + artifact_pairs, coded, expectation_sql, expected_pairs, failing_result_row, relation, retained, + statements, strings, +}; +use crate::scenario_command::tests::test_types::{ + CaptureRowTestCase, LocalArtifactsTestCase, ResultRowTestCase, WarehouseArtifactsTestCase, +}; + +const RA0: &str = "orders_paid FAIL 1.23s\n error[X506]: expected comparison failed\n second line\n = help: Compare the CTE.\n expect expected fact_orders FAIL 2 mismatched\n error[X506]: mismatch\n = help: look\n expect expected dim_customers PASS\n expect assertion no_negative FAIL 1 row\n expect assertion few FAIL 3 rows\n plain error\n expect assertion ok PASS\n Retained relations:\n source raw_orders -> main.__sqb_abc__source__raw_orders\n model fact_orders -> main.__sqb_abc__model__fact_orders\n expected fact_orders: actual=3 expected=2 mismatched=2\n assertion no_negative: failing_rows=1\n sample: (1, 'a', None)\n assertion few: failing_rows=3\n"; +const RA1: &str = "orders_paid \u{1b}[38;5;167mFAIL\u{1b}[0m 1.23s\n \u{1b}[38;5;167m\u{1b}[1merror[X506]:\u{1b}[0m expected comparison failed\n \u{1b}[2msecond line\u{1b}[0m\n \u{1b}[2m= help:\u{1b}[0m Compare the CTE.\n expect expected fact_orders \u{1b}[38;5;167mFAIL\u{1b}[0m 2 mismatched\n \u{1b}[38;5;167m\u{1b}[1merror[X506]:\u{1b}[0m mismatch\n \u{1b}[2m= help:\u{1b}[0m look\n expect expected dim_customers \u{1b}[32mPASS\u{1b}[0m\n expect assertion no_negative \u{1b}[38;5;167mFAIL\u{1b}[0m 1 row\n expect assertion few \u{1b}[38;5;167mFAIL\u{1b}[0m 3 rows\n plain error\n expect assertion ok \u{1b}[32mPASS\u{1b}[0m\n Retained relations:\n source raw_orders -> main.__sqb_abc__source__raw_orders\n model fact_orders -> main.__sqb_abc__model__fact_orders\n expected fact_orders: actual=3 expected=2 mismatched=2\n assertion no_negative: failing_rows=1\n sample: (1, 'a', None)\n assertion few: failing_rows=3\n"; +const RB0: &str = "orders_local ERROR\n snapshot missing\n Retained local DuckDB: /work/target/run/scenarios/orders_local/local.duckdb\n"; +const RC0: &str = "orders_warehouse FAIL\n boom\n Rerun with --retain to inspect scenario-owned artifacts.\n"; +const RD1: &str = "orders_ok \u{1b}[32mPASS\u{1b}[0m 0.01s\n expect expected m \u{1b}[32mPASS\u{1b}[0m\n"; +const CE0: &str = "orders_paid PASS 2 relations, 3 rows\n source raw_orders PASS 2 rows, 41 B\n seed countries PASS 1 row, 1.5 KB\n snapshot tests/_scenario_snapshots/orders_paid/scenario.json\n"; +const SE0R: &str = "orders_paid REFRESHED 2 relations, 3 rows\n source raw_orders PASS 2 rows, 41 B\n seed countries PASS 1 row, 1.5 KB\n snapshot tests/_scenario_snapshots/orders_paid/scenario.json\n"; +const CF1: &str = "orders_big \u{1b}[38;5;167mFAIL\u{1b}[0m 0 relations, 0 rows\n \u{1b}[38;5;167m\u{1b}[1merror[X512]:\u{1b}[0m too many rows\n \u{1b}[2m= help:\u{1b}[0m narrow it\n Rerun with --retain to inspect scenario-owned artifacts.\n source raw_orders \u{1b}[38;5;167mFAIL\u{1b}[0m 0 rows, 0 B\n snapshot /elsewhere/scenario.json\n"; +const SF1: &str = "orders_big \u{1b}[38;5;167mFAIL\u{1b}[0m 0 relations, 0 rows\n \u{1b}[38;5;167m\u{1b}[1merror[X512]:\u{1b}[0m too many rows\n \u{1b}[2m= help:\u{1b}[0m narrow it\n source raw_orders \u{1b}[38;5;167mFAIL\u{1b}[0m 0 rows, 0 B\n snapshot /elsewhere/scenario.json\n"; +const CG0: &str = "orders_kept FAIL\n no capture\n Retained relations:\n source raw_orders -> main.__sqb_x__source__raw_orders\n seed countries\n"; +const SG0: &str = + "orders_kept FAIL\n no capture\n"; +const ASSERTION_SQL: &str = "SELECT COUNT(*) FROM (SELECT * FROM t WHERE x < 0) AS __scenario_assertion_failures;\n\nSELECT * FROM (SELECT * FROM t WHERE x < 0) AS __scenario_assertion_failures\nLIMIT 10;"; + +#[test] +fn given_scenario_result_when_rendering_then_matches_result_row_layout() { + let test_cases = [ + ResultRowTestCase { + description: "failing retained run without color", + row: failing_result_row(), + use_color: false, + expected_text: RA0, + }, + ResultRowTestCase { + description: "failing retained run styles statuses semantically", + row: failing_result_row(), + use_color: true, + expected_text: RA1, + }, + ResultRowTestCase { + description: "local error keeps its DuckDB path without a retain hint", + row: ScenarioResultRow { + name: "orders_local".to_owned(), + status: "ERROR".to_owned(), + local: true, + retained: true, + local_duckdb_path: Some( + "/work/target/run/scenarios/orders_local/local.duckdb".to_owned(), + ), + error: coded(None, "snapshot missing", None), + ..ScenarioResultRow::default() + }, + use_color: false, + expected_text: RB0, + }, + ResultRowTestCase { + description: "warehouse error suggests retaining", + row: ScenarioResultRow { + name: "orders_warehouse".to_owned(), + status: "FAIL".to_owned(), + error: coded(None, "boom", None), + ..ScenarioResultRow::default() + }, + use_color: false, + expected_text: RC0, + }, + ResultRowTestCase { + description: "passing run rounds its duration", + row: ScenarioResultRow { + name: "orders_ok".to_owned(), + status: "PASS".to_owned(), + duration_ms: Some(5.0), + expected: vec![ExpectedRow { + model_name: "m".to_owned(), + ..ExpectedRow::default() + }], + ..ScenarioResultRow::default() + }, + use_color: true, + expected_text: RD1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + render_scenario_result( + &test_case.row, + CliStyle { + use_color: test_case.use_color + } + ), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_capture_result_when_rendering_then_matches_capture_and_sync_rows() { + let test_cases = [ + CaptureRowTestCase { + description: "captured relations with sizes", + row: CaptureRow { + name: "orders_paid".to_owned(), + passed: true, + relations: Some(vec![ + relation("source", "raw_orders", (true, 2, 41)), + relation("seed", "countries", (true, 1, 1536)), + ]), + snapshot_path: Some( + "tests/_scenario_snapshots/orders_paid/scenario.json".to_owned(), + ), + ..CaptureRow::default() + }, + use_color: false, + refresh: true, + expected_capture_text: CE0, + expected_sync_text: SE0R, + }, + CaptureRowTestCase { + description: "a coded failure with color", + row: CaptureRow { + name: "orders_big".to_owned(), + error: coded(Some("X512"), "too many rows", Some("narrow it")), + relations: Some(vec![relation("source", "raw_orders", (false, 0, 0))]), + snapshot_path: Some("/elsewhere/scenario.json".to_owned()), + ..CaptureRow::default() + }, + use_color: true, + refresh: false, + expected_capture_text: CF1, + expected_sync_text: SF1, + }, + CaptureRowTestCase { + description: "retained inputs without a snapshot", + row: CaptureRow { + name: "orders_kept".to_owned(), + error: coded(None, "no capture", None), + retained: true, + retained_inputs: Some(RetainedInputs { + fixtures: vec![retained( + "source", + "raw_orders", + "main.__sqb_x__source__raw_orders", + )], + seeds: strings(&["countries"]), + }), + ..CaptureRow::default() + }, + use_color: false, + refresh: false, + expected_capture_text: CG0, + expected_sync_text: SG0, + }, + ]; + + for test_case in test_cases { + let style: CliStyle = CliStyle { + use_color: test_case.use_color, + }; + assert_eq!( + ( + render_capture_result(&test_case.row, style), + render_snapshot_sync_result(&test_case.row, test_case.refresh, style) + ), + ( + test_case.expected_capture_text.to_owned(), + test_case.expected_sync_text.to_owned() + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_warehouse_run_when_building_artifacts_then_orders_and_terminates_statements() { + let test_cases = [WarehouseArtifactsTestCase { + description: "every lifecycle stage, skipping empty and unplanned SQL", + inputs: WarehouseRuntimeInputs { + prepare_cleanup: Some(statements(&["DROP TABLE a", "DROP TABLE b; \n"])), + fixtures: vec![ + FixtureStatements { + kind: "source".to_owned(), + logical_name: "raw/orders".to_owned(), + statements: statements(&["CREATE TABLE f AS SELECT 1"]), + }, + FixtureStatements { + kind: "source".to_owned(), + logical_name: "empty".to_owned(), + statements: Vec::new(), + }, + ], + seeds: vec![NamedStatements { + name: "countries".to_owned(), + statements: statements(&["INSERT INTO c VALUES (1)", " "]), + }], + models: vec![ + NamedStatements { + name: "orders".to_owned(), + statements: statements(&["CREATE TABLE o AS SELECT 1"]), + }, + NamedStatements { + name: "staging".to_owned(), + statements: statements(&["CREATE VIEW s AS SELECT 1"]), + }, + NamedStatements { + name: "unplanned".to_owned(), + statements: statements(&["SELECT 1"]), + }, + ], + model_entries: vec![ + ModelArtifactEntry { + name: "orders".to_owned(), + relative_path: PathBuf::from("models/marts/orders.sql"), + }, + ModelArtifactEntry { + name: "staging".to_owned(), + relative_path: PathBuf::from("staging.sql"), + }, + ], + expectations: expectation_sql(), + final_cleanup: Some(Vec::new()), + }, + expected_artifacts: vec![ + ( + PathBuf::from("cleanup/prepare.sql"), + "DROP TABLE a;\n\nDROP TABLE b;", + ), + ( + PathBuf::from("fixtures/source__raw__orders.sql"), + "CREATE TABLE f AS SELECT 1;", + ), + ( + PathBuf::from("seeds/countries.sql"), + "INSERT INTO c VALUES (1);\n\n ", + ), + ( + PathBuf::from("models/marts/orders.sql"), + "CREATE TABLE o AS SELECT 1;", + ), + ( + PathBuf::from("models/staging.sql"), + "CREATE VIEW s AS SELECT 1;", + ), + ( + PathBuf::from("expectations/expected__order_totals.sql"), + "SELECT 1 EXCEPT SELECT 2", + ), + ( + PathBuf::from("expectations/assertion__no_negative.sql"), + ASSERTION_SQL, + ), + ], + }]; + + for test_case in test_cases { + assert_eq!( + artifact_pairs(build_runtime_artifacts(&test_case.inputs)), + expected_pairs(&test_case.expected_artifacts), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_local_replay_when_building_artifacts_then_writes_loads_functions_and_models() { + let test_cases = [LocalArtifactsTestCase { + description: "loaded snapshots, functions under their language and model SQL", + inputs: LocalRuntimeInputs { + loaded_relations: vec![LoadedRelation { + kind: "source".to_owned(), + logical_name: "raw_orders".to_owned(), + file_path: "/p/tests/_scenario_snapshots/s/sources/raw_orders.jsonl".to_owned(), + table_name: "__sqb_local__source__raw_orders".to_owned(), + row_count: 2, + }], + functions: vec![ + FunctionStatements { + relative_path: PathBuf::from("functions/sql/add_one.sql"), + language: "sql".to_owned(), + statements: statements(&["CREATE MACRO add_one(x) AS x + 1"]), + }, + FunctionStatements { + relative_path: PathBuf::from("util/score.py"), + language: "python".to_owned(), + statements: statements(&["CREATE FUNCTION score()", "SELECT 1;"]), + }, + ], + models: vec![(PathBuf::from("orders.sql"), "SELECT 1\n\n".to_owned())], + expectations: expectation_sql(), + }, + expected_artifacts: vec![ + ( + PathBuf::from("fixtures/source__raw_orders.sql"), + "-- loaded from /p/tests/_scenario_snapshots/s/sources/raw_orders.jsonl\n-- rows: 2\nCREATE TABLE \"__sqb_local__source__raw_orders\" AS SELECT * FROM '/p/tests/_scenario_snapshots/s/sources/raw_orders.jsonl';", + ), + ( + PathBuf::from("functions/sql/add_one.sql"), + "CREATE MACRO add_one(x) AS x + 1;", + ), + ( + PathBuf::from("functions/python/util/score.sql"), + "CREATE FUNCTION score();\n\nSELECT 1;", + ), + (PathBuf::from("models/orders.sql"), "SELECT 1\n\n"), + ( + PathBuf::from("expectations/expected__order_totals.sql"), + "SELECT 1 EXCEPT SELECT 2", + ), + ( + PathBuf::from("expectations/assertion__no_negative.sql"), + ASSERTION_SQL, + ), + ], + }]; + + for test_case in test_cases { + assert_eq!( + artifact_pairs(build_local_runtime_artifacts(&test_case.inputs)), + expected_pairs(&test_case.expected_artifacts), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/scenario_command/tests/test_types.rs b/crates/sqlbuild-core/src/scenario_command/tests/test_types.rs new file mode 100644 index 0000000000..299696eba4 --- /dev/null +++ b/crates/sqlbuild-core/src/scenario_command/tests/test_types.rs @@ -0,0 +1,90 @@ +use std::path::PathBuf; + +use crate::scenario_command::errors::ScenarioCommandError; +use crate::scenario_command::models::{ + CaptureRow, LocalRuntimeInputs, NamespaceCandidates, ScenarioFile, ScenarioNamespace, + ScenarioResultRow, ScenarioTestFlags, SnapshotLimitConfig, SnapshotLimitInputs, SnapshotLimits, + SqlAnalysisCommand, WarehouseRuntimeInputs, +}; + +pub(super) struct NamespaceTestCase { + pub(super) description: &'static str, + pub(super) candidates: NamespaceCandidates, + pub(super) expected_namespace: Result, +} + +pub(super) struct SelectionTestCase { + pub(super) description: &'static str, + pub(super) scenario_files: Vec, + pub(super) selectors: Vec, + pub(super) exclude: Vec, + pub(super) expected_names: Result, (&'static str, &'static str)>, +} + +pub(super) struct FlagsTestCase { + pub(super) description: &'static str, + pub(super) flags: ScenarioTestFlags, + pub(super) expected_code: Option<&'static str>, +} + +pub(super) struct LimitsTestCase { + pub(super) description: &'static str, + pub(super) inputs: SnapshotLimitInputs, + pub(super) config: SnapshotLimitConfig, + pub(super) expected_limits: SnapshotLimits, +} + +pub(super) struct CapturedAtTestCase { + pub(super) description: &'static str, + pub(super) epoch_seconds: i64, + pub(super) expected_text: &'static str, +} + +pub(super) struct ResultRowTestCase { + pub(super) description: &'static str, + pub(super) row: ScenarioResultRow, + pub(super) use_color: bool, + pub(super) expected_text: &'static str, +} + +pub(super) struct CaptureRowTestCase { + pub(super) description: &'static str, + pub(super) row: CaptureRow, + pub(super) use_color: bool, + pub(super) refresh: bool, + pub(super) expected_capture_text: &'static str, + pub(super) expected_sync_text: &'static str, +} + +pub(super) struct WarehouseArtifactsTestCase { + pub(super) description: &'static str, + pub(super) inputs: WarehouseRuntimeInputs, + pub(super) expected_artifacts: Vec<(PathBuf, &'static str)>, +} + +pub(super) struct LocalArtifactsTestCase { + pub(super) description: &'static str, + pub(super) inputs: LocalRuntimeInputs, + pub(super) expected_artifacts: Vec<(PathBuf, &'static str)>, +} + +pub(super) struct NamespaceTextTestCase { + pub(super) description: &'static str, + pub(super) namespace: ScenarioNamespace, + pub(super) expected_header: &'static str, + pub(super) expected_activity: &'static str, +} + +pub(super) struct SqlAnalysisTestCase { + pub(super) description: &'static str, + pub(super) command: SqlAnalysisCommand, + pub(super) disabled_by_flag: bool, + pub(super) guidance: Option<(String, String)>, + pub(super) expected_error: Option<(&'static str, &'static str)>, +} + +pub(super) struct WarningTestCase { + pub(super) description: &'static str, + pub(super) force: bool, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs index 7e429641a5..1140339494 100644 --- a/crates/sqlbuild-core/src/text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod cleandoc; pub(crate) mod decimal_digits; pub(crate) mod glob; pub(crate) mod python_tables; +pub(crate) mod text_wrap; diff --git a/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs b/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs new file mode 100644 index 0000000000..28ed46aa88 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/text_wrap.rs @@ -0,0 +1,199 @@ +//! Python's `textwrap.wrap` with its default options. + +use crate::text::constants::{MINIMUM_EM_DASH, TAB_SIZE}; +use crate::text::main::is_decimal_digit::is_decimal_digit; +use crate::text::main::python_isalnum::python_isalnum; + +/// `text.expandtabs()` with every remaining whitespace character replaced by a space. +fn munge_whitespace(text: &str) -> Vec { + let mut munged: Vec = Vec::with_capacity(text.len()); + let mut column: usize = 0; + for character in text.chars() { + match character { + '\t' => { + let spaces: usize = TAB_SIZE - column % TAB_SIZE; + munged.extend(std::iter::repeat_n(' ', spaces)); + column += spaces; + } + '\n' | '\r' => { + munged.push(' '); + column = 0; + } + '\x0b' | '\x0c' => { + munged.push(' '); + column += 1; + } + _ => { + munged.push(character); + column += 1; + } + } + } + munged +} + +/// `textwrap.wrap(text, width, initial_indent=..., subsequent_indent=...)`. +pub(crate) fn wrap(text: &str, width: usize, (initial, subsequent): (&str, &str)) -> Vec { + let characters: Vec = munge_whitespace(text); + let mut chunks: Vec> = split_chunks(&characters); + chunks.reverse(); + let mut lines: Vec = Vec::new(); + while !chunks.is_empty() { + let mut line: Vec> = Vec::new(); + let mut length: usize = 0; + let indent: &str = if lines.is_empty() { + initial + } else { + subsequent + }; + let available: i64 = width as i64 - indent.chars().count() as i64; + if !lines.is_empty() && chunks.last().is_some_and(|chunk| is_blank(chunk)) { + chunks.pop(); + } + while let Some(chunk) = chunks.last() { + if (length + chunk.len()) as i64 <= available { + length += chunk.len(); + line.push(chunks.pop().unwrap_or_default()); + } else { + break; + } + } + if let Some(chunk) = chunks.pop_if(|chunk| chunk.len() as i64 > available) { + let (head, rest) = split_long_word(chunk, line.is_empty(), length, available); + line.extend(head); + chunks.extend(rest); + } + if line.last().is_some_and(|chunk| is_blank(chunk)) { + line.pop(); + } + if !line.is_empty() { + let joined: String = line.iter().flatten().collect(); + lines.push(format!("{indent}{joined}")); + } + } + lines +} + +/// `_handle_long_word`: the part of a too-long chunk the line takes, and what stays queued. +fn split_long_word( + mut chunk: Vec, + line_is_empty: bool, + length: usize, + width: i64, +) -> (Option>, Option>) { + let space_left: i64 = if width < 1 { 1 } else { width - length as i64 }; + if space_left <= 0 { + return if line_is_empty { + (Some(chunk), None) + } else { + (None, Some(chunk)) + }; + } + let space_left: usize = space_left as usize; + let mut end: usize = space_left.min(chunk.len()); + if chunk.len() > space_left + && let Some(hyphen) = chunk[..space_left] + .iter() + .rposition(|character| *character == '-') + && hyphen > 0 + && chunk[..hyphen].iter().any(|character| *character != '-') + { + end = hyphen + 1; + } + let rest: Vec = chunk.split_off(end); + (Some(chunk), Some(rest)) +} + +fn is_blank(chunk: &[char]) -> bool { + chunk.iter().all(|character| character.is_whitespace()) +} + +fn is_space(character: char) -> bool { + matches!(character, '\t' | '\n' | '\x0b' | '\x0c' | '\r' | ' ') +} + +fn is_word(character: char) -> bool { + character == '_' || python_isalnum(character) +} + +fn is_word_punct(character: char) -> bool { + is_word(character) || matches!(character, '!' | '"' | '\'' | '&' | '.' | ',' | '?') +} + +fn is_letter(character: char) -> bool { + is_word(character) && !is_decimal_digit(character) +} + +/// `TextWrapper.wordsep_re.split`: whitespace runs, em-dashes and possibly hyphenated words. +fn split_chunks(text: &[char]) -> Vec> { + let mut chunks: Vec> = Vec::new(); + let mut start: usize = 0; + while start < text.len() { + let end: usize = chunk_end(text, start); + chunks.push(text[start..end].to_vec()); + start = end; + } + chunks +} + +fn at(text: &[char], index: usize) -> Option { + text.get(index).copied() +} + +fn letter_at(text: &[char], index: Option) -> bool { + index + .and_then(|index| at(text, index)) + .is_some_and(is_letter) +} + +/// The end of a `-{2,}(?=\w)` match at `start`, if there is one. +fn em_dash_end(text: &[char], start: usize) -> Option { + let count: usize = text[start..] + .iter() + .take_while(|character| **character == '-') + .count(); + (count >= MINIMUM_EM_DASH && at(text, start + count).is_some_and(is_word)) + .then_some(start + count) +} + +fn chunk_end(text: &[char], start: usize) -> usize { + if is_space(text[start]) { + return start + + text[start..] + .iter() + .take_while(|character| is_space(**character)) + .count(); + } + if start > 0 + && is_word_punct(text[start - 1]) + && let Some(end) = em_dash_end(text, start) + { + return end; + } + let mut position: usize = start + 1; + loop { + if at(text, position) == Some('-') { + let two_letters: bool = letter_at(text, position.checked_sub(2)) + && letter_at(text, position.checked_sub(1)); + let letter_hyphen_letter: bool = letter_at(text, position.checked_sub(3)) + && at(text, position.wrapping_sub(2)) == Some('-') + && letter_at(text, position.checked_sub(1)); + let ahead: bool = letter_at(text, Some(position + 1)) + && (letter_at(text, Some(position + 2)) + || (at(text, position + 2) == Some('-') + && letter_at(text, Some(position + 3)))); + if (two_letters || letter_hyphen_letter) && ahead { + return position + 1; + } + } + match at(text, position) { + None => return position, + Some(character) if is_space(character) => return position, + Some(_) => {} + } + if is_word_punct(text[position - 1]) && em_dash_end(text, position).is_some() { + return position; + } + position += 1; + } +} diff --git a/crates/sqlbuild-core/src/text/constants.rs b/crates/sqlbuild-core/src/text/constants.rs index 4e313e1612..0a531b1d2e 100644 --- a/crates/sqlbuild-core/src/text/constants.rs +++ b/crates/sqlbuild-core/src/text/constants.rs @@ -3,3 +3,9 @@ pub(crate) const DIGITS_PER_RUN: u32 = 10; /// The information separators Python's `str.isspace` accepts and `char::is_whitespace` does not. pub(crate) const PYTHON_SEPARATOR_CONTROLS: std::ops::RangeInclusive = '\u{1c}'..='\u{1f}'; + +/// `textwrap`'s em-dash is at least this many hyphens. +pub(crate) const MINIMUM_EM_DASH: usize = 2; + +/// The tab stop `str.expandtabs` uses by default. +pub(crate) const TAB_SIZE: usize = 8; diff --git a/crates/sqlbuild-core/src/text/main/cell_len.rs b/crates/sqlbuild-core/src/text/main/cell_len.rs new file mode 100644 index 0000000000..8bf86bde2f --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/cell_len.rs @@ -0,0 +1,9 @@ +//! The terminal cells a string occupies. + +use crate::text::main::char_cells::char_cells; + +/// The terminal cells `text` occupies. +#[must_use] +pub fn cell_len(text: &str) -> usize { + text.chars().map(char_cells).sum() +} diff --git a/crates/sqlbuild-core/src/text/main/char_cells.rs b/crates/sqlbuild-core/src/text/main/char_cells.rs new file mode 100644 index 0000000000..ee330698fb --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/char_cells.rs @@ -0,0 +1,35 @@ +//! The terminal cells one character occupies. + +/// The terminal cells one character occupies: zero for combining marks, two for wide characters. +#[must_use] +pub fn char_cells(character: char) -> usize { + let code: u32 = u32::from(character); + let zero_width: bool = matches!( + code, + 0x0300..=0x036F | 0x200B..=0x200F | 0x20D0..=0x20FF | 0xFE00..=0xFE0F | 0xFE20..=0xFE2F + ); + let wide: bool = matches!( + code, + 0x1100..=0x115F + | 0x2E80..=0x303E + | 0x3041..=0x33FF + | 0x3400..=0x4DBF + | 0x4E00..=0x9FFF + | 0xA000..=0xA4CF + | 0xAC00..=0xD7A3 + | 0xF900..=0xFAFF + | 0xFE30..=0xFE4F + | 0xFF00..=0xFF60 + | 0xFFE0..=0xFFE6 + | 0x1F300..=0x1F64F + | 0x1F900..=0x1F9FF + | 0x20000..=0x3FFFD + ); + if zero_width { + 0 + } else if wide { + 2 + } else { + 1 + } +} diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index a92cd0d2e8..b74967aa28 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -1,3 +1,5 @@ +pub mod cell_len; +pub mod char_cells; pub mod close_matches; pub mod decimal_digit_value; pub mod decode_text; @@ -15,3 +17,4 @@ pub mod python_isspace; pub mod python_splitlines; pub mod python_str_repr; pub mod python_strip; +pub mod text_wrap; diff --git a/crates/sqlbuild-core/src/text/main/text_wrap.rs b/crates/sqlbuild-core/src/text/main/text_wrap.rs new file mode 100644 index 0000000000..b2e32cccee --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/text_wrap.rs @@ -0,0 +1,9 @@ +//! Python's `textwrap.wrap` with its default options. + +use crate::text::_helpers::text_wrap::wrap; + +/// `textwrap.wrap(text, width, initial_indent=initial, subsequent_indent=subsequent)`. +#[must_use] +pub fn text_wrap(text: &str, width: usize, indents: (&str, &str)) -> Vec { + wrap(text, width, indents) +} diff --git a/crates/sqlbuild-core/src/text/tests.rs b/crates/sqlbuild-core/src/text/tests.rs index 58c350466d..2a14e5b9b9 100644 --- a/crates/sqlbuild-core/src/text/tests.rs +++ b/crates/sqlbuild-core/src/text/tests.rs @@ -6,3 +6,5 @@ mod python_str; mod test_types; #[path = "tests/test_text.rs"] mod text; +#[path = "tests/test_text_wrap.rs"] +mod text_wrap; diff --git a/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs b/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs new file mode 100644 index 0000000000..7aabeeb99a --- /dev/null +++ b/crates/sqlbuild-core/src/text/tests/test_text_wrap.rs @@ -0,0 +1,72 @@ +use crate::text::main::cell_len::cell_len; +use crate::text::main::text_wrap::text_wrap; +use crate::text::tests::test_types::{CellLenTestCase, TextWrapTestCase}; + +#[test] +fn given_prose_when_wrapping_then_lines_match_python_textwrap() { + let test_cases = [ + TextWrapTestCase { + description: "the first line takes the initial indent, later lines the subsequent one", + text: "add a description to the MODEL header", + width: 30, + indents: (" = help: ", " "), + expected_lines: &[ + " = help: add a description to", + " the MODEL header", + ], + }, + TextWrapTestCase { + description: "newlines and tabs become spaces", + text: "a\tb\nc", + width: 40, + indents: ("", ""), + expected_lines: &["a b c"], + }, + TextWrapTestCase { + description: "empty text has no lines", + text: "", + width: 10, + indents: ("> ", "> "), + expected_lines: &[], + }, + ]; + + for test_case in test_cases { + assert_eq!( + text_wrap(test_case.text, test_case.width, test_case.indents), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_text_when_measuring_cells_then_wide_characters_count_twice() { + let test_cases = [ + CellLenTestCase { + description: "ascii is one cell per character", + text: "orders", + expected_cells: 6, + }, + CellLenTestCase { + description: "CJK characters are wide", + text: "注文", + expected_cells: 4, + }, + CellLenTestCase { + description: "combining marks take no cells", + text: "e\u{301}", + expected_cells: 1, + }, + ]; + + for test_case in test_cases { + assert_eq!( + cell_len(test_case.text), + test_case.expected_cells, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs index a5fe685bbe..4d797997b7 100644 --- a/crates/sqlbuild-core/src/text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -65,3 +65,17 @@ pub(super) struct SplitlinesTestCase { pub(super) text: &'static str, pub(super) expected_lines: &'static [&'static str], } + +pub(super) struct TextWrapTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) width: usize, + pub(super) indents: (&'static str, &'static str), + pub(super) expected_lines: &'static [&'static str], +} + +pub(super) struct CellLenTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_cells: usize, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs new file mode 100644 index 0000000000..4f9d85d50d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -0,0 +1,506 @@ +//! `sqb audit`: compile the selected audits, run them with nested progress, and report. + +use std::collections::BTreeSet; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_core::audit_command::main::audit_exit_code::audit_exit_code; +use sqlbuild_core::audit_command::main::render_audit_header::render_audit_header; +use sqlbuild_core::audit_command::main::render_audit_summary::render_audit_summary; +use sqlbuild_core::audit_command::models::AuditCounts; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const AUDIT_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._audit"; +const PROGRESS_REPORTER_MODULE: &str = "sqlbuild.cli.progress.classes.audit_progress_reporter"; +const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; +const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; +const PROJECTION_MODULE: &str = "sqlbuild.executor.auditing.main._current_audit_result_projection"; +const EXECUTION_COMMAND: &str = "sqb audit"; +const PASS_OUTCOME: &str = "pass"; +const WARN_OUTCOME: &str = "warn"; +const ERROR_OUTCOME: &str = "error"; +const INSUFFICIENT_OUTCOME: &str = "insufficient"; +const PROJECTION_FAILURE: &str = "audit_result_projection_failure"; + +fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +/// Forward audit results to the event writer and connection timing to its reporter. +#[pyclass(module = "sqlbuild._native", frozen)] +struct AuditCallbackRelay { + event_writer: Py, + plan: Py, + connection: Py, +} + +#[pymethods] +impl AuditCallbackRelay { + fn on_result(&self, py: Python<'_>, result: Bound<'_, PyAny>) -> PyResult<()> { + call_with( + &self.event_writer.bind(py).getattr("write_build_result")?, + &[ + ("result", result), + ("plan", self.plan.bind(py).clone()), + ("command", text(py, "audit")), + ], + )?; + Ok(()) + } + + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.forward( + py, + "on_connection_complete", + (connection_count, elapsed_seconds), + ) + } + + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.forward( + py, + "on_connection_error", + (connection_count, elapsed_seconds), + ) + } +} + +impl AuditCallbackRelay { + fn forward( + &self, + py: Python<'_>, + method: &str, + (connection_count, elapsed_seconds): (i64, f64), + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("connection_count", connection_count)?; + options.set_item("elapsed_seconds", elapsed_seconds)?; + self.connection + .bind(py) + .call_method(method, (), Some(&options))?; + Ok(()) + } +} + +/// Compile the audit plan for the selected scope. +fn compile_audits<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let discovered: Bound<'py, PyAny> = invocation.getattr("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = invocation.getattr("adapter")?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", invocation.getattr("effective_project_dir")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &[ + ("selected_target", request.getattr("selected_target")?), + ("no_sql_validation", request.getattr("no_sql_validation")?), + ("defer_to", request.getattr("defer_to")?), + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("cli_vars", request.getattr("cli_vars")?), + ("external_sql_reference_resolver", resolver), + ], + )?; + let context: Bound<'py, PyAny> = call_with( + &imported(py, STATIC_COMMAND_MODULE, "compile_static_command_context")?, + &[ + ("discovered_inputs", discovered), + ("adapter", adapter.clone()), + ("options", options), + ( + "on_progress", + invocation + .getattr("planning_progress")? + .getattr("on_progress")?, + ), + ], + )?; + let plan_output: Bound<'py, PyAny> = call_with( + &imported(py, AUDIT_PLAN_MODULE, "build_audit_command_plan")?, + &[ + ("project", context.getattr("project")?), + ("adapter", adapter), + ("scope", context.getattr("scope")?), + ("relations", context.getattr("relations")?), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "build_static_pipeline_result")?, + &[("context", context), ("plan_output", plan_output)], + ) +} + +/// The audits run with their progress and connection reporters. +struct AuditRun<'py> { + progress: Bound<'py, PyAny>, + connection: Bound<'py, PyAny>, + concurrency: i64, + worker_count: i64, +} + +/// Create the nested progress and connection reporters before the execution header. +fn prepare_run<'py>( + request: &Bound<'py, PyAny>, + invocation: &Bound<'py, PyAny>, + pipeline: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let entries: Bound<'py, PyAny> = pipeline.getattr("plan_output")?.getattr("audit_entries")?; + let requested: Bound<'py, PyAny> = request.getattr("concurrency")?; + let concurrency: i64 = if requested.is_none() { + pipeline + .getattr("project")? + .getattr("settings")? + .getattr("concurrency")? + .extract::()? + } else { + requested.extract::()? + }; + let audit_count: usize = entries.len()?; + let worker_count: i64 = concurrency.min(i64::try_from(audit_count).unwrap_or(i64::MAX)); + let stream: Bound<'py, PyAny> = invocation.getattr("progress_stream")?; + let use_color: Bound<'py, PyAny> = invocation.getattr("use_color")?; + let progress: Bound<'py, PyAny> = call_with( + &imported(py, PROGRESS_REPORTER_MODULE, "AuditProgressReporter")?, + &[ + ("entries", entries.clone()), + ("worker_limit", PyInt::new(py, worker_count).into_any()), + ("stream", stream.clone()), + ("use_color", use_color.clone()), + ], + )?; + let projector: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + if !projector.is_none() { + for entry in entries.try_iter()? { + call_with( + &projector.getattr("expect_resource_enrichment")?, + &[("resource_name", entry?.getattr("name")?)], + )?; + } + } + let connection: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", invocation.getattr("adapter_name")?), + ("stream", stream), + ("use_color", use_color), + ], + )?; + Ok(AuditRun { + progress, + connection, + concurrency, + worker_count, + }) +} + +/// Write the execution header, then the selected audits header. +fn write_headers( + invocation: &Bound<'_, PyAny>, + pipeline: &Bound<'_, PyAny>, + concurrency: i64, +) -> PyResult<()> { + let py: Python<'_> = invocation.py(); + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let use_color: Bound<'_, PyAny> = invocation.getattr("use_color")?; + stream.call_method1("write", ("\n",))?; + let header: Bound<'_, PyDict> = PyDict::new(py); + header.set_item("stream", &stream)?; + header.set_item("command", EXECUTION_COMMAND)?; + header.set_item("target", py.None())?; + header.set_item("concurrency", concurrency)?; + header.set_item("use_color", &use_color)?; + imported(py, NATIVE_MODULE, "write_execution_header")?.call((), Some(&header))?; + let entries: Bound<'_, PyAny> = pipeline.getattr("plan_output")?.getattr("audit_entries")?; + let mut models: BTreeSet = BTreeSet::new(); + for entry in entries.try_iter()? { + let target: Bound<'_, PyAny> = entry?.getattr("attached_target_name")?; + if target.is_truthy()? { + models.insert(target.extract()?); + } + } + let style: CliStyle = CliStyle { + use_color: use_color.is_truthy()?, + }; + stream.call_method1( + "write", + (render_audit_header((entries.len()?, models.len()), style),), + )?; + stream.call_method0("flush")?; + Ok(()) +} + +/// Run the audits inside the run's cost scope; returns their results. +fn execute<'py>( + invocation: &Bound<'py, PyAny>, + pipeline: &Bound<'py, PyAny>, + run: &AuditRun<'py>, +) -> PyResult>> { + let py: Python<'py> = invocation.py(); + let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let event_writer: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "ExecutionEventWriter")?.call0()?; + let relay: Bound<'py, AuditCallbackRelay> = Bound::new( + py, + AuditCallbackRelay { + event_writer: event_writer.clone().unbind(), + plan: plan.clone().unbind(), + connection: run.connection.clone().unbind(), + }, + )?; + let executed: PyResult>> = (|| { + run.progress + .call_method1("set_result_callback", (relay.getattr("on_result")?,))?; + let projector: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "current_native_progress_projector")?.call0()?; + if !projector.is_none() { + let ordinals: Bound<'py, PyDict> = PyDict::new(py); + let mut total: usize = 0; + for entry in plan.getattr("audit_entries")?.try_iter()? { + total += 1; + ordinals.set_item(entry?.getattr("name")?, total)?; + } + call_with( + &projector.getattr("configure_resources")?, + &[ + ("ordinals", ordinals.into_any()), + ("total", PyInt::new(py, total).into_any()), + ], + )?; + } + let run_id: Bound<'py, PyAny> = project.getattr("run_id")?; + let ledger_path: Bound<'py, PyAny> = invocation + .getattr("effective_project_dir")? + .call_method1("__truediv__", ("target",))? + .call_method1("__truediv__", ("executions",))? + .call_method1("__truediv__", (&run_id,))? + .call_method1("__truediv__", ("statements.jsonl",))?; + let scope: Bound<'py, PyAny> = call_with( + &imported(py, COST_CONTEXT_MODULE, "CostContext")?.getattr("scope")?, + &[ + ("run_id", run_id.clone()), + ("resource_type", text(py, "run")), + ("resource_name", text(py, "audit")), + ("ledger_path", ledger_path), + ("phase", text(py, "audit")), + ], + )?; + within(&scope, |_| { + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_RUN_MODULE, "AuditPipelineCallbacks")?, + &[ + ( + "on_connection_start", + run.connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ("on_audit_start", run.progress.getattr("on_item_start")?), + ( + "on_audit_physical_complete", + run.progress.getattr("on_item_physical_complete")?, + ), + ( + "on_audit_complete", + run.progress.getattr("on_item_complete")?, + ), + ("on_audit_error", run.progress.getattr("on_item_error")?), + ], + )?; + call_with( + &imported(py, EXECUTOR_RUN_MODULE, "run_audit_pipeline_with_callbacks")?, + &[ + ("plan", plan.clone()), + ( + "connection_config", + invocation.getattr("connection_config")?, + ), + ("adapter", invocation.getattr("adapter")?), + ( + "max_concurrency", + PyInt::new(py, run.concurrency).into_any(), + ), + ("callbacks", callbacks), + ("run_id", run_id.clone()), + ( + "storage_database", + project.getattr("effective_target_database")?, + ), + ( + "storage_schema", + project.getattr("effective_target_schema")?, + ), + ], + )? + .try_iter()? + .collect() + }) + })(); + let progress_closed: PyResult> = run.progress.call_method0("close"); + let writer_closed: PyResult> = event_writer.call_method0("close"); + let results: Vec> = executed?; + progress_closed?; + writer_closed?; + Ok(results) +} + +/// The audit JSON document, with the result projection counts when audits were projected. +fn audit_payload<'py>( + results: &Bound<'py, PyAny>, + (concurrency, worker_count): (i64, i64), +) -> PyResult> { + let py: Python<'py> = results.py(); + let payload: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_audit_execution_json")?, + &[ + ("results", results.clone()), + ( + "configured_concurrency", + PyInt::new(py, concurrency).into_any(), + ), + ("worker_count", PyInt::new(py, worker_count).into_any()), + ], + )?; + let projection: Bound<'py, PyAny> = + imported(py, PROJECTION_MODULE, "current_audit_result_projection")?.call0()?; + if projection.is_none() { + return Ok(payload); + } + let json: Bound<'py, PyAny> = py.import("json")?.into_any(); + let document: Bound<'py, PyAny> = json.call_method1("loads", (payload,))?; + let counts: Bound<'py, PyDict> = PyDict::new(py); + for field in ["attempted_count", "written_count", "failed_count"] { + counts.set_item(field, projection.getattr(field)?)?; + } + document.set_item("audit_result_projection", &counts)?; + if projection.getattr("degraded")?.is_truthy()? { + document.set_item("projection_degraded", true)?; + let reason: Bound<'py, PyDict> = PyDict::new(py); + reason.set_item("reason", PROJECTION_FAILURE)?; + for field in ["attempted_count", "written_count", "failed_count"] { + reason.set_item(field, projection.getattr(field)?)?; + } + let reasons: Bound<'py, PyList> = PyList::empty(py); + reasons.append(reason)?; + document.set_item("projection_degradation_reasons", reasons)?; + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("indent", 2)?; + let dumped: String = json + .call_method("dumps", (document,), Some(&options))? + .extract()?; + Ok(PyString::new(py, &format!("{dumped}\n")).into_any()) +} + +/// Write the summary footer and JSON output; returns the exit code. +fn complete( + request: &Bound<'_, PyAny>, + invocation: &Bound<'_, PyAny>, + results: &[Bound<'_, PyAny>], + run: &AuditRun<'_>, +) -> PyResult { + let py: Python<'_> = request.py(); + let mut counts: AuditCounts = AuditCounts { + total: results.len(), + ..AuditCounts::default() + }; + for result in results { + match result + .getattr("outcome")? + .getattr("value")? + .extract::()? + .as_str() + { + PASS_OUTCOME => counts.passed += 1, + WARN_OUTCOME => counts.warned += 1, + ERROR_OUTCOME => counts.failed += 1, + INSUFFICIENT_OUTCOME => counts.insufficient += 1, + _ => {} + } + } + let stream: Bound<'_, PyAny> = invocation.getattr("progress_stream")?; + let style: CliStyle = CliStyle { + use_color: invocation.getattr("use_color")?.is_truthy()?, + }; + stream.call_method1("write", (render_audit_summary(counts, style),))?; + stream.call_method0("flush")?; + let payload: Bound<'_, PyAny> = audit_payload( + &PyTuple::new(py, results)?.into_any(), + (run.concurrency, run.worker_count), + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(audit_exit_code(counts.failed)) +} + +/// Run one `sqb audit` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_audit_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let invocation: Bound<'_, PyAny> = call_with( + &imported(py, INVOCATION_MODULE, "resolve_reported_command_invocation")?, + &[ + ("request", request.clone()), + ( + "invocation_type", + imported(py, COMMAND_MODELS, "AuditInvocation")?, + ), + ], + )?; + let pipeline: Bound<'_, PyAny> = compile_audits(request, &invocation)?; + let run: AuditRun<'_> = prepare_run(request, &invocation, &pipeline)?; + write_headers(&invocation, &pipeline, run.concurrency)?; + let results: Vec> = execute(&invocation, &pipeline, &run)?; + complete(request, &invocation, &results, &run) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_audit_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/mod.rs new file mode 100644 index 0000000000..2ef3aae6d5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod command; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/registration.rs new file mode 100644 index 0000000000..f11af80d5f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb audit` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::audit_command::command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs new file mode 100644 index 0000000000..593374b7c0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/command.rs @@ -0,0 +1,121 @@ +//! `sqb build`: resolve, compile, gate, execute and report one direct build. + +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::build_command::completion::{ + complete_direct_build, finalize_exceptional_cost, finalize_no_work_with_timings, + record_pending_cost, write_partial_timings, +}; +use crate::bindings::_helpers::build_command::execution::execute_direct_build; +use crate::bindings::_helpers::build_command::invocation::{BuildContext, resolve_build_context}; +use crate::bindings::_helpers::build_command::planning::{ + compile_build_plan, enforce_build_plan_policies, run_defer_clone_prephase, + write_compile_target, write_plan_text, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const TIMING_TRACKER_MODULE: &str = "sqlbuild.diagnostics.classes.build_phase_timing_tracker"; +const RESOURCE_REPORTING_MODULE: &str = "sqlbuild.diagnostics.main.process_resource_reporting"; +const DEADLINE_MODULE: &str = "sqlbuild.runtime.execution_limits.main.execution_deadline_scope"; +const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; + +fn run_with_providers(context: &BuildContext<'_>) -> PyResult { + let py: Python<'_> = context.py(); + let session: Bound<'_, PyAny> = call_with( + &imported(py, PROVIDER_SESSION_MODULE, "build_provider_session")?, + &[( + "discovered_providers", + context.get("discovered_inputs")?.getattr("providers")?, + )], + )?; + let built: PyResult = run_planned_build(context, &session.getattr("providers")?); + let closed: PyResult> = session.call_method0("close"); + let exit_code: i32 = built?; + closed?; + Ok(exit_code) +} + +fn run_planned_build(context: &BuildContext<'_>, providers: &Bound<'_, PyAny>) -> PyResult { + run_defer_clone_prephase(context)?; + let pipeline: Bound<'_, PyAny> = compile_build_plan(context)?; + write_plan_text(context, &pipeline)?; + enforce_build_plan_policies(context, &pipeline)?; + write_compile_target(context, &pipeline)?; + if finalize_no_work_with_timings(context, &pipeline)? { + return Ok(0); + } + let started_at: Bound<'_, PyAny> = record_pending_cost(context, &pipeline)?; + match execute_direct_build(context, &pipeline, providers) { + Ok((preparation, outcome, checks)) => complete_direct_build( + context, + &pipeline, + (&preparation, &outcome, &checks), + &started_at, + ), + Err(error) => { + finalize_exceptional_cost(context, &pipeline, (&started_at, &error)); + Err(error) + } + } +} + +fn run_within_limits( + request: &Bound<'_, PyAny>, + tracker: &Bound<'_, PyAny>, + started: Instant, +) -> PyResult { + let py: Python<'_> = request.py(); + let context: BuildContext<'_> = resolve_build_context(request, tracker, started)?; + let limits: Bound<'_, PyAny> = context.get("execution_limits")?; + let deadline: Bound<'_, PyAny> = call_with( + &imported(py, DEADLINE_MODULE, "execution_deadline_scope")?, + &[ + ( + "max_duration_seconds", + limits.getattr("max_duration_seconds")?, + ), + ("max_duration", limits.getattr("max_duration")?), + ("target_name", context.get("effective_target_name")?), + ("remediation", limits.getattr("remediation")?), + ], + )?; + within(&deadline, |_| run_with_providers(&context)) +} + +/// Run one `sqb build` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_build_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let tracker: Bound<'_, PyAny> = + imported(py, TIMING_TRACKER_MODULE, "BuildPhaseTimingTracker")?.call0()?; + let reporting: Bound<'_, PyAny> = call_with( + &imported(py, RESOURCE_REPORTING_MODULE, "process_resource_reporting")?, + &[("enabled", request.getattr("debug")?)], + )?; + within(&reporting, |_| { + within(&tracker.call_method0("scope")?, |_| { + let started: Instant = Instant::now(); + run_within_limits(request, &tracker, started).inspect_err(|_| { + let verbose: bool = request + .getattr("verbose") + .and_then(|value| value.is_truthy()) + .unwrap_or(false) + || request + .getattr("debug") + .and_then(|value| value.is_truthy()) + .unwrap_or(false); + if verbose { + let _ = write_partial_timings(py, &tracker); + } + }) + }) + }) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_build_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs new file mode 100644 index 0000000000..a4ef5802f0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -0,0 +1,412 @@ +//! Cost finalization, runtime targets, the footer, execution JSON and phase timings. + +use std::time::Instant; + +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_core::build_command::main::build_exit_code::build_exit_code; +use sqlbuild_core::build_command::main::python_node_failed::python_node_failed; +use sqlbuild_core::build_command::main::render_phase_timings::render_phase_timings; +use sqlbuild_core::build_command::models::PhaseTimings; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::build_command::execution::{ + Outcome, Preparation, write_run_context, +}; +use crate::bindings::_helpers::build_command::invocation::BuildContext; +use crate::bindings::_helpers::build_command::support::{ + boolean, emit, joined, none, text, utc_now, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const COST_COLLECTION_MODULE: &str = "sqlbuild.cli.commands._helpers.cost.collection"; +const COST_TYPES: &str = "sqlbuild.cost.types"; +const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; +const BUILD_TYPES: &str = "sqlbuild.executor.build.types"; +const CALLBACKS_MODULE: &str = "sqlbuild.cli.commands.classes.build_progress_callbacks"; +const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_runtime_target"; +const CHECK_TARGET_MODULE: &str = + "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; +const SUCCESS_STATUS: &str = "success"; +const PENDING_MESSAGE: &str = "Build cost collection has not completed."; +const INTERRUPTED_MESSAGE: &str = "Build was interrupted before cost collection completed."; + +/// How one cost finalization differs from a completed collection. +pub(crate) struct CostRequest<'py> { + pub(crate) status: &'static str, + pub(crate) started_at: Bound<'py, PyAny>, + pub(crate) completed_at: Bound<'py, PyAny>, + pub(crate) overrides: Vec<(&'static str, Bound<'py, PyAny>)>, +} + +/// `finalize_build_cost(BuildCostFinalization(...))`. +pub(crate) fn finalize_build_cost<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + cost: CostRequest<'py>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let mut fields: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ("project_dir", context.get("effective_project_dir")?), + ("adapter_name", context.get("adapter_name")?), + ("adapter", context.get("adapter")?), + ("connection_config", context.get("connection_config")?), + ("target_name", project.getattr("effective_target_name")?), + ( + "target_database", + project.getattr("effective_target_database")?, + ), + ("run_id", project.getattr("run_id")?), + ("build_status", text(py, cost.status)), + ("started_at", cost.started_at), + ("completed_at", cost.completed_at), + ( + "config", + context + .get("discovered_inputs")? + .getattr("project_config")? + .getattr("cost")?, + ), + ("output_stream", context.stream.clone()), + ("use_color", boolean(py, context.use_color)), + ]; + fields.extend(cost.overrides); + let finalization: Bound<'py, PyAny> = call_with( + &imported(py, COMMAND_MODELS, "BuildCostFinalization")?, + &fields, + )?; + imported(py, COST_COLLECTION_MODULE, "finalize_build_cost")?.call1((finalization,)) +} + +fn cost_status<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, COST_TYPES, "CostStatus")?.getattr(name) +} + +/// Record a pending cost state before execution starts. +pub(crate) fn record_pending_cost<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let started_at: Bound<'py, PyAny> = utc_now(py)?; + finalize_build_cost( + context, + pipeline, + CostRequest { + status: "running", + started_at: started_at.clone(), + completed_at: started_at.clone(), + overrides: vec![ + ("collect", boolean(py, false)), + ("render", boolean(py, false)), + ("cost_status", cost_status(py, "PENDING")?), + ("cost_message", text(py, PENDING_MESSAGE)), + ], + }, + )?; + Ok(started_at) +} + +/// Retain cost collection elapsed time on the active timing tracker. +fn finish_cost_timing(context: &BuildContext<'_>, started: Instant) -> f64 { + let elapsed: f64 = started.elapsed().as_secs_f64(); + let _ = context.tracker.setattr("cost_collection_seconds", elapsed); + elapsed +} + +fn style(context: &BuildContext<'_>) -> CliStyle { + CliStyle { + use_color: context.use_color, + } +} + +fn write_timings(context: &BuildContext<'_>, timings: &PhaseTimings) -> PyResult<()> { + emit( + &context.stream, + &render_phase_timings(timings, style(context)), + ) +} + +/// Write the phases the tracker measured before an exceptional exit, to stderr without colour. +pub(crate) fn write_partial_timings(py: Python<'_>, tracker: &Bound<'_, PyAny>) -> PyResult<()> { + let snapshot: Bound<'_, PyAny> = tracker.call_method0("snapshot")?; + let seconds = |name: &str| -> PyResult> { snapshot.getattr(name)?.extract() }; + let timings: PhaseTimings = PhaseTimings { + compile_seconds: seconds("compile_seconds")?, + planning_seconds: seconds("planning_seconds")?, + connection_preparation_seconds: seconds("connection_preparation_seconds")?, + schema_preparation_seconds: seconds("schema_preparation_seconds")?, + execution_seconds: seconds("execution_seconds")?, + cost_collection_seconds: seconds("cost_collection_seconds")?, + total_seconds: seconds("total_seconds")?, + }; + emit( + &py.import("sys")?.getattr("stderr")?, + &render_phase_timings(&timings, CliStyle { use_color: false }), + ) +} + +fn cost_payload<'py>(py: Python<'py>, record: &Bound<'py, PyAny>) -> PyResult> { + if record.is_none() { + return Ok(none(py)); + } + call_with( + &imported(py, NATIVE_MODULE, "cost_output_payload")?, + &[("record", record.clone())], + ) +} + +fn run_id_for<'py>( + pipeline: &Bound<'py, PyAny>, + record: &Bound<'py, PyAny>, +) -> PyResult> { + if record.is_none() { + return Ok(none(pipeline.py())); + } + pipeline.getattr("project")?.getattr("run_id") +} + +fn write_json(context: &BuildContext<'_>, payload: Bound<'_, PyAny>) -> PyResult<()> { + call_with( + &imported(context.py(), NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", context.request.getattr("json_output")?), + ( + "json_output_path", + context.request.getattr("json_output_path")?, + ), + ], + )?; + Ok(()) +} + +/// Persist and emit a completed zero-cost record when the plan has no work. +fn finalize_no_work(context: &BuildContext<'_>, pipeline: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = context.py(); + let plan: Bound<'_, PyAny> = pipeline.getattr("plan_output")?; + if call_with( + &imported(py, NATIVE_MODULE, "plan_has_executable_work")?, + &[ + ("plan", plan.clone()), + ( + "python_plan_entries", + pipeline.getattr("python_plan_entries")?, + ), + ], + )? + .is_truthy()? + { + return Ok(false); + } + if context.verbose { + write_run_context(context, pipeline)?; + } + let completed_at: Bound<'_, PyAny> = utc_now(py)?; + let record: Bound<'_, PyAny> = finalize_build_cost( + context, + pipeline, + CostRequest { + status: SUCCESS_STATUS, + started_at: completed_at.clone(), + completed_at, + overrides: vec![ + ("collect", boolean(py, false)), + ("had_executable_work", boolean(py, false)), + ], + }, + )?; + let result: Bound<'_, PyAny> = call_with( + &imported(py, BUILD_MODELS, "BuildExecutionResult")?, + &[( + "status", + imported(py, BUILD_TYPES, "BuildStatus")?.getattr("SUCCESS")?, + )], + )?; + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_build_execution_json")?, + &[ + ("result", result), + ("plan", plan), + ("run_id", run_id_for(pipeline, &record)?), + ("cost", cost_payload(py, &record)?), + ], + )?; + write_json(context, payload)?; + Ok(true) +} + +/// Finalize a build without executable work, with verbose phase timings. +pub(crate) fn finalize_no_work_with_timings( + context: &BuildContext<'_>, + pipeline: &Bound<'_, PyAny>, +) -> PyResult { + let cost_started: Instant = Instant::now(); + let finalized: PyResult = finalize_no_work(context, pipeline); + let cost_seconds: f64 = finish_cost_timing(context, cost_started); + let finalized: bool = finalized?; + if finalized && context.verbose { + write_timings( + context, + &PhaseTimings { + compile_seconds: pipeline.getattr("compile_seconds")?.extract()?, + planning_seconds: pipeline.getattr("planning_seconds")?.extract()?, + cost_collection_seconds: Some(cost_seconds), + total_seconds: Some(context.started.elapsed().as_secs_f64()), + ..PhaseTimings::default() + }, + )?; + } + Ok(finalized) +} + +/// Persist failed or interrupted cost state without masking the active error. +pub(crate) fn finalize_exceptional_cost( + context: &BuildContext<'_>, + pipeline: &Bound<'_, PyAny>, + (started_at, error): (&Bound<'_, PyAny>, &PyErr), +) { + let py: Python<'_> = context.py(); + let cost_started: Instant = Instant::now(); + let interrupted: bool = error.is_instance_of::(py); + let _ = (|| -> PyResult> { + finalize_build_cost( + context, + pipeline, + CostRequest { + status: if interrupted { "interrupted" } else { "failed" }, + started_at: started_at.clone(), + completed_at: utc_now(py)?, + overrides: vec![ + ("collect", boolean(py, !interrupted)), + ("render", boolean(py, false)), + ("cost_status", cost_status(py, "PARTIAL")?), + ("cost_message", text(py, INTERRUPTED_MESSAGE)), + ], + }, + ) + })(); + let _ = finish_cost_timing(context, cost_started); +} + +fn any_failed( + results: &Bound<'_, PyAny>, + failed: impl Fn(&Bound<'_, PyAny>) -> PyResult, +) -> PyResult { + for result in results.try_iter()? { + if failed(&result?)? { + return Ok(true); + } + } + Ok(false) +} + +/// Persist and render a completed direct build and return its exit code. +pub(crate) fn complete_direct_build<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + (preparation, outcome, check_results): (&Preparation<'py>, &Outcome<'py>, &Bound<'py, PyAny>), + started_at: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = context.py(); + let completed_at: Bound<'py, PyAny> = utc_now(py)?; + let result: &Bound<'py, PyAny> = &outcome.result; + let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let succeeded: bool = result.getattr("status")?.str()?.extract::()? == SUCCESS_STATUS; + let python_failed: bool = any_failed(&outcome.python_results, |result| { + Ok(python_node_failed( + &result.getattr("status")?.str()?.extract::()?, + )) + })?; + let checks_failed: bool = any_failed(check_results, |result| { + result.getattr("failed")?.is_truthy() + })?; + let exit_code: i32 = build_exit_code(succeeded, (python_failed, checks_failed)); + let target_dir: Bound<'py, PyAny> = + joined(&context.get("effective_project_dir")?, &["target"])?; + call_with( + &imported(py, RUNTIME_TARGET_MODULE, "write_runtime_target")?, + &[ + ("target_dir", target_dir.clone()), + ("plan_output", plan.clone()), + ("result", result.clone()), + ], + )?; + call_with( + &imported(py, CHECK_TARGET_MODULE, "write_python_check_runtime_target")?, + &[ + ("target_dir", target_dir), + ("results", check_results.clone()), + ], + )?; + let cost_started: Instant = Instant::now(); + let record: PyResult> = finalize_build_cost( + context, + pipeline, + CostRequest { + status: if exit_code == 0 { + SUCCESS_STATUS + } else { + "failed" + }, + started_at: started_at.clone(), + completed_at, + overrides: vec![("render", boolean(py, false))], + }, + ); + let cost_seconds: f64 = finish_cost_timing(context, cost_started); + let record: Bound<'py, PyAny> = record?; + let footer: String = call_with( + &imported(py, CALLBACKS_MODULE, "format_build_footer")?, + &[ + ("result", result.clone()), + ("elapsed", preparation.callbacks.getattr("elapsed")?), + ("use_color", boolean(py, context.use_color)), + ("python_node_results", outcome.python_results.clone()), + ], + )? + .extract()?; + emit(&context.stream, &format!("\n{footer}\n"))?; + let payload: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_degradable_build_execution_json")?, + &[ + ("result", result.clone()), + ("plan", plan), + ("python_node_results", outcome.python_results.clone()), + ("python_check_results", check_results.clone()), + ("command", text(py, "build")), + ("run_id", run_id_for(pipeline, &record)?), + ("cost", cost_payload(py, &record)?), + ("execution_succeeded", boolean(py, exit_code == 0)), + ], + )?; + write_json(context, payload)?; + call_with( + &imported(py, COST_COLLECTION_MODULE, "render_build_cost")?, + &[ + ("record", record), + ("output_stream", context.stream.clone()), + ("use_color", boolean(py, context.use_color)), + ], + )?; + if context.verbose { + let timings: Bound<'py, PyAny> = result.getattr("timings")?; + let seconds = |name: &str| -> PyResult> { timings.getattr(name)?.extract() }; + write_timings( + context, + &PhaseTimings { + compile_seconds: pipeline.getattr("compile_seconds")?.extract()?, + planning_seconds: pipeline.getattr("planning_seconds")?.extract()?, + connection_preparation_seconds: seconds("connection_preparation_seconds")?, + schema_preparation_seconds: seconds("schema_preparation_seconds")?, + execution_seconds: seconds("execution_seconds")?, + cost_collection_seconds: Some(cost_seconds), + total_seconds: Some(context.started.elapsed().as_secs_f64()), + }, + )?; + } + Ok(exit_code) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs new file mode 100644 index 0000000000..b31ca9c1cf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -0,0 +1,654 @@ +//! Prepare build execution, run the build pipeline and the post-build Python checks. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_core::build_command::main::render_run_context::render_run_context; +use sqlbuild_core::build_command::models::{RunContextFacts, SelectorFileFact}; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::build_command::invocation::BuildContext; +use crate::bindings::_helpers::build_command::lifecycle::{ + DirectPythonLifecycle, LifecycleInputs, prepare_direct_python_lifecycle, +}; +use crate::bindings::_helpers::build_command::support::{ + boolean, display_text, emit, frozenset, joined, keyword_relay, line_writer, none, text, tuple, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const CALLBACKS_MODULE: &str = "sqlbuild.cli.commands.classes.build_progress_callbacks"; +const PARSING_MODULE: &str = "sqlbuild.cli.commands._helpers.input.parsing"; +const CURSOR_OVERRIDES_MODULE: &str = + "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; +const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; +const BUILD_MODELS: &str = "sqlbuild.executor.build.models"; +const PIPELINE_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; +const FRESHNESS_MODULE: &str = "sqlbuild.cli.commands._helpers.freshness.source_freshness"; +const COST_CONTEXT_MODULE: &str = "sqlbuild.cost.classes.cost_context"; +const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; +const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; +const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; +const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; +const EXECUTION_COMMAND: &str = "sqb build"; +const SUCCESS_STATUS: &str = "success"; +const NODE_START: [&str; 2] = ["name", "resource_kind"]; +const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; + +/// Prepared callbacks, concurrency, cursors and Python lifecycle. +pub(crate) struct Preparation<'py> { + pub(crate) callbacks: Bound<'py, PyAny>, + pub(crate) concurrency: Bound<'py, PyAny>, + connection_progress: Bound<'py, PyAny>, + lifecycle: Bound<'py, DirectPythonLifecycle>, + cursors: [Bound<'py, PyAny>; 4], +} + +/// The build result and its finalized Python node results. +pub(crate) struct Outcome<'py> { + pub(crate) result: Bound<'py, PyAny>, + pub(crate) python_results: Bound<'py, PyAny>, +} + +fn count(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + value.getattr(name)?.len() +} + +fn validated_date(py: Python<'_>, value: Option>) -> PyResult> { + let Some(value) = value.filter(|value| value.is_instance_of::()) else { + return Ok(None); + }; + let datetime: Bound<'_, PyAny> = py.import("datetime")?.into_any(); + for kind in ["date", "datetime"] { + match datetime + .getattr(kind)? + .call_method1("fromisoformat", (&value,)) + { + Ok(_) => return Ok(Some(value.extract()?)), + Err(error) if error.is_instance_of::(py) => {} + Err(error) => return Err(error), + } + } + Ok(None) +} + +fn run_context_facts( + context: &BuildContext<'_>, + pipeline: &Bound<'_, PyAny>, + concurrency: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = context.py(); + let project: Bound<'_, PyAny> = pipeline.getattr("project")?; + let plan: Bound<'_, PyAny> = pipeline.getattr("plan_output")?; + let discovered: Bound<'_, PyAny> = context.get("discovered_inputs")?; + let selected_count: usize = count(&plan, "model_entries")? + + count(&plan, "seed_entries")? + + count(&plan, "function_entries")? + + count(&plan, "source_load_entries")? + + count(pipeline, "python_plan_entries")?; + let mut loaded_sources: usize = 0; + for source in project.getattr("sources")?.try_iter()? { + if !source? + .getattr("source_entry")? + .getattr("loader")? + .is_none() + { + loaded_sources += 1; + } + } + let total_count: usize = count(&project, "models")? + + count(&project, "seeds")? + + count(&project, "functions")? + + loaded_sources + + count(&discovered, "task_functions")? + + count(&discovered, "asset_functions")?; + let resolved: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "describe_connection_warehouse")?, + &[ + ("discovered_inputs", discovered), + ("selected_target", project.getattr("effective_target_name")?), + ("connection_config", context.get("connection_config")?), + ], + )?; + let warehouse: Bound<'_, PyAny> = resolved.getattr("warehouse")?; + let warehouse: Option<(String, String)> = if warehouse.is_none() { + None + } else { + Some(( + display_text(&warehouse)?.unwrap_or_else(|| "not set".to_owned()), + resolved.getattr("source")?.str()?.extract()?, + )) + }; + let variables: Bound<'_, PyAny> = project.getattr("effective_vars")?; + let variable = |name: &str| -> PyResult>> { + let value: Bound<'_, PyAny> = variables.call_method1("get", (name,))?; + Ok((!value.is_none()).then_some(value)) + }; + let mut selector_files: Vec = Vec::new(); + for file in context.request.getattr("selector_files")?.try_iter()? { + let file: Bound<'_, PyAny> = file?; + selector_files.push(SelectorFileFact { + path: file.getattr("path")?.str()?.extract()?, + selector_count: file.getattr("selector_count")?.extract()?, + }); + } + Ok(RunContextFacts { + command: EXECUTION_COMMAND.to_owned(), + run_id: display_text(&project.getattr("run_id")?)?, + target: display_text(&project.getattr("effective_target_name")?)?, + database: display_text(&project.getattr("effective_target_database")?)?, + schema: display_text(&project.getattr("effective_target_schema")?)?, + warehouse, + concurrency: concurrency.extract()?, + full_refresh: context.request_flag("full_refresh")?, + selected_count, + total_count, + date_vars: ( + validated_date(py, variable("start_date")?)?, + validated_date(py, variable("end_date")?)?, + ), + selector_files, + }) +} + +/// Write the resolved non-secret placement, mode and selection context. +pub(crate) fn write_run_context( + context: &BuildContext<'_>, + pipeline: &Bound<'_, PyAny>, +) -> PyResult<()> { + let concurrency: Bound<'_, PyAny> = context.concurrency(&pipeline.getattr("project")?)?; + let facts: RunContextFacts = run_context_facts(context, pipeline, &concurrency)?; + emit( + &context.stream, + &render_run_context( + &facts, + CliStyle { + use_color: context.use_color, + }, + ), + ) +} + +fn cursors<'py>( + context: &BuildContext<'py>, + plan: &Bound<'py, PyAny>, +) -> PyResult<[Bound<'py, PyAny>; 4]> { + let py: Python<'py> = context.py(); + let mut overrides: Bound<'py, PyAny> = plan.getattr("cursor_overrides")?; + if !overrides.is_truthy()? { + overrides = call_with( + &imported(py, CURSOR_OVERRIDES_MODULE, "typed_cursor_overrides")?, + &[( + "cursor_overrides", + context.request.getattr("cursor_overrides")?, + )], + )?; + } + if !overrides.is_truthy()? { + overrides = imported(py, PLANNER_MODELS, "CursorOverrides")?.call0()?; + } + let timestamp: Bound<'py, PyAny> = imported(py, PARSING_MODULE, "parse_cursor_timestamp")?; + let integer: Bound<'py, PyAny> = imported(py, PARSING_MODULE, "parse_cursor_integer")?; + Ok([ + timestamp.call1((overrides.getattr("start_ts")?,))?, + timestamp.call1((overrides.getattr("end_ts")?,))?, + integer.call1((overrides.getattr("start_int")?,))?, + integer.call1((overrides.getattr("end_int")?,))?, + ]) +} + +/// Build callbacks, the execution header, the execution connection reporter and the Python lifecycle. +pub(crate) fn prepare_build_execution<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + providers: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let debug: bool = context.request_flag("debug")? || context.request_flag("json_output")?; + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, CALLBACKS_MODULE, "BuildProgressCallbacks")?, + &[ + ("plan", plan.clone()), + ("use_color", boolean(py, context.use_color)), + ("verbose", context.request.getattr("verbose")?), + ("debug", boolean(py, debug)), + ( + "event_output_path", + context.request.getattr("event_output_path")?, + ), + ], + )?; + let concurrency: Bound<'py, PyAny> = context.concurrency(&pipeline.getattr("project")?)?; + if context.verbose { + write_run_context(context, pipeline)?; + } else { + call_with( + &imported(py, NATIVE_MODULE, "write_execution_header")?, + &[ + ("stream", context.stream.clone()), + ("command", text(py, EXECUTION_COMMAND)), + ("target", none(py)), + ("concurrency", concurrency.clone()), + ("use_color", boolean(py, context.use_color)), + ], + )?; + } + let connection_progress: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", context.get("adapter_name")?), + ("stream", context.stream.clone()), + ( + "blank_line_before_start", + boolean(py, plan.getattr("source_load_entries")?.is_truthy()?), + ), + ("blank_line_after_complete", boolean(py, true)), + ("use_color", boolean(py, context.use_color)), + ], + )?; + let cursors: [Bound<'py, PyAny>; 4] = cursors(context, &plan)?; + let inputs: LifecycleInputs<'py> = LifecycleInputs { + cursors: cursors.clone(), + on_node_start: keyword_relay(callbacks.getattr("on_node_start")?, &NODE_START)?, + on_node_complete: callbacks.getattr("on_node_complete")?, + providers: providers.clone(), + }; + let lifecycle: Bound<'py, DirectPythonLifecycle> = + prepare_direct_python_lifecycle(context, pipeline, &inputs)?; + Ok(Preparation { + callbacks, + concurrency, + connection_progress, + lifecycle, + cursors, + }) +} + +fn verbose_callback<'py>( + context: &BuildContext<'py>, + callbacks: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + if context.verbose { + return callbacks.getattr(name); + } + Ok(none(context.py())) +} + +/// Run the build pipeline, record eligible source freshness and finalize the Python lifecycle. +pub(crate) fn execute_build_plan<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + preparation: &Preparation<'py>, + providers: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let adapter: Bound<'py, PyAny> = context.get("adapter")?; + let connection_config: Bound<'py, PyAny> = context.get("connection_config")?; + let callbacks: &Bound<'py, PyAny> = &preparation.callbacks; + let lifecycle: &DirectPythonLifecycle = preparation.lifecycle.get(); + let [start_ts, end_ts, start_int, end_int] = &preparation.cursors; + let request = |name: &str| context.request.getattr(name); + let runtime: Bound<'py, PyAny> = call_with( + &imported(py, BUILD_MODELS, "BuildRuntimeParams")?, + &[ + ("run_id", project.getattr("run_id")?), + ( + "runtime_dir", + joined(&context.get("effective_project_dir")?, &["target"])?, + ), + ( + "snapshots", + context + .get("discovered_inputs")? + .getattr("project_config")? + .getattr("snapshots")?, + ), + ( + "allow_snapshot_schema_change", + request("allow_snapshot_schema_change")?, + ), + ("run_tests", request("run_tests")?), + ("run_audits", request("run_audits")?), + ("fail_fast", request("fail_fast")?), + ("max_concurrency", preparation.concurrency.clone()), + ("loader_is_reload", request("reload_sources")?), + ("start_cursor_ts", start_ts.clone()), + ("end_cursor_ts", end_ts.clone()), + ("start_cursor_int", start_int.clone()), + ("end_cursor_int", end_int.clone()), + ("use_color", boolean(py, context.use_color)), + ("providers", providers.clone()), + ( + "project_relations", + call_with( + &imported(py, NATIVE_MODULE, "build_project_relation_targets")?, + &[("adapter", adapter.clone()), ("plan_output", plan.clone())], + )?, + ), + ], + )?; + let connection: &Bound<'py, PyAny> = &preparation.connection_progress; + let build_callbacks: Bound<'py, PyAny> = call_with( + &imported(py, BUILD_MODELS, "BuildCallbacks")?, + &[ + ("on_migration_progress", line_writer(&context.stream)?), + ( + "on_node_start", + keyword_relay(callbacks.getattr("on_node_start")?, &NODE_START)?, + ), + ( + "on_node_complete", + preparation.lifecycle.getattr("on_node_complete")?, + ), + ("on_sub_progress", callbacks.getattr("on_sub_progress")?), + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + keyword_relay( + connection.getattr("on_connection_complete")?, + &CONNECTION_TIMING, + )?, + ), + ( + "on_connection_error", + keyword_relay( + connection.getattr("on_connection_error")?, + &CONNECTION_TIMING, + )?, + ), + ( + "on_scheduler_state", + verbose_callback(context, callbacks, "on_scheduler_state")?, + ), + ( + "on_statement_complete", + verbose_callback(context, callbacks, "on_statement_complete")?, + ), + ], + )?; + let customizations: Bound<'py, PyAny> = call_with( + &imported(py, BUILD_MODELS, "BuildCustomizations")?, + &[ + ( + "custom_materializations", + pipeline.getattr("custom_materializations")?, + ), + ("loader_functions", lifecycle.loader_functions(py)?), + ], + )?; + let initial_state: Bound<'py, PyAny> = call_with( + &imported(py, BUILD_MODELS, "BuildInitialState")?, + &[ + ("precompleted_keys", lifecycle.precompleted_keys(py)?), + ("initial_load_results", lifecycle.ingress_load_results(py)), + ("initial_failed_keys", lifecycle.blocked_keys(py)?), + ], + )?; + let result: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_RUN_MODULE, "run_build_pipeline")?, + &[ + ("plan", plan.clone()), + ("connection_config", connection_config.clone()), + ("adapter", adapter.clone()), + ("settings", project.getattr("settings")?), + ("runtime", runtime), + ("callbacks", build_callbacks), + ("customizations", customizations), + ("initial_state", initial_state), + ], + )?; + call_with( + &imported( + py, + FRESHNESS_MODULE, + "append_eligible_direct_source_freshness_records", + )?, + &[ + ("plan", plan), + ("result", result.clone()), + ("adapter", adapter), + ("connection_config", connection_config), + ("run_id", project.getattr("run_id")?), + ], + )?; + lifecycle.finalize(py)?; + Ok(Outcome { + result, + python_results: lifecycle.python_results(py)?, + }) +} + +fn check_runtime<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + (connection, graph, checks): (&Bound<'py, PyAny>, &Bound<'py, PyAny>, &[Bound<'py, PyAny>]), + providers: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let adapter: Bound<'py, PyAny> = context.get("adapter")?; + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let names: Vec> = checks + .iter() + .map(|check| check.getattr("name")) + .collect::>()?; + call_with( + &imported(py, EXECUTOR_PYTHON_MODELS, "PythonNodeRuntime")?, + &[ + ("adapter", adapter.clone()), + ("connection_config", context.get("connection_config")?), + ("connection", connection.clone()), + ("run_id", project.getattr("run_id")?), + ("target", project.getattr("effective_target_name")?), + ("vars", project.getattr("effective_vars")?), + ("is_reload", context.request.getattr("reload_sources")?), + ( + "default_database", + adapter.call_method0("default_database")?, + ), + ("default_schema", adapter.call_method0("default_schema")?), + ( + "relation_targets", + call_with( + &imported(py, CHECK_CORE_MODULE, "build_check_relation_targets")?, + &[ + ("adapter", adapter.clone()), + ("pipeline_result", pipeline.clone()), + ("python_graph", graph.clone()), + ("selected_python_names", frozenset(py, &names)?), + ], + )?, + ), + ( + "project_relations", + call_with( + &imported(py, NATIVE_MODULE, "build_project_relation_targets")?, + &[ + ("adapter", adapter), + ("plan_output", pipeline.getattr("plan_output")?), + ], + )?, + ), + ("providers", providers.clone()), + ], + ) +} + +fn names_of<'py>(values: &Bound<'py, PyAny>, attribute: &str) -> PyResult>> { + values + .try_iter()? + .map(|value| value?.getattr(attribute)) + .collect() +} + +/// Execute relevant Python check functions after a successful build. +pub(crate) fn run_post_build_python_checks<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + (outcome, preparation): (&Outcome<'py>, &Preparation<'py>), + providers: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let result: &Bound<'py, PyAny> = &outcome.result; + if result.getattr("status")?.str()?.extract::()? != SUCCESS_STATUS { + return tuple(py, &[]); + } + let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; + let plan: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let load_results: Bound<'py, PyAny> = result.getattr("load_results")?; + let graph: Bound<'py, PyAny> = call_with( + &imported(py, GRAPH_MODULE, "build_discovered_python_node_graph")?, + &[("discovered_inputs", discovered.clone())], + )?; + let by_loader = || { + call_with( + &imported(py, CHECK_CORE_MODULE, "load_results_by_loader_name")?, + &[ + ("source_map", plan.getattr("source_map")?), + ("load_results", load_results.clone()), + ], + ) + }; + let mut dependencies: Vec> = names_of(&outcome.python_results, "node_name")?; + for name in by_loader()?.try_iter()? { + dependencies.push(name?); + } + let mut completed: Vec> = + names_of(&result.getattr("model_results")?, "model_name")?; + completed.extend(names_of(&result.getattr("seed_results")?, "seed_name")?); + completed.extend(names_of(&load_results, "source_name")?); + let checks: Vec> = call_with( + &imported(py, CHECK_CORE_MODULE, "relevant_check_functions")?, + &[ + ("discovered_inputs", discovered.clone()), + ("python_graph", graph.clone()), + ("exclude", context.request.getattr("exclude")?), + ("selected_dependency_names", frozenset(py, &dependencies)?), + ("completed_sql_names", frozenset(py, &completed)?), + ], + )? + .try_iter()? + .collect::>()?; + if checks.is_empty() { + return tuple(py, &[]); + } + let adapter: Bound<'py, PyAny> = context.get("adapter")?; + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (context.get("connection_config")?,))?; + let executed: PyResult> = (|| { + let run_state: Bound<'py, PyAny> = + imported(py, EXECUTOR_PYTHON_MODELS, "PythonNodeRunState")?.call0()?; + call_with( + &imported(py, CHECK_CORE_MODULE, "record_python_run_state_results")?, + &[ + ("discovered_inputs", discovered.clone()), + ("run_state", run_state.clone()), + ("python_results", outcome.python_results.clone()), + ("load_results", load_results.clone()), + ("source_map", plan.getattr("source_map")?), + ], + )?; + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_PYTHON_MODELS, "PythonCheckCallbacks")?, + &[( + "on_check_complete", + preparation.callbacks.getattr("write_execution_event")?, + )], + )?; + call_with( + &imported(py, CHECKS_MODULE, "execute_python_checks")?, + &[ + ("check_functions", tuple(py, &checks)?), + ("python_graph", graph.clone()), + ("upstream_python_results", outcome.python_results.clone()), + ("upstream_load_results", load_results.clone()), + ("upstream_load_results_by_loader_name", by_loader()?), + ( + "runtime", + check_runtime(context, pipeline, (&connection, &graph, &checks), providers)?, + ), + ("run_state", run_state), + ("callbacks", callbacks), + ], + ) + })(); + let closed: PyResult> = adapter.call_method1("close", (connection,)); + let check_results: Bound<'py, PyAny> = executed?; + closed?; + call_with( + &imported(py, NATIVE_MODULE, "write_check_results")?, + &[ + ("stream", context.stream.clone()), + ("results", check_results.clone()), + ("use_color", boolean(py, context.use_color)), + ("check_functions", tuple(py, &checks)?), + ("python_graph", graph), + ], + )?; + Ok(check_results) +} + +/// Run the prepared build and its checks inside the run cost context. +pub(crate) fn execute_direct_build<'py>( + context: &BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + providers: &Bound<'py, PyAny>, +) -> PyResult<(Preparation<'py>, Outcome<'py>, Bound<'py, PyAny>)> { + let py: Python<'py> = context.py(); + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let run_id: Bound<'py, PyAny> = project.getattr("run_id")?; + let target_name: Bound<'py, PyAny> = project.getattr("effective_target_name")?; + let resource_name: Bound<'py, PyAny> = if target_name.is_truthy()? { + target_name + } else { + context.get("adapter_name")? + }; + let cost_context: Bound<'py, PyAny> = imported(py, COST_CONTEXT_MODULE, "CostContext")?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("run_id", &run_id)?; + options.set_item("resource_type", "run")?; + options.set_item("resource_name", &resource_name)?; + options.set_item( + "ledger_path", + joined( + &context.get("effective_project_dir")?, + &[ + "target", + "executions", + &run_id.str()?.extract::()?, + "statements.jsonl", + ], + )?, + )?; + options.set_item("phase", "build")?; + let scope: Bound<'py, PyAny> = cost_context.call_method("scope", (), Some(&options))?; + within(&scope, |_| { + let preparation: Preparation<'py> = prepare_build_execution(context, pipeline, providers)?; + let ran: PyResult<(Outcome<'py>, Bound<'py, PyAny>)> = (|| { + let outcome: Outcome<'py> = + execute_build_plan(context, pipeline, &preparation, providers)?; + let checks_scope: Bound<'py, PyAny> = call_with( + &cost_context.getattr("resource_scope")?, + &[ + ("resource_type", text(py, "run")), + ("resource_name", resource_name.clone()), + ("phase", text(py, "post_build_checks")), + ], + )?; + let checks: Bound<'py, PyAny> = within(&checks_scope, |_| { + run_post_build_python_checks(context, pipeline, (&outcome, &preparation), providers) + })?; + Ok((outcome, checks)) + })(); + let closed: PyResult> = preparation.callbacks.call_method0("close"); + let (outcome, checks) = ran?; + closed?; + Ok((preparation, outcome, checks)) + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs new file mode 100644 index 0000000000..7ae951885e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -0,0 +1,228 @@ +//! Resolve discovery, target, adapter, limits, connection and reporters for one build. + +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_core::build_command::errors::BuildCommandError; +use sqlbuild_core::build_command::main::duration_limit_error::duration_limit_error; +use sqlbuild_core::build_command::models::DurationLimit; + +use crate::bindings::_helpers::build_command::policies::build_command_error; +use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const REPORTERS_MODULE: &str = "sqlbuild.cli.progress.main._build_command_progress_reporters"; +const SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const CONTRACT_MODELS: &str = "sqlbuild.spec.contracts.models"; +const DEFER_CONFLICT_CODE: &str = "C408"; + +/// One build's request, resolved invocation and output settings. +pub(crate) struct BuildContext<'py> { + pub(crate) request: Bound<'py, PyAny>, + pub(crate) invocation: Bound<'py, PyAny>, + pub(crate) stream: Bound<'py, PyAny>, + pub(crate) use_color: bool, + pub(crate) verbose: bool, + pub(crate) started: Instant, + pub(crate) tracker: Bound<'py, PyAny>, +} + +impl<'py> BuildContext<'py> { + pub(crate) fn py(&self) -> Python<'py> { + self.request.py() + } + + pub(crate) fn request_flag(&self, name: &str) -> PyResult { + self.request.getattr(name)?.is_truthy() + } + + pub(crate) fn get(&self, name: &str) -> PyResult> { + self.invocation.getattr(name) + } + + /// The requested concurrency, or the project setting. + pub(crate) fn concurrency(&self, project: &Bound<'py, PyAny>) -> PyResult> { + or_none(self.request.getattr("concurrency")?, || { + project.getattr("settings")?.getattr("concurrency") + }) + } +} + +fn target_config<'py>( + discovered: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("target_name", target_name.clone()), + ], + ) +} + +fn execution_limits<'py>( + discovered: &Bound<'py, PyAny>, + target_name: &Bound<'py, PyAny>, + (adapter_name, adapter): (&str, &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let limits: Bound<'py, PyAny> = if target_name.is_none() { + imported(py, CONTRACT_MODELS, "ExecutionLimitsConfig")?.call0()? + } else { + target_config(discovered, target_name)?.getattr("execution_limits")? + }; + let limit: DurationLimit = DurationLimit { + target_name: target_name.extract()?, + adapter_name: adapter_name.to_owned(), + configures_max_duration: !limits.getattr("max_duration")?.is_none(), + max_duration_seconds: limits.getattr("max_duration_seconds")?.extract()?, + adapter_limit_seconds: adapter + .getattr("execution_duration_limit_seconds")? + .extract()?, + remediation: limits.getattr("remediation")?.extract()?, + }; + if let Some(error) = duration_limit_error(&limit) { + return Err(build_command_error(py, error)); + } + Ok(limits) +} + +/// Discover the project and resolve everything the build reads before compiling. +pub(crate) fn resolve_build_context<'py>( + request: &Bound<'py, PyAny>, + tracker: &Bound<'py, PyAny>, + started: Instant, +) -> PyResult> { + let py: Python<'py> = request.py(); + let project_dir: Bound<'py, PyAny> = or_none(request.getattr("project_dir")?, || { + imported(py, "pathlib", "Path")?.call_method0("cwd") + })?; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", project_dir.clone()), + ("extract_output_column_locations", boolean(py, false)), + ], + )?; + let selected_target: Bound<'py, PyAny> = request.getattr("selected_target")?; + let target_name: Bound<'py, PyAny> = call_with( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ("selected_target", selected_target.clone()), + ], + )?; + let defer_clone_from: Bound<'py, PyAny> = + or_none(request.getattr("defer_clone_from")?, || { + if target_name.is_none() { + return Ok(target_name.clone()); + } + target_config(&discovered, &target_name)?.getattr("defer_clone_from") + })?; + if !request.getattr("defer_to")?.is_none() && !defer_clone_from.is_none() { + return Err(build_command_error( + py, + BuildCommandError::new( + Some(DEFER_CONFLICT_CODE), + "--defer-clone-from cannot be used with --defer-to".to_owned(), + None, + ), + )); + } + let adapter_name: String = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )? + .extract()?; + let adapter: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", text(py, &adapter_name)), + ("project_dir", project_dir.clone()), + ], + )?; + let limits: Bound<'py, PyAny> = + execution_limits(&discovered, &target_name, (&adapter_name, &adapter))?; + let connection_config: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("project_dir", project_dir.clone()), + ("selected_target", selected_target), + ("cli_vars", request.getattr("cli_vars")?), + ], + )?; + let machine_output: bool = request.getattr("json_output")?.is_truthy()?; + let debug: bool = request.getattr("debug")?.is_truthy()?; + let use_color: bool = !request.getattr("no_color")?.is_truthy()? + && !machine_output + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + let stream: Bound<'py, PyAny> = py.import("sys")?.getattr(if debug || machine_output { + "stderr" + } else { + "stdout" + })?; + let reporters: Bound<'py, PyAny> = call_with( + &imported(py, REPORTERS_MODULE, "build_command_progress_reporters")?, + &[ + ("adapter_name", text(py, &adapter_name)), + ("stream", stream.clone()), + ("use_color", boolean(py, use_color)), + ], + )?; + let load_sources: Bound<'py, PyAny> = request.getattr("load_sources")?; + let should_load_sources: bool = request.getattr("reload_sources")?.is_truthy()? + || if load_sources.is_none() { + call_with( + &imported(py, SETTINGS_MODULE, "build_effective_settings_config")?, + &[("discovered_inputs", discovered.clone())], + )? + .getattr("auto_load_sources")? + .is_truthy()? + } else { + load_sources.is_truthy()? + }; + let invocation: Bound<'py, PyAny> = call_with( + &imported(py, COMMAND_MODELS, "BuildInvocation")?, + &[ + ("effective_project_dir", project_dir), + ("discovered_inputs", discovered), + ("effective_defer_clone_from", defer_clone_from), + ("adapter_name", text(py, &adapter_name)), + ("adapter", adapter), + ("connection_config", connection_config), + ("use_color", boolean(py, use_color)), + ("progress_stream", stream.clone()), + ("connection_progress", reporters.getattr("connection")?), + ("planning_progress", reporters.getattr("planning")?), + ("should_load_sources", boolean(py, should_load_sources)), + ("effective_target_name", target_name), + ("execution_limits", limits), + ], + )?; + Ok(BuildContext { + request: request.clone(), + invocation, + stream, + use_color, + verbose: request.getattr("verbose")?.is_truthy()? || debug, + started, + tracker: tracker.clone(), + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs new file mode 100644 index 0000000000..07ddd6113d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -0,0 +1,524 @@ +//! Ingress Python before the build and read-side tasks and assets dispatched as SQL completes. + +use std::collections::HashSet; +use std::sync::Mutex; + +use pyo3::exceptions::PyRuntimeError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::build_command::main::render_python_node_rows::render_python_node_rows; +use sqlbuild_core::build_command::models::PythonNodeRow; +use sqlbuild_core::cli_text::models::CliStyle; + +use crate::bindings::_helpers::build_command::support::{ + boolean, emit, frozenset, or_none, text, tuple, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; +const RUN_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; +const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; +const LOADER_DAG_MODULE: &str = "sqlbuild.compiler.planner.main.selection._loader_dag"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; +const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main._ingress"; +const READ_SIDE_MODULE: &str = "sqlbuild.executor.python_nodes.main.read_side"; +const EXECUTOR_PYTHON_MODELS: &str = "sqlbuild.executor.python_nodes.models"; +const SUCCESS_STATUS: &str = "success"; + +/// The cursor window and output settings the Python lifecycle runs with. +pub(crate) struct LifecycleInputs<'py> { + pub(crate) cursors: [Bound<'py, PyAny>; 4], + pub(crate) on_node_start: Bound<'py, PyAny>, + pub(crate) on_node_complete: Bound<'py, PyAny>, + pub(crate) providers: Bound<'py, PyAny>, +} + +fn node_rows(results: &Bound<'_, PyAny>) -> PyResult> { + let mut rows: Vec = Vec::new(); + for result in results.try_iter()? { + let result: Bound<'_, PyAny> = result?; + rows.push(PythonNodeRow { + kind: result.getattr("kind")?.getattr("value")?.extract()?, + node_name: result.getattr("node_name")?.extract()?, + status: result.getattr("status")?.str()?.extract()?, + error_message: result.getattr("error_message")?.extract()?, + skip_reason: result.getattr("skip_reason")?.extract()?, + }); + } + Ok(rows) +} + +/// Write human-readable task and asset rows. +pub(crate) fn write_python_node_results( + stream: &Bound<'_, PyAny>, + results: &Bound<'_, PyAny>, + use_color: bool, +) -> PyResult<()> { + let rendered: String = render_python_node_rows(&node_rows(results)?, CliStyle { use_color }); + emit(stream, &rendered) +} + +fn result_names(results: &Bound<'_, PyAny>) -> PyResult> { + results + .try_iter()? + .map(|result| result?.getattr("node_name")?.extract()) + .collect() +} + +fn names_set<'py>(py: Python<'py>, names: &[String]) -> PyResult> { + let values: Vec> = names.iter().map(|name| text(py, name)).collect(); + frozenset(py, &values) +} + +/// Mutable state of direct-mode Python lifecycle execution. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct DirectPythonLifecycle { + plan: Py, + discovered: Py, + adapter: Py, + stream: Py, + use_color: bool, + base_on_node_complete: Py, + read_side_connection: Mutex>>, + read_side_tracker: Option>, + ingress_python_results: Py, + ingress_load_results: Py, + ingress_loader_names: Py, +} + +#[pymethods] +impl DirectPythonLifecycle { + /// Record a finished SQL node, then dispatch and report newly ready Python nodes. + fn on_node_complete(&self, py: Python<'_>, node_result: Bound<'_, PyAny>) -> PyResult<()> { + let base: &Bound<'_, PyAny> = self.base_on_node_complete.bind(py); + base.call1((&node_result,))?; + let Some(tracker) = &self.read_side_tracker else { + return Ok(()); + }; + let tracker: &Bound<'_, PyAny> = tracker.bind(py); + let previous: HashSet = result_names(&tracker.getattr("results")?)?; + tracker.call_method1("record_sql_result", (node_result,))?; + let mut new_results: Vec> = Vec::new(); + for result in tracker.getattr("results")?.try_iter()? { + let result: Bound<'_, PyAny> = result?; + if !previous.contains(&result.getattr("node_name")?.extract::()?) { + new_results.push(result); + } + } + self.report(py, &new_results) + } +} + +impl DirectPythonLifecycle { + fn report(&self, py: Python<'_>, results: &[Bound<'_, PyAny>]) -> PyResult<()> { + if results.is_empty() { + return Ok(()); + } + for result in results { + self.base_on_node_complete.bind(py).call1((result,))?; + } + write_python_node_results(self.stream.bind(py), &tuple(py, results)?, self.use_color) + } + + /// Ingress results, then read-side results. + pub(crate) fn python_results<'py>(&self, py: Python<'py>) -> PyResult> { + let mut results: Vec> = self + .ingress_python_results + .bind(py) + .try_iter()? + .collect::>()?; + if let Some(tracker) = &self.read_side_tracker { + for result in tracker.bind(py).getattr("results")?.try_iter()? { + results.push(result?); + } + } + tuple(py, &results) + } + + /// SQL-stage loaders, keeping dependencies Python ingress already satisfied only when needed. + pub(crate) fn loader_functions<'py>(&self, py: Python<'py>) -> PyResult> { + let loaders: Vec> = self + .discovered + .bind(py) + .getattr("loader_functions")? + .try_iter()? + .collect::>()?; + let functions: Vec> = loaders + .iter() + .map(|loader| loader.getattr("function")) + .collect::>()?; + let functions: Bound<'py, PyAny> = frozenset(py, &functions)?; + let ingress: &Bound<'py, PyAny> = self.ingress_loader_names.bind(py); + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let mut handoff: Vec> = Vec::new(); + for loader in loaders { + let keeps_all: bool = !ingress.contains(loader.getattr("name")?)?; + let mut dependencies: Vec> = Vec::new(); + for dependency in loader.getattr("depends_on")?.try_iter()? { + let dependency: Bound<'py, PyAny> = dependency?; + if keeps_all || functions.contains(&dependency)? { + dependencies.push(dependency); + } + } + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item("depends_on", tuple(py, &dependencies)?)?; + handoff.push(replace.call((loader,), Some(&changes))?); + } + tuple(py, &handoff) + } + + fn ingress_keys<'py>(&self, py: Python<'py>, failed_only: bool) -> PyResult> { + let entries: Vec> = self + .plan + .bind(py) + .getattr("source_load_entries")? + .try_iter()? + .collect::>()?; + let mut keys: Vec> = Vec::new(); + for result in self.ingress_load_results.bind(py).try_iter()? { + let result: Bound<'py, PyAny> = result?; + if failed_only + && result.getattr("status")?.str()?.extract::()? == SUCCESS_STATUS + { + continue; + } + let name: Bound<'py, PyAny> = result.getattr("source_name")?; + for entry in &entries { + if entry.getattr("name")?.eq(&name)? { + keys.push(entry.getattr("key")?); + break; + } + } + } + frozenset(py, &keys) + } + + /// Plan keys of SQL source loads Python ingress already ran. + pub(crate) fn precompleted_keys<'py>(&self, py: Python<'py>) -> PyResult> { + self.ingress_keys(py, false) + } + + /// Plan keys of SQL source loads Python ingress ran without success. + pub(crate) fn blocked_keys<'py>(&self, py: Python<'py>) -> PyResult> { + self.ingress_keys(py, true) + } + + pub(crate) fn ingress_load_results<'py>(&self, py: Python<'py>) -> Bound<'py, PyAny> { + self.ingress_load_results.bind(py).clone() + } + + /// Close the read-side connection and report Python nodes that never ran. + pub(crate) fn finalize(&self, py: Python<'_>) -> PyResult<()> { + let connection: Option> = self + .read_side_connection + .lock() + .map_err(|_| PyRuntimeError::new_err("read-side connection lock poisoned"))? + .take(); + if let Some(connection) = connection { + self.adapter.bind(py).call_method1("close", (connection,))?; + } + let Some(tracker) = &self.read_side_tracker else { + return Ok(()); + }; + let finalized: Vec> = tracker + .bind(py) + .call_method0("finalize_unrun_python_nodes")? + .try_iter()? + .collect::>()?; + self.report(py, &finalized) + } +} + +/// The models, seeds and source loads this run executes and reports results for. +fn scheduled_sql_names(plan: &Bound<'_, PyAny>) -> PyResult> { + let scheduled: Bound<'_, PyAny> = + imported(plan.py(), "builtins", "frozenset")?.call1((plan.getattr("execution_order")?,))?; + let mut names: HashSet = HashSet::new(); + for kind in ["model_entries", "seed_entries", "source_load_entries"] { + for entry in plan.getattr(kind)?.try_iter()? { + let entry: Bound<'_, PyAny> = entry?; + if scheduled.contains(entry.getattr("key")?)? { + names.insert(entry.getattr("name")?.extract()?); + } + } + } + Ok(names) +} + +struct RuntimeParts<'a, 'py> { + adapter: &'a Bound<'py, PyAny>, + connection_config: &'a Bound<'py, PyAny>, + project: &'a Bound<'py, PyAny>, + reload: bool, + defaults: (Bound<'py, PyAny>, Bound<'py, PyAny>), + relations: (Bound<'py, PyAny>, Bound<'py, PyAny>), + inputs: &'a LifecycleInputs<'py>, +} + +fn node_runtime<'py>( + parts: &RuntimeParts<'_, 'py>, + connection: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = connection.py(); + let [start_ts, end_ts, start_int, end_int] = &parts.inputs.cursors; + call_with( + &imported(py, EXECUTOR_PYTHON_MODELS, "PythonNodeRuntime")?, + &[ + ("adapter", parts.adapter.clone()), + ("connection_config", parts.connection_config.clone()), + ("connection", connection.clone()), + ("run_id", parts.project.getattr("run_id")?), + ("target", parts.project.getattr("effective_target_name")?), + ("vars", parts.project.getattr("effective_vars")?), + ("is_reload", boolean(py, parts.reload)), + ("default_database", parts.defaults.0.clone()), + ("default_schema", parts.defaults.1.clone()), + ("start_cursor_ts", start_ts.clone()), + ("end_cursor_ts", end_ts.clone()), + ("start_cursor_int", start_int.clone()), + ("end_cursor_int", end_int.clone()), + ("relation_targets", parts.relations.0.clone()), + ("project_relations", parts.relations.1.clone()), + ("providers", parts.inputs.providers.clone()), + ], + ) +} + +/// Lifecycle plan, graph and source map shared by ingress and read-side execution. +struct LifecyclePlan<'py> { + graph: Bound<'py, PyAny>, + plan: Bound<'py, PyAny>, + source_map: Bound<'py, PyAny>, +} + +fn lifecycle_plan<'py>( + discovered: &Bound<'py, PyAny>, + pipeline: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let graph: Bound<'py, PyAny> = call_with( + &imported(py, GRAPH_MODULE, "build_discovered_python_node_graph")?, + &[("discovered_inputs", discovered.clone())], + )?; + let selection: Bound<'py, PyAny> = call_with( + &imported(py, PYTHON_NODE_MODELS, "PythonSqlRunSelection")?, + &[ + ( + "sql_keys", + imported(py, "builtins", "frozenset")? + .call1((plan_output.getattr("upstream_deps")?,))?, + ), + ("python_node_names", pipeline.getattr("python_node_names")?), + ], + )?; + let plan: Bound<'py, PyAny> = call_with( + &imported(py, RUN_LIFECYCLE_MODULE, "build_python_sql_run_lifecycle")?, + &[("selection", selection), ("python_graph", graph.clone())], + )?; + let source_type: Bound<'py, PyAny> = + imported(py, COMPILE_TYPES, "CompiledResourceType")?.getattr("SOURCE")?; + let key_class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS, "CompiledObjectKey")?; + let mut keys: Vec> = Vec::new(); + for name in plan.getattr("ingress_loader_names")?.try_iter()? { + keys.push(call_with( + &key_class, + &[("name", name?), ("resource_type", source_type.clone())], + )?); + } + let source_map: Bound<'py, PyAny> = + imported(py, "builtins", "dict")?.call1((plan_output.getattr("source_map")?,))?; + source_map.call_method1( + "update", + (call_with( + &imported(py, LOADER_DAG_MODULE, "build_intermediate_source_map")?, + &[ + ("project", pipeline.getattr("project")?), + ("selected_keys", frozenset(py, &keys)?), + ], + )?,), + )?; + Ok(LifecyclePlan { + graph, + plan, + source_map, + }) +} + +fn task_asset_names( + discovered: &Bound<'_, PyAny>, + pipeline: &Bound<'_, PyAny>, +) -> PyResult> { + let mut known: HashSet = HashSet::new(); + for kind in ["task_functions", "asset_functions"] { + for node in discovered.getattr(kind)?.try_iter()? { + known.insert(node?.getattr("name")?.extract()?); + } + } + let mut names: Vec = Vec::new(); + for name in pipeline.getattr("python_node_names")?.try_iter()? { + let name: String = name?.extract()?; + if known.contains(&name) { + names.push(name); + } + } + Ok(names) +} + +/// Execute ingress Python and prepare read-side dispatch for a direct build. +pub(crate) fn prepare_direct_python_lifecycle<'py>( + context: &crate::bindings::_helpers::build_command::invocation::BuildContext<'py>, + pipeline: &Bound<'py, PyAny>, + inputs: &LifecycleInputs<'py>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; + let adapter: Bound<'py, PyAny> = context.get("adapter")?; + let connection_config: Bound<'py, PyAny> = context.get("connection_config")?; + let project: Bound<'py, PyAny> = pipeline.getattr("project")?; + let plan_output: Bound<'py, PyAny> = pipeline.getattr("plan_output")?; + let selected: Vec = if context.request_flag("include_python")? { + task_asset_names(&discovered, pipeline)? + } else { + Vec::new() + }; + let lifecycle: LifecyclePlan<'py> = lifecycle_plan(&discovered, pipeline)?; + let ingress_names: Bound<'py, PyAny> = lifecycle.plan.getattr("ingress_python_node_names")?; + let required: Bound<'py, PyAny> = call_with( + &lifecycle.graph.getattr("selected_sql_refs")?, + &[( + "selected_names", + ingress_names.call_method1( + "__or__", + (lifecycle.plan.getattr("read_side_python_node_names")?,), + )?, + )], + )?; + let relation_targets: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_python_relation_targets")?, + &[ + ("adapter", adapter.clone()), + ("project", project.clone()), + ("plan_output", plan_output.clone()), + ("required_refs", required), + ], + )?; + let project_relations: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_project_relation_targets")?, + &[ + ("adapter", adapter.clone()), + ("plan_output", plan_output.clone()), + ], + )?; + let parts: RuntimeParts<'_, 'py> = RuntimeParts { + adapter: &adapter, + connection_config: &connection_config, + project: &project, + reload: context.request_flag("reload_sources")?, + defaults: ( + or_none(project.getattr("effective_target_database")?, || { + adapter.call_method0("default_database") + })?, + or_none(project.getattr("effective_target_schema")?, || { + adapter.call_method0("default_schema") + })?, + ), + relations: (relation_targets, project_relations), + inputs, + }; + let mut ingress_python: Bound<'py, PyAny> = tuple(py, &[])?; + let mut ingress_loads: Bound<'py, PyAny> = tuple(py, &[])?; + if ingress_names.is_truthy()? { + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (connection_config.clone(),))?; + let executed: PyResult> = (|| { + let callbacks: Bound<'py, PyAny> = call_with( + &imported(py, EXECUTOR_PYTHON_MODELS, "IngressCallbacks")?, + &[ + ("use_color", boolean(py, context.use_color)), + ("on_node_start", inputs.on_node_start.clone()), + ("on_node_complete", inputs.on_node_complete.clone()), + ], + )?; + call_with( + &imported(py, INGRESS_MODULE, "run_ingress_python_loader_nodes")?, + &[ + ("python_graph", lifecycle.graph.clone()), + ("selected_python_names", ingress_names.clone()), + ("loader_functions", discovered.getattr("loader_functions")?), + ("source_map", lifecycle.source_map.clone()), + ("runtime", node_runtime(&parts, &connection)?), + ("callbacks", callbacks), + ], + ) + })(); + let closed: PyResult> = adapter.call_method1("close", (connection,)); + let executed: Bound<'py, PyAny> = executed?; + closed?; + ingress_python = executed.getattr("python_results")?; + ingress_loads = executed.getattr("load_results")?; + write_python_node_results(&context.stream, &ingress_python, context.use_color)?; + } + let ingress_set: HashSet = ingress_names + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + let ran: HashSet = result_names(&ingress_python)?; + let read_side: Vec = selected + .into_iter() + .filter(|name| !ingress_set.contains(name) && !ran.contains(name)) + .collect(); + let mut read_side_connection: Option> = None; + let mut read_side_tracker: Option> = None; + if !read_side.is_empty() { + let connection: Bound<'py, PyAny> = + adapter.call_method1("connect", (connection_config.clone(),))?; + read_side_connection = Some(connection.clone().unbind()); + let mut scheduled: HashSet = scheduled_sql_names(&plan_output)?; + for result in ingress_loads.try_iter()? { + scheduled.insert(result?.getattr("source_name")?.extract()?); + } + let scheduled: Vec = scheduled.into_iter().collect(); + let tracker: Bound<'py, PyAny> = call_with( + &imported( + py, + READ_SIDE_MODULE, + "create_read_side_python_execution_tracker", + )?, + &[ + ("python_graph", lifecycle.graph.clone()), + ("selected_python_names", names_set(py, &read_side)?), + ("scheduled_sql_names", names_set(py, &scheduled)?), + ("runtime", node_runtime(&parts, &connection)?), + ], + )?; + for result in ingress_loads.try_iter()? { + tracker.call_method1("record_sql_result", (result?,))?; + } + tracker.call_method0("dispatch_ready_python_nodes")?; + let initial: Bound<'py, PyAny> = tracker.getattr("results")?; + if initial.is_truthy()? { + write_python_node_results(&context.stream, &initial, context.use_color)?; + } + read_side_tracker = Some(tracker.unbind()); + } + Bound::new( + py, + DirectPythonLifecycle { + plan: plan_output.unbind(), + discovered: discovered.unbind(), + adapter: adapter.clone().unbind(), + stream: context.stream.clone().unbind(), + use_color: context.use_color, + base_on_node_complete: inputs.on_node_complete.clone().unbind(), + read_side_connection: Mutex::new(read_side_connection), + read_side_tracker, + ingress_python_results: ingress_python.unbind(), + ingress_load_results: ingress_loads.unbind(), + ingress_loader_names: lifecycle.plan.getattr("ingress_loader_names")?.unbind(), + }, + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs new file mode 100644 index 0000000000..dc7093648e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod command; +pub(crate) mod completion; +pub(crate) mod execution; +pub(crate) mod invocation; +pub(crate) mod lifecycle; +pub(crate) mod planning; +pub(crate) mod policies; +pub(crate) mod registration; +pub(crate) mod support; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs new file mode 100644 index 0000000000..9bd9b0908c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -0,0 +1,247 @@ +//! Compile the build plan, show it, gate it and write the compile target. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_core::build_command::models::PlanSafetyGates; + +use crate::bindings::_helpers::build_command::invocation::BuildContext; +use crate::bindings::_helpers::build_command::policies::{enforce_plan_policies, model_limit}; +use crate::bindings::_helpers::build_command::support::{ + boolean, emit, joined, keyword_relay, none, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const CONSTANTS_MODULE: &str = "sqlbuild.cli.commands.constants"; +const DEFER_CLONE_MODULE: &str = "sqlbuild.cli.commands._helpers.build_planning.defer_clone"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const VERBOSE_INSPECTION_MODULE: &str = "sqlbuild.cli.progress.main.verbose_inspection_output"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; +const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main.build"; +const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; + +/// Clone out-of-selection boundaries from the defer-clone origin before planning. +pub(crate) fn run_defer_clone_prephase(context: &BuildContext<'_>) -> PyResult<()> { + let origin: Bound<'_, PyAny> = context.get("effective_defer_clone_from")?; + if origin.is_none() { + return Ok(()); + } + call_with( + &imported( + context.py(), + DEFER_CLONE_MODULE, + "run_defer_clone_boundary_prephase", + )?, + &[ + ("request", context.request.clone()), + ("invocation", context.invocation.clone()), + ("origin_target_name", origin), + ], + )?; + Ok(()) +} + +fn request_options<'py>(context: &BuildContext<'py>) -> PyResult> { + let py: Python<'py> = context.py(); + let request: &Bound<'py, PyAny> = &context.request; + let field = |name: &str| request.getattr(name); + let should_load_sources: Bound<'py, PyAny> = context.get("should_load_sources")?; + let python_selectors: bool = + context.request_flag("include_python")? || should_load_sources.is_truthy()?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", context.get("effective_project_dir")?), + ("discovered_inputs", context.get("discovered_inputs")?), + ], + )?; + call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &[ + ("no_sql_validation", field("no_sql_validation")?), + ("no_cache", field("no_cache")?), + ("selected_target", field("selected_target")?), + ("defer_to", field("defer_to")?), + ("defer_sources_to", field("defer_sources_to")?), + ("cursor_overrides", field("cursor_overrides")?), + ("select", field("select")?), + ("exclude", field("exclude")?), + ("full_refresh", field("full_refresh")?), + ("auto_load_sources", should_load_sources), + ("reload_sources", field("reload_sources")?), + ("connection_config", context.get("connection_config")?), + ("cli_vars", field("cli_vars")?), + ("external_sql_reference_resolver", resolver), + ( + "resolve_python_run_selectors", + boolean(py, python_selectors), + ), + ("max_microbatches", field("max_microbatches")?), + ("selection_diagnostics", field("selection_diagnostics")?), + ("plan_sql_tests", field("run_tests")?), + ("accepts_unit_test_selectors", boolean(py, true)), + ], + ) +} + +/// Run the compile pipeline with planning and connection progress. +pub(crate) fn compile_build_plan<'py>(context: &BuildContext<'py>) -> PyResult> { + let py: Python<'py> = context.py(); + emit(&context.stream, "\n")?; + let connection: Bound<'py, PyAny> = context.get("connection_progress")?; + let hooks: Bound<'py, PyAny> = call_with( + &imported(py, RUNTIME_CONTRACT_MODELS, "ConnectionHooks")?, + &[ + ( + "on_progress", + context.get("planning_progress")?.getattr("on_progress")?, + ), + ( + "on_connection_start", + connection.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + keyword_relay( + connection.getattr("on_connection_complete")?, + &CONNECTION_TIMING, + )?, + ), + ( + "on_connection_error", + keyword_relay( + connection.getattr("on_connection_error")?, + &CONNECTION_TIMING, + )?, + ), + ], + )?; + let manager: Bound<'py, PyAny> = call_with( + &imported(py, VERBOSE_INSPECTION_MODULE, "verbose_inspection_output")?, + &[("enabled", context.request.getattr("verbose")?)], + )?; + within(&manager, |_| { + call_with( + &imported(py, NATIVE_MODULE, "run_compile_pipeline")?, + &[ + ("discovered_inputs", context.get("discovered_inputs")?), + ("adapter", context.get("adapter")?), + ("options", request_options(context)?), + ("hooks", hooks), + ], + ) + }) +} + +/// Write the formatted plan to stdout, or stderr for debug and JSON runs. +pub(crate) fn write_plan_text( + context: &BuildContext<'_>, + pipeline: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = context.py(); + let machine: bool = context.request_flag("debug")? || context.request_flag("json_output")?; + let stream: Bound<'_, PyAny> = + py.import("sys")? + .getattr(if machine { "stderr" } else { "stdout" })?; + let plan_text: String = call_with( + &imported(py, NATIVE_MODULE, "format_plan")?, + &[ + ("plan", pipeline.getattr("plan_output")?), + ("full_refresh", context.request.getattr("full_refresh")?), + ("use_color", boolean(py, context.use_color)), + ( + "python_plan_entries", + pipeline.getattr("python_plan_entries")?, + ), + ("include_direct_freshness_diagnostics", boolean(py, false)), + ], + )? + .extract()?; + emit(&stream, &format!("\n{plan_text}\n\n")) +} + +fn constant(py: Python<'_>, name: &str) -> PyResult { + imported(py, CONSTANTS_MODULE, name)?.extract() +} + +/// Apply every plan safety gate with the build's allow flags. +pub(crate) fn enforce_build_plan_policies( + context: &BuildContext<'_>, + pipeline: &Bound<'_, PyAny>, +) -> Result<(), PyErr> { + let py: Python<'_> = context.py(); + let gates: PlanSafetyGates = PlanSafetyGates { + allow_missing_migration_origin: context.request_flag("allow_missing_migration_origin")?, + allow_snapshot_full_refresh: context.request_flag("allow_snapshot_full_refresh")?, + allow_table_type_downgrade: context.request_flag("allow_table_type_downgrade")?, + allow_retention_decrease: context.request_flag("allow_retention_decrease")?, + missing_origin_help: constant(py, "MISSING_ORIGIN_BUILD_HELP")?, + snapshot_full_refresh_help: constant(py, "SNAPSHOT_FULL_REFRESH_BUILD_HELP")?, + table_type_downgrade_help: constant(py, "TABLE_TYPE_DOWNGRADE_BUILD_HELP")?, + retention_decrease_help: constant(py, "RETENTION_DECREASE_BUILD_HELP")?, + execution_limit_note: constant(py, "EXECUTION_LIMIT_BUILD_NOTE")?, + }; + let sys: Bound<'_, PyAny> = py.import("sys")?.into_any(); + enforce_plan_policies( + &pipeline.getattr("plan_output")?, + ( + &gates, + &context + .get("discovered_inputs")? + .getattr("project_config")? + .getattr("snapshots")?, + ), + &model_limit( + &context.get("effective_target_name")?, + &context.get("execution_limits")?, + )?, + (&sys.getattr("stdin")?, &sys.getattr("stdout")?), + ) +} + +/// Write the compile target artifacts with the optional manifest payload. +pub(crate) fn write_compile_target( + context: &BuildContext<'_>, + pipeline: &Bound<'_, PyAny>, +) -> PyResult<()> { + let py: Python<'_> = context.py(); + let plan: Bound<'_, PyAny> = pipeline.getattr("plan_output")?; + let manifest: Bound<'_, PyAny> = if context.request_flag("manifest")? { + call_with( + &imported(py, MANIFEST_MODULE, "build_manifest")?, + &[ + ("project", pipeline.getattr("project")?), + ("plan_output", plan.clone()), + ( + "project_name", + context + .get("discovered_inputs")? + .getattr("project_config")? + .getattr("name")?, + ), + ("adapter_type", context.get("adapter_name")?), + ("upstream_deps", plan.getattr("upstream_deps")?), + ("downstream_deps", plan.getattr("downstream_deps")?), + ], + )? + } else { + none(py) + }; + call_with( + &imported(py, NATIVE_MODULE, "write_compile_target")?, + &[ + ( + "target_dir", + joined(&context.get("effective_project_dir")?, &["target"])?, + ), + ("adapter", context.get("adapter")?), + ("plan_output", plan), + ("manifest", manifest), + ], + )?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs new file mode 100644 index 0000000000..a63ccadc3a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs @@ -0,0 +1,292 @@ +//! Plan safety gates and the model-count limit, read from a plan and enforced on the given streams. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::build_command::constants::POLICY_REQUIRE_CONFIRMATION; +use sqlbuild_core::build_command::errors::BuildCommandError; +use sqlbuild_core::build_command::main::model_limit_error::model_limit_error; +use sqlbuild_core::build_command::main::plan_safety_steps::plan_safety_steps; +use sqlbuild_core::build_command::main::render_confirmation_prompt::render_confirmation_prompt; +use sqlbuild_core::build_command::models::{ + ColumnMigrationFact, ModelLimit, ModelMigrationFact, PlanSafetyFacts, PlanSafetyGates, + PlanWarningFact, PolicyStep, RetentionFact, SnapshotPolicies, SnapshotRefreshFact, + TableTypeFact, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; + +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const SNAPSHOT_VALUE: &str = "snapshot"; +const FULL_REFRESH_REASON: &str = "full_refresh"; +const SKIP_ACTION: &str = "skip"; +const ERROR_SEVERITY: &str = "error"; +const POLICY_OPTIONS: (&[&str], &[&str]) = ( + &[ + "plan", + "gates", + "snapshots_config", + "target_name", + "execution_limits", + "streams", + ], + &[], +); +const LIMIT_OPTIONS: (&[&str], &[&str]) = ( + &["model_count", "target_name", "limits", "refusal_note"], + &[], +); + +/// `CliUserError(message, code=..., help=...)`. +pub(crate) fn build_command_error(py: Python<'_>, error: BuildCommandError) -> PyErr { + let raised: PyResult> = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", error.code)?; + options.set_item("help", error.help)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((error.message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + value.getattr(name)?.str()?.extract() +} + +fn optional_text(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let attribute: Bound<'_, PyAny> = value.getattr(name)?; + if attribute.is_none() { + return Ok(None); + } + Ok(Some(attribute.str()?.extract()?)) +} + +fn flag(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + value.getattr(name)?.is_truthy() +} + +fn model_migration(entry: &Bound<'_, PyAny>) -> PyResult { + let origin: Bound<'_, PyAny> = entry.getattr("origin")?; + let qualified: Bound<'_, PyAny> = origin.getattr("qualified_name")?; + let origin_label: String = if qualified.is_truthy()? { + qualified.str()?.extract()? + } else { + text(&origin, "name")? + }; + Ok(ModelMigrationFact { + model_name: text(entry, "model_name")?, + origin_label, + decision: text(entry, "decision")?, + blocks_build: flag(entry, "blocks_build")?, + origin_hidden: flag(entry, "origin_hidden")?, + origin_missing: flag(entry, "origin_missing")?, + confirm_missing_origin: text(entry, "missing_origin_policy")? + == POLICY_REQUIRE_CONFIRMATION, + }) +} + +fn column_migration(entry: &Bound<'_, PyAny>) -> PyResult { + let decision: Bound<'_, PyAny> = entry.getattr("decision")?; + Ok(ColumnMigrationFact { + model_name: text(entry, "model_name")?, + destination_column: text(entry, "destination_column")?, + origin_column: text(entry, "origin_column")?, + decision: decision.str()?.extract()?, + decision_label: text(&decision, "label")?, + blocks_build: flag(entry, "blocks_build")?, + origin_missing: flag(entry, "origin_missing")?, + confirm_missing_origin: text(entry, "missing_origin_policy")? + == POLICY_REQUIRE_CONFIRMATION, + }) +} + +fn entries<'py>(plan: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + plan.getattr(name)?.try_iter()?.collect() +} + +/// The models a plan executes: every model entry that is not skipped. +pub(crate) fn executable_model_count(plan: &Bound<'_, PyAny>) -> PyResult { + let mut count: usize = 0; + for entry in entries(plan, "model_entries")? { + if text(&entry, "action")? != SKIP_ACTION { + count += 1; + } + } + Ok(count) +} + +fn plan_facts(plan: &Bound<'_, PyAny>) -> PyResult { + let mut facts: PlanSafetyFacts = PlanSafetyFacts { + executable_model_count: executable_model_count(plan)?, + ..PlanSafetyFacts::default() + }; + for entry in entries(plan, "migration_entries")? { + facts.model_migrations.push(model_migration(&entry)?); + } + for entry in entries(plan, "column_migration_entries")? { + facts.column_migrations.push(column_migration(&entry)?); + } + for warning in entries(plan, "warnings")? { + if text(&warning, "severity")? == ERROR_SEVERITY { + facts.error_warnings.push(PlanWarningFact { + code: optional_text(&warning, "code")?, + message: text(&warning, "message")?, + }); + } + } + for entry in entries(plan, "model_entries")? { + if optional_text(&entry, "materialization_type")?.as_deref() == Some(SNAPSHOT_VALUE) + && text(&entry, "reason")? == FULL_REFRESH_REASON + { + facts.snapshot_refreshes.push(SnapshotRefreshFact { + name: text(&entry, "name")?, + historical: !entry.getattr("observed_at_column")?.is_none(), + model_policy: optional_text(&entry, "snapshot_full_refresh")?, + }); + } + } + for entry in entries(plan, "table_type_entries")? { + if flag(&entry, "downgrade")? { + facts.table_type_downgrades.push(TableTypeFact { + model_name: text(&entry, "model_name")?, + policy: text(&entry, "downgrade_policy")?, + }); + } + } + for entry in entries(plan, "retention_entries")? { + if flag(&entry, "decreases")? { + facts.retention_decreases.push(RetentionFact { + model_names: entry.getattr("model_names")?.extract()?, + policy: text(&entry, "decrease_policy")?, + }); + } + } + Ok(facts) +} + +fn read_gates(value: &Bound<'_, PyAny>) -> PyResult { + Ok(PlanSafetyGates { + allow_missing_migration_origin: flag(value, "allow_missing_migration_origin")?, + allow_snapshot_full_refresh: flag(value, "allow_snapshot_full_refresh")?, + allow_table_type_downgrade: flag(value, "allow_table_type_downgrade")?, + allow_retention_decrease: flag(value, "allow_retention_decrease")?, + missing_origin_help: text(value, "missing_origin_help")?, + snapshot_full_refresh_help: text(value, "snapshot_full_refresh_help")?, + table_type_downgrade_help: text(value, "table_type_downgrade_help")?, + retention_decrease_help: text(value, "retention_decrease_help")?, + execution_limit_note: text(value, "execution_limit_note")?, + }) +} + +/// The target's model limit from its `ExecutionLimitsConfig`. +pub(crate) fn model_limit( + target_name: &Bound<'_, PyAny>, + limits: &Bound<'_, PyAny>, +) -> PyResult { + Ok(ModelLimit { + target_name: target_name.extract()?, + max_models: limits.getattr("max_models")?.extract()?, + remediation: limits.getattr("remediation")?.extract()?, + }) +} + +/// Ask for each confirmation on the terminal and raise each refusal, in order. +fn apply_steps( + py: Python<'_>, + steps: Vec, + (input, output): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + for step in steps { + let confirmation = match step { + PolicyStep::Refuse(error) => return Err(build_command_error(py, error)), + PolicyStep::Confirm(confirmation) => confirmation, + }; + if !input.call_method0("isatty")?.is_truthy()? { + return Err(build_command_error(py, confirmation.refusal)); + } + output.call_method1("write", (render_confirmation_prompt(&confirmation),))?; + output.call_method0("flush")?; + let answer: String = input.call_method0("readline")?.extract()?; + if answer.trim() != confirmation.expected { + return Err(build_command_error(py, confirmation.cancellation)); + } + } + Ok(()) +} + +/// Apply every plan safety gate to a plan, prompting on the given streams. +pub(crate) fn enforce_plan_policies( + plan: &Bound<'_, PyAny>, + (gates, snapshots_config): (&PlanSafetyGates, &Bound<'_, PyAny>), + limit: &ModelLimit, + streams: (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> Result<(), PyErr> { + let py: Python<'_> = plan.py(); + let policies: SnapshotPolicies = SnapshotPolicies { + historical: text(snapshots_config, "historical_full_refresh")?, + current_state: text(snapshots_config, "current_state_full_refresh")?, + }; + let steps: Vec = plan_safety_steps(&plan_facts(plan)?, gates, (limit, &policies)) + .map_err(|error| build_command_error(py, error))?; + apply_steps(py, steps, streams) +} + +/// Apply every plan safety gate before a build executes anything. +#[pyfunction] +#[pyo3(name = "enforce_plan_safety_policies", signature = (**options))] +fn enforce_plan_safety_policies_binding<'py>( + py: Python<'py>, + options: Option>, +) -> Result<(), PyErr> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "enforce_plan_safety_policies", options, POLICY_OPTIONS)?; + let streams: (Bound<'py, PyAny>, Bound<'py, PyAny>) = arguments.value("streams")?.extract()?; + enforce_plan_policies( + &arguments.value("plan")?, + ( + &read_gates(&arguments.value("gates")?)?, + &arguments.value("snapshots_config")?, + ), + &model_limit( + &arguments.value("target_name")?, + &arguments.value("execution_limits")?, + )?, + (&streams.0, &streams.1), + ) +} + +/// Reject a build whose expanded model count exceeds its target policy. +#[pyfunction] +#[pyo3(name = "enforce_model_execution_limit", signature = (**options))] +fn enforce_model_execution_limit_binding<'py>( + py: Python<'py>, + options: Option>, +) -> Result<(), PyErr> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "enforce_model_execution_limit", options, LIMIT_OPTIONS)?; + let note: String = arguments.value("refusal_note")?.extract()?; + let limit: ModelLimit = model_limit( + &arguments.value("target_name")?, + &arguments.value("limits")?, + )?; + match model_limit_error(arguments.value("model_count")?.extract()?, &limit, ¬e) { + Some(error) => Err(build_command_error(py, error)), + None => Ok(()), + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + enforce_plan_safety_policies_binding, + module + )?)?; + module.add_function(wrap_pyfunction!( + enforce_model_execution_limit_binding, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/registration.rs new file mode 100644 index 0000000000..4263b665d3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb build` command and its plan safety gates. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::build_command::{command, policies}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + policies::register(module)?; + command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs new file mode 100644 index 0000000000..30feeb51c3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/support.rs @@ -0,0 +1,146 @@ +//! Small Python value helpers and callback relays the build command shares. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyFrozenSet, PyInt, PyString, PyTuple, PyTupleMethods, +}; +use pyo3::{IntoPyObject, pyclass, pymethods}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const DATETIME_MODULE: &str = "datetime"; + +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +pub(crate) fn boolean(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} + +pub(crate) fn none(py: Python<'_>) -> Bound<'_, PyAny> { + py.None().into_bound(py) +} + +pub(crate) fn tuple<'py>( + py: Python<'py>, + values: &[Bound<'py, PyAny>], +) -> PyResult> { + Ok(PyTuple::new(py, values)?.into_any()) +} + +pub(crate) fn frozenset<'py>( + py: Python<'py>, + values: &[Bound<'py, PyAny>], +) -> PyResult> { + Ok(PyFrozenSet::new(py, values)?.into_any()) +} + +/// The value, or `default` when it is `None`. +pub(crate) fn or_none<'py>( + value: Bound<'py, PyAny>, + default: impl FnOnce() -> PyResult>, +) -> PyResult> { + if value.is_none() { + return default(); + } + Ok(value) +} + +/// `base / part`. +pub(crate) fn joined<'py>(base: &Bound<'py, PyAny>, parts: &[&str]) -> PyResult> { + let mut path: Bound<'py, PyAny> = base.clone(); + for part in parts { + path = path.call_method1("__truediv__", (*part,))?; + } + Ok(path) +} + +/// Write `value` to the stream and flush it. +pub(crate) fn emit(stream: &Bound<'_, PyAny>, value: &str) -> PyResult<()> { + stream.call_method1("write", (value,))?; + stream.call_method0("flush")?; + Ok(()) +} + +/// `datetime.now(UTC)`. +pub(crate) fn utc_now(py: Python<'_>) -> PyResult> { + let utc: Bound<'_, PyAny> = imported(py, DATETIME_MODULE, "UTC")?; + imported(py, DATETIME_MODULE, "datetime")?.call_method1("now", (utc,)) +} + +/// A string, integer or float shown as text; anything else is unset. +pub(crate) fn display_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() + || !(value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::()) + { + return Ok(None); + } + Ok(Some(value.str()?.extract()?)) +} + +/// Forward positional callback arguments to a callable that takes them by keyword. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct KeywordRelay { + target: Py, + names: Vec<&'static str>, +} + +#[pymethods] +impl KeywordRelay { + #[pyo3(signature = (*args, **kwargs))] + fn __call__( + &self, + py: Python<'_>, + args: Bound<'_, PyTuple>, + kwargs: Option>, + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + for (name, value) in self.names.iter().zip(args.iter()) { + options.set_item(*name, value)?; + } + if let Some(kwargs) = kwargs { + options.update(kwargs.as_mapping())?; + } + self.target.bind(py).call((), Some(&options))?; + Ok(()) + } +} + +/// A relay calling `target` with `names` as keywords. +pub(crate) fn keyword_relay<'py>( + target: Bound<'py, PyAny>, + names: &[&'static str], +) -> PyResult> { + let py: Python<'py> = target.py(); + Ok(KeywordRelay { + target: target.unbind(), + names: names.to_vec(), + } + .into_pyobject(py)? + .into_any()) +} + +/// Write each message as one flushed line. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ProgressLineWriter { + stream: Py, +} + +#[pymethods] +impl ProgressLineWriter { + fn __call__(&self, py: Python<'_>, message: &str) -> PyResult<()> { + emit(self.stream.bind(py), &format!("{message}\n")) + } +} + +/// A callable writing each message as a line on `stream`. +pub(crate) fn line_writer<'py>(stream: &Bound<'py, PyAny>) -> PyResult> { + Ok(ProgressLineWriter { + stream: stream.clone().unbind(), + } + .into_pyobject(stream.py())? + .into_any()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs index a53c860a6b..bd9d3ff0ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/check/check_command.rs @@ -24,21 +24,21 @@ const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_ const CHECK_CORE_MODULE: &str = "sqlbuild.cli.commands._helpers.check.core"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const RELATION_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._relation_plan"; const SELECTOR_PARSE_MODULE: &str = "sqlbuild.compiler.planner.main.selection.selector_parse"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES: &str = "sqlbuild.compiler.compile.types"; -const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; +const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const PYTHON_SELECTORS_MODULE: &str = "sqlbuild.compiler.python_nodes.main.selectors"; -const PYTHON_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main.run_lifecycle"; +const PYTHON_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; const PYTHON_NODE_MODELS: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES: &str = "sqlbuild.compiler.python_nodes.types"; const EXECUTOR_MODELS: &str = "sqlbuild.executor.python_nodes.models"; -const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main.checks"; -const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main.ingress"; +const CHECKS_MODULE: &str = "sqlbuild.executor.python_nodes.main._checks"; +const INGRESS_MODULE: &str = "sqlbuild.executor.python_nodes.main._ingress"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; const RUNTIME_TARGET_MODULE: &str = "sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index 032882f74a..90df2f52a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -1,15 +1,19 @@ //! `sqb` command dispatch: the native route table applied to a parsed namespace. +use std::path::PathBuf; + use crate::bindings::_helpers::diff_report::support::cli_error; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; +use sqlbuild_cli::diff_report::errors::DiffUserError; use sqlbuild_cli::diff_report::main::parse_diff_name_range::parse_diff_name_range; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; use sqlbuild_cli::dispatch::main::unavailable_handler::unavailable_handler; use sqlbuild_cli::dispatch::models::{ ClassRef, CommandFacts, DiffSide, Dispatch, DispatchFailure, Field, Route, Source, }; +use sqlbuild_core::format_command::main::format_path_selectors::format_path_selectors; use sqlbuild_core::text::main::python_strip::python_strip; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; @@ -19,7 +23,6 @@ const ENTRY_MODELS_MODULE: &str = "sqlbuild.cli.entry.models"; const WAREHOUSES_MODULE: &str = "sqlbuild._native"; const WAREHOUSE_SCOPE_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.warehouse_scope"; const LAUNCH_RECORD_MODULE: &str = "sqlbuild.cli.entry.main._withhold_launch_record"; -const FORMAT_PATHS_MODULE: &str = "sqlbuild.cli.commands._helpers.lint.positional_paths"; const PATHLIB_MODULE: &str = "pathlib"; const COMMENT_PREFIX: &str = "#"; const UTF8: &str = "utf-8"; @@ -100,15 +103,36 @@ impl<'py> Invocation<'py> { } Source::DiffName(side) => self.diff_name(side), Source::FormatSelect => { - let paths: Bound<'py, PyAny> = call_with( - &imported(py, FORMAT_PATHS_MODULE, "format_path_selectors")?, - &[ - ("paths", tuple_of(&self.attr("format_paths")?)?), - ("project_dir", self.project_dir.clone()), - ("cwd", path_class(py)?.call_method0("cwd")?), - ], - )?; - concatenated(&self.select.clone().into_any(), &paths) + let raw_paths: Bound<'py, PyAny> = self.attr("format_paths")?; + let raw_paths: Vec = if raw_paths.is_none() { + Vec::new() + } else { + raw_paths.extract()? + }; + let project_dir: Option = if self.project_dir.is_none() { + None + } else { + Some(self.project_dir.extract()?) + }; + let cwd: PathBuf = path_class(py)?.call_method0("cwd")?.extract()?; + let selectors: Vec = + format_path_selectors(&raw_paths, project_dir.as_deref(), &cwd).map_err( + |error| { + cli_error( + py, + "CliUserError", + &DiffUserError { + message: error.message, + code: error.code.to_owned(), + help: error.help, + }, + ) + }, + )?; + concatenated( + &self.select.clone().into_any(), + &PyTuple::new(py, selectors)?.into_any(), + ) } Source::ScenarioSelect => concatenated( &self.attr("scenario_selector")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 179b9abaf4..7f2a199b2f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -4,9 +4,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::cli::{argv, entry, progress, progress_reporters, progress_text}; use crate::bindings::_helpers::{ - check, clone, command_runtime, cost_report, dbt_interop, debug_command, diff_report, freshness, - janitor, load, presentation_text, result_output, seed_command, sql_test_command, - workspace_commands, + audit_command, build_command, check, clone, command_runtime, contract_command, cost_command, + cost_report, dbt_interop, debug_command, diff_report, format_command, freshness, init_command, + janitor, load, presentation_text, query_command, result_output, scenario_command, seed_command, + sql_test_command, workspace_commands, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -30,5 +31,13 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { check::registration::register(module)?; sql_test_command::registration::register(module)?; seed_command::registration::register(module)?; + audit_command::registration::register(module)?; + build_command::registration::register(module)?; + query_command::registration::register(module)?; + scenario_command::registration::register(module)?; + contract_command::registration::register(module)?; + init_command::registration::register(module)?; + format_command::registration::register(module)?; + cost_command::registration::register(module)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index 8a3277ae13..ce8694d646 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -14,7 +14,7 @@ use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main._build_graph_manifest"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; -const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; +const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs index 083dba3be3..8e3d4733bd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs @@ -16,8 +16,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const PIPELINE_MODELS_MODULE: &str = "sqlbuild.compiler.pipeline.models"; const PLANNER_MODELS_MODULE: &str = "sqlbuild.compiler.planner.models"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; -const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; -const RUN_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main.run_lifecycle"; +const NODE_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; +const RUN_LIFECYCLE_MODULE: &str = "sqlbuild.compiler.python_nodes.main._run_lifecycle"; const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.compiler.python_nodes.models"; const PYTHON_NODE_TYPES_MODULE: &str = "sqlbuild.compiler.python_nodes.types"; const REF_TYPES_MODULE: &str = "sqlbuild.python_nodes.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs new file mode 100644 index 0000000000..b710376283 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs @@ -0,0 +1,457 @@ +//! `sqb contract`: compare selected contracts with one target, optionally adopt them. + +use std::time::Instant; + +use pyo3::exceptions::{PyException, PyValueError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{ + PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString, PyTuple, PyTupleMethods, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::contract_command::main::contract_exit_code::contract_exit_code; +use sqlbuild_core::contract_command::main::contract_progress_message::contract_progress_message; +use sqlbuild_core::contract_command::main::render_contract_json::render_contract_json; +use sqlbuild_core::contract_command::main::render_contract_text::render_contract_text; +use sqlbuild_core::contract_command::models::{ + ContractFindingText, ContractProgress, ContractReport, ContractResourceText, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const ADAPTER_CONTEXT_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapter_context"; +const COMPILE_MODELS: &str = "sqlbuild.compiler.compile.models"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; +const ADOPTION_MODELS: &str = "sqlbuild.compiler.contract_adoption.models"; +const WRITE_MODULE: &str = "sqlbuild.compiler.contract_adoption.main._write"; +const GENERATE_ACTION: &str = "generate"; + +/// A `CliUserError` with its code. +fn user_error(py: Python<'_>, message: &str, code: &str) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", code)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn optional_text(value: Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.to_string())) +} + +/// The selected graph, compiled against the source target, and its selected models and sources. +struct Selection<'py> { + graph: Bound<'py, PyAny>, + models: Bound<'py, PyTuple>, + sources: Bound<'py, PyTuple>, +} + +fn selected_of<'py>( + resources: Bound<'py, PyAny>, + selected: &Bound<'py, PyAny>, +) -> PyResult> { + let mut kept: Vec> = Vec::new(); + for resource in resources.try_iter()? { + let resource: Bound<'py, PyAny> = resource?; + if selected.contains(resource.getattr("key")?)? { + kept.push(resource); + } + } + PyTuple::new(resources.py(), kept) +} + +fn select_resources<'py>( + request: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let analysis: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS, "CompileAnalysisSelection")?, + &[("no_cache", PyBool::new(py, true).to_owned().into_any())], + )?; + let project: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_compiled_project")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", context.getattr("adapter")?), + ("selected_target", request.getattr("from_target")?), + ("cli_vars", request.getattr("cli_vars")?), + ("resolved_connection", context.getattr("connection_config")?), + ("analysis_selection", analysis), + ], + )?; + let graph: Bound<'py, PyAny> = call_with( + &imported( + py, + NATIVE_MODULE, + "build_project_graph_from_compiled_project", + )?, + &[("project", project)], + )?; + let selected: Bound<'py, PyAny> = call_with( + &imported(py, GRAPH_SELECTION_MODULE, "select_project_graph")?, + &[ + ("graph", graph.clone()), + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ], + )?; + let compiled: Bound<'py, PyAny> = graph.getattr("project")?; + let models: Bound<'py, PyTuple> = selected_of(compiled.getattr("models")?, &selected)?; + let sources: Bound<'py, PyTuple> = selected_of(compiled.getattr("sources")?, &selected)?; + if models.is_empty() && sources.is_empty() { + return Err(user_error( + py, + "contract selection contains no models or sources", + "C473", + )); + } + Ok(Selection { + graph, + models, + sources, + }) +} + +/// Where progress goes and whether it is coloured. +struct Progress<'py> { + stream: Bound<'py, PyAny>, + use_color: bool, +} + +impl<'py> Progress<'py> { + fn status(&self) -> PyResult> { + let py: Python<'py> = self.stream.py(); + call_with( + &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, + &[ + ("stream", self.stream.clone()), + ( + "use_color", + PyBool::new(py, self.use_color).to_owned().into_any(), + ), + ], + ) + } +} + +fn exception(py: Python<'_>, error: &PyErr) -> bool { + error.is_instance_of::(py) +} + +fn connection_timing(reporter: &Bound<'_, PyAny>, method: &str, started: Instant) -> PyResult<()> { + let py: Python<'_> = reporter.py(); + call_with( + &reporter.getattr(method)?, + &[ + ("connection_count", PyInt::new(py, 1).into_any()), + ( + "elapsed_seconds", + PyFloat::new(py, started.elapsed().as_secs_f64()).into_any(), + ), + ], + )?; + Ok(()) +} + +/// Compare the selected contracts over one connection that is always closed. +fn compare<'py>( + request: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, + selection: &Selection<'py>, + progress: &Progress<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let adapter: Bound<'py, PyAny> = context.getattr("adapter")?; + let reporter: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", context.getattr("adapter_name")?), + ("stream", progress.stream.clone()), + ( + "use_color", + PyBool::new(py, progress.use_color).to_owned().into_any(), + ), + ], + )?; + let connection_started: Instant = Instant::now(); + reporter.call_method1("on_connection_start", (1,))?; + let connection: Bound<'py, PyAny> = + match adapter.call_method1("connect", (context.getattr("connection_config")?,)) { + Ok(connection) => connection, + Err(error) => { + if exception(py, &error) { + connection_timing(&reporter, "on_connection_error", connection_started)?; + } + return Err(error); + } + }; + let compared: PyResult> = (|| { + connection_timing(&reporter, "on_connection_complete", connection_started)?; + let count: usize = selection.models.len() + selection.sources.len(); + let target: String = request.getattr("from_target")?.str()?.to_string(); + let status: Bound<'py, PyAny> = progress.status()?; + let started: Instant = Instant::now(); + status.call_method1( + "start", + (contract_progress_message( + &ContractProgress::InspectionStart { + count, + target: target.clone(), + }, + ),), + )?; + let evidence: Bound<'py, PyAny> = match call_with( + &imported(py, NATIVE_MODULE, "compare_contracts")?, + &[ + ("adapter", adapter.clone()), + ("connection", connection.clone()), + ("models", selection.models.clone().into_any()), + ("sources", selection.sources.clone().into_any()), + ], + ) { + Ok(evidence) => evidence, + Err(error) => { + if exception(py, &error) { + status.call_method1( + "error", + (contract_progress_message( + &ContractProgress::InspectionFailed { + seconds: started.elapsed().as_secs_f64(), + }, + ),), + )?; + } + return Err(error); + } + }; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item( + "message", + contract_progress_message(&ContractProgress::InspectionComplete { + count, + target, + seconds: started.elapsed().as_secs_f64(), + }), + )?; + status.call_method("complete", (), Some(&options))?; + Ok(evidence) + })(); + let closed: PyResult> = adapter.call_method1("close", (connection,)); + let evidence: Bound<'py, PyAny> = compared?; + closed?; + Ok(evidence) +} + +/// Write the adopted declarations into the repository and report the files written. +fn write_updates<'py>( + request: &Bound<'py, PyAny>, + (project_dir, context): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (graph, result): (&Bound<'py, PyAny>, Bound<'py, PyAny>), + progress: &Progress<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let status: Bound<'py, PyAny> = progress.status()?; + let started: Instant = Instant::now(); + status.call_method1("start", ("Updating repository contracts...",))?; + let updated: Bound<'py, PyAny> = match call_with( + &imported(py, WRITE_MODULE, "write_contracts")?, + &[ + ("project_dir", project_dir.clone()), + ("graph", graph.clone()), + ("result", result), + ("overwrite", request.getattr("overwrite")?), + ("adapter", context.getattr("adapter")?), + ("resolved_connection", context.getattr("connection_config")?), + ("cli_vars", request.getattr("cli_vars")?), + ], + ) { + Ok(updated) => updated, + Err(error) => { + if exception(py, &error) { + status.call_method1( + "error", + (contract_progress_message(&ContractProgress::WriteFailed { + seconds: started.elapsed().as_secs_f64(), + }),), + )?; + } + return Err(error); + } + }; + let written: usize = updated.getattr("written_paths")?.len()?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item( + "message", + contract_progress_message(&ContractProgress::WriteComplete { + count: written, + seconds: started.elapsed().as_secs_f64(), + }), + )?; + status.call_method("complete", (), Some(&options))?; + Ok(updated) +} + +fn finding_text(finding: Bound<'_, PyAny>) -> PyResult { + Ok(ContractFindingText { + kind: finding.getattr("kind")?.str()?.to_string(), + column: optional_text(finding.getattr("column_name")?)?, + declared_type: optional_text(finding.getattr("declared_type")?)?, + physical_type: optional_text(finding.getattr("physical_type")?)?, + message: finding.getattr("message")?.str()?.to_string(), + }) +} + +fn resource_text(item: Bound<'_, PyAny>) -> PyResult { + let mut parts: Vec = Vec::new(); + for field in ["database", "schema", "relation"] { + let value: Bound<'_, PyAny> = item.getattr(field)?; + if value.is_truthy()? { + parts.push(value.str()?.to_string()); + } + } + Ok(ContractResourceText { + resource_type: item.getattr("resource_type")?.str()?.to_string(), + resource_name: item.getattr("resource_name")?.str()?.to_string(), + relation: parts.join("."), + findings: item + .getattr("findings")? + .try_iter()? + .map(|finding| finding_text(finding?)) + .collect::>()?, + }) +} + +fn report(result: &Bound<'_, PyAny>) -> PyResult { + Ok(ContractReport { + from_target: result.getattr("from_target")?.str()?.to_string(), + resources: result + .getattr("evidence")? + .try_iter()? + .map(|item| resource_text(item?)) + .collect::>()?, + written_paths: result + .getattr("written_paths")? + .try_iter()? + .map(|path| Ok(path?.str()?.to_string())) + .collect::>()?, + }) +} + +fn progress_for<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let json_output: bool = request.getattr("json_output")?.is_truthy()?; + let stream: Bound<'py, PyAny> = + py.import("sys")? + .getattr(if json_output { "stderr" } else { "stdout" })?; + let use_color: bool = !request.getattr("no_color")?.is_truthy()? + && !json_output + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + Ok(Progress { stream, use_color }) +} + +/// Compare or adopt selected contracts from one read-only target. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_contract_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let action: String = request.getattr("action")?.str()?.to_string(); + let generate_write: bool = + action == GENERATE_ACTION && request.getattr("write")?.is_truthy()?; + if request.getattr("overwrite")?.is_truthy()? && !generate_write { + return Err(user_error( + py, + "contract --overwrite requires generate --write", + "C471", + )); + } + let requested_dir: Bound<'_, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'_, PyAny> = if requested_dir.is_truthy()? { + requested_dir + } else { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + }; + let discovered: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", request.getattr("select")?), + ("exclude", request.getattr("exclude")?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let from_target: Bound<'_, PyAny> = request.getattr("from_target")?; + if !discovered + .getattr("project_config")? + .getattr("targets")? + .contains(&from_target)? + { + return Err(user_error( + py, + &format!("unknown contract source target '{}'", from_target.str()?), + "C472", + )); + } + let context: Bound<'_, PyAny> = call_with( + &imported( + py, + ADAPTER_CONTEXT_MODULE, + "resolve_adapter_connection_context", + )?, + &[ + ("discovered_inputs", discovered.clone()), + ("effective_project_dir", project_dir.clone()), + ("selected_target", py.None().into_bound(py)), + ("cli_vars", request.getattr("cli_vars")?), + ], + )?; + let selection: Selection<'_> = select_resources(request, &discovered, &context)?; + let progress: Progress<'_> = progress_for(request)?; + let evidence: Bound<'_, PyAny> = compare(request, &context, &selection, &progress)?; + let mut result: Bound<'_, PyAny> = call_with( + &imported(py, ADOPTION_MODELS, "ContractAdoptionResult")?, + &[("from_target", from_target), ("evidence", evidence)], + )?; + if generate_write { + result = write_updates( + request, + (&project_dir, &context), + (&selection.graph, result), + &progress, + )?; + } + let report: ContractReport = report(&result)?; + let rendered: String = if request.getattr("json_output")?.is_truthy()? { + render_contract_json(&report) + .map_err(|error| PyValueError::new_err(format!("{error:?}")))? + } else { + render_contract_text(&report) + }; + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (PyString::new(py, &rendered),))?; + Ok(contract_exit_code(report.difference_count())) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_contract_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs new file mode 100644 index 0000000000..2ef3aae6d5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod command; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/registration.rs new file mode 100644 index 0000000000..64ff1bcbf6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb contract` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::contract_command::command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs new file mode 100644 index 0000000000..924dcecfe5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs @@ -0,0 +1,142 @@ +//! `--since` and `--until` as UTC datetimes, parsed with Python's ISO rules. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyInt}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cost_command::constants::ISO_DATE_LENGTH; +use sqlbuild_core::cost_command::errors::CostCommandError; +use sqlbuild_core::cost_command::main::is_zero_cost_duration::is_zero_cost_duration; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::cost_command::refusal::cost_error; + +const DATETIME_MODULE: &str = "datetime"; +const DURATION_MODELS: &str = "sqlbuild.cursor_algebra.models"; +const DURATION_CONSTANTS: &str = "sqlbuild.cursor_algebra.constants"; + +fn utc(py: Python<'_>) -> PyResult> { + imported(py, DATETIME_MODULE, "UTC") +} + +fn timedelta<'py>( + py: Python<'py>, + unit: &str, + amount: Bound<'py, PyAny>, +) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item(unit, amount)?; + imported(py, DATETIME_MODULE, "timedelta")?.call((), Some(&options)) +} + +fn now(py: Python<'_>) -> PyResult> { + imported(py, DATETIME_MODULE, "datetime")?.call_method1("now", (utc(py)?,)) +} + +fn invalid(py: Python<'_>, label: &str, error: PyErr) -> PyErr { + let wrapped: PyErr = cost_error( + py, + &CostCommandError::new( + "C707", + format!("cost {label} must be an ISO date or datetime"), + ), + ); + wrapped.set_cause(py, Some(error)); + wrapped +} + +/// An ISO date (its UTC start, or the next day's for an end bound) or a zoned datetime. +pub(crate) fn parse_bound<'py>( + py: Python<'py>, + value: &str, + label: &str, + end_of_day: bool, +) -> PyResult> { + let parsed: PyResult> = (|| { + if value.chars().count() == ISO_DATE_LENGTH { + let day: Bound<'py, PyAny> = + imported(py, DATETIME_MODULE, "date")?.call_method1("fromisoformat", (value,))?; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("tzinfo", utc(py)?)?; + let start: Bound<'py, PyAny> = imported(py, DATETIME_MODULE, "datetime")?.call_method( + "combine", + (day, imported(py, DATETIME_MODULE, "time")?.getattr("min")?), + Some(&options), + )?; + if end_of_day { + return start.add(timedelta(py, "days", PyInt::new(py, 1).into_any())?); + } + return Ok(start); + } + imported(py, DATETIME_MODULE, "datetime")? + .call_method1("fromisoformat", (value.replace('Z', "+00:00"),)) + })(); + let parsed: Bound<'py, PyAny> = match parsed { + Ok(parsed) => parsed, + Err(error) if error.is_instance_of::(py) => { + return Err(invalid(py, label, error)); + } + Err(error) => return Err(error), + }; + if value.chars().count() == ISO_DATE_LENGTH { + return Ok(parsed); + } + if parsed.getattr("tzinfo")?.is_none() { + return Err(cost_error( + py, + &CostCommandError::new( + "C708", + format!("cost {label} datetime must include a timezone"), + ), + )); + } + parsed.call_method1("astimezone", (utc(py)?,)) +} + +/// `--since`: now for a zero window, a relative day/hour/minute window, or an ISO bound. +pub(crate) fn parse_since<'py>(py: Python<'py>, value: &str) -> PyResult> { + if is_zero_cost_duration(value) { + return now(py); + } + let duration: Bound<'py, PyAny> = + imported(py, DURATION_MODELS, "Duration")?.call_method1("parse", (value,))?; + if duration.is_none() { + return parse_bound(py, value, "--since", false); + } + let units: Bound<'py, PyAny> = + imported(py, DURATION_CONSTANTS, "MINUTE_TO_DAY_DURATION_UNITS")?; + if !duration + .call_method1("is_single_unit_in", (units,))? + .is_truthy()? + { + return Err(cost_error( + py, + &CostCommandError::new( + "C707", + "cost --since relative duration must use one of: d, h, m", + ), + )); + } + now(py)?.sub(timedelta( + py, + "seconds", + duration.getattr("fixed_seconds")?, + )?) +} + +/// Parse a `cost --since` value into a UTC datetime. +#[pyfunction] +#[pyo3(signature = (*, value))] +fn parse_cost_since_bound<'py>( + py: Python<'py>, + value: Option<&str>, +) -> PyResult>> { + value.map(|value| parse_since(py, value)).transpose() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(parse_cost_since_bound, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs new file mode 100644 index 0000000000..5c8a4f55a3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs @@ -0,0 +1,318 @@ +//! `sqb cost`: persisted run-cost history or one run's breakdown. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cost_command::constants::{ + DEFAULT_DETAIL_SORT, DEFAULT_HISTORY_SORT, DESCENDING_ORDER, HISTORY_SELECTOR, ISO_DATE_LENGTH, + LATEST_SELECTOR, +}; +use sqlbuild_core::cost_command::errors::CostCommandError; +use sqlbuild_core::cost_command::main::cost_history_limit::cost_history_limit; +use sqlbuild_core::cost_command::main::select_cost_run::select_cost_run; +use sqlbuild_core::cost_command::main::validate_cost_request::validate_cost_request; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::cost_command::bounds::{parse_bound, parse_since}; +use crate::bindings::_helpers::cost_command::refresh::refresh_pending_cost_run; +use crate::bindings::_helpers::cost_command::refusal::cost_error; +use crate::bindings::_helpers::cost_command::sorting::{sort_history, sort_resources}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; + +/// The request fields every cost view reads. +struct CostRequest<'py> { + request: Bound<'py, PyAny>, + project_dir: Bound<'py, PyAny>, + selector: String, + limit: Option, + no_limit: bool, + descending: bool, + since: Option, + until: Option, +} + +impl<'py> CostRequest<'py> { + fn read(request: &Bound<'py, PyAny>) -> PyResult { + let py: Python<'py> = request.py(); + let requested: Bound<'py, PyAny> = request.getattr("project_dir")?; + let order: Option = request.getattr("order")?.extract()?; + Ok(Self { + request: request.clone(), + project_dir: if requested.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + requested + }, + selector: request.getattr("selector")?.extract()?, + limit: request.getattr("limit")?.extract()?, + no_limit: request.getattr("no_limit")?.is_truthy()?, + descending: order + .filter(|order| !order.is_empty()) + .is_none_or(|order| order == DESCENDING_ORDER), + since: request.getattr("since")?.extract()?, + until: request.getattr("until")?.extract()?, + }) + } + + fn sort_field(&self, default: &str) -> PyResult { + let sort: Option = self.request.getattr("sort")?.extract()?; + Ok(sort + .filter(|sort| !sort.is_empty()) + .unwrap_or_else(|| default.to_owned())) + } + + fn use_color(&self) -> PyResult { + if self.request.getattr("no_color")?.is_truthy()? { + return Ok(false); + } + imported(self.request.py(), COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy() + } + + /// Write the JSON file when asked, then the JSON or the rendered text to stdout. + fn emit(&self, payload: &str, text: impl FnOnce() -> PyResult) -> PyResult<()> { + let py: Python<'py> = self.request.py(); + let path: Bound<'py, PyAny> = self.request.getattr("json_output_path")?; + if !path.is_none() { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("parents", true)?; + options.set_item("exist_ok", true)?; + path.getattr("parent")? + .call_method("mkdir", (), Some(&options))?; + let encoding: Bound<'py, PyDict> = PyDict::new(py); + encoding.set_item("encoding", "utf-8")?; + path.call_method("write_text", (format!("{payload}\n"),), Some(&encoding))?; + } + let rendered: String = if self.request.getattr("json_output")?.is_truthy()? { + payload.to_owned() + } else { + text()? + }; + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (PyString::new(py, &format!("{rendered}\n")),))?; + Ok(()) + } +} + +fn refused(py: Python<'_>, result: Result<(), CostCommandError>) -> PyResult<()> { + result.map_err(|error| cost_error(py, &error)) +} + +fn cost_runs<'py>(request: &CostRequest<'py>) -> PyResult>> { + let py: Python<'py> = request.request.py(); + call_with( + &imported(py, NATIVE_MODULE, "list_cost_runs")?, + &[("project_dir", request.project_dir.clone())], + )? + .try_iter()? + .collect() +} + +/// Whether the record completed inside the `--since`/`--until` window. +fn in_window( + record: &Bound<'_, PyAny>, + since: Option<&Bound<'_, PyAny>>, + (until, until_is_date): (Option<&Bound<'_, PyAny>>, bool), +) -> PyResult { + let completed: Bound<'_, PyAny> = record.getattr("completed_at")?; + if let Some(since) = since + && !completed.ge(since)? + { + return Ok(false); + } + match until { + Some(until) if until_is_date => completed.lt(until), + Some(until) => completed.le(until), + None => Ok(true), + } +} + +fn run_history(request: &CostRequest<'_>) -> PyResult { + let py: Python<'_> = request.request.py(); + let sort_field: String = request.sort_field(DEFAULT_HISTORY_SORT)?; + refused( + py, + validate_cost_request( + &request.selector, + (request.limit, request.no_limit), + &sort_field, + request.since.is_some() || request.until.is_some(), + ), + )?; + let since: Option> = request + .since + .as_deref() + .map(|value| parse_since(py, value)) + .transpose()?; + let until: Option> = request + .until + .as_deref() + .map(|value| parse_bound(py, value, "--until", true)) + .transpose()?; + if let (Some(since), Some(until)) = (&since, &until) + && since.gt(until)? + { + return Err(cost_error( + py, + &CostCommandError::new("C711", "cost --since must not be after --until"), + )); + } + let until_is_date: bool = request + .until + .as_deref() + .is_some_and(|value| value.chars().count() == ISO_DATE_LENGTH); + let mut matching: Vec> = Vec::new(); + for record in cost_runs(request)? { + if in_window(&record, since.as_ref(), (until.as_ref(), until_is_date))? { + matching.push(record); + } + } + let matching_count: usize = matching.len(); + let mut records: Vec> = + sort_history(py, matching, &sort_field, request.descending)?; + if let Some(limit) = cost_history_limit(request.limit, request.no_limit) { + records.truncate(usize::try_from(limit).unwrap_or(0)); + } + let records: Bound<'_, PyAny> = PyTuple::new(py, records)?.into_any(); + let count: Bound<'_, PyAny> = PyInt::new(py, matching_count).into_any(); + let payload: String = call_with( + &imported(py, NATIVE_MODULE, "format_cost_history_json")?, + &[ + ("records", records.clone()), + ("matching_count", count.clone()), + ], + )? + .extract()?; + request.emit(&payload, || { + call_with( + &imported(py, NATIVE_MODULE, "format_cost_history")?, + &[ + ("records", records), + ("matching_count", count), + ( + "use_color", + PyBool::new(py, request.use_color()?).to_owned().into_any(), + ), + ], + )? + .extract() + })?; + Ok(0) +} + +/// The persisted run the selector names. +fn resolve_run<'py>(request: &CostRequest<'py>) -> PyResult> { + let py: Python<'py> = request.request.py(); + let records: Vec> = cost_runs(request)?; + if request.selector == LATEST_SELECTOR { + let latest: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_cost_run")?, + &[ + ("project_dir", request.project_dir.clone()), + ("selector", PyString::new(py, &request.selector).into_any()), + ], + )?; + if latest.is_none() { + return Err(cost_error( + py, + &CostCommandError::new("C704", "No persisted SQLBuild cost runs were found"), + )); + } + return Ok(latest); + } + let run_ids: Vec = records + .iter() + .map(|record| record.getattr("run_id")?.extract()) + .collect::>()?; + let index: usize = + select_cost_run(&run_ids, &request.selector).map_err(|error| cost_error(py, &error))?; + Ok(records[index].clone()) +} + +/// `dataclasses.replace(record, cost=replace(record.cost, resources=resources))`. +fn with_resources<'py>( + record: &Bound<'py, PyAny>, + resources: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = record.py(); + let replace: Bound<'py, PyAny> = imported(py, "dataclasses", "replace")?; + let cost_changes: Bound<'py, PyDict> = PyDict::new(py); + cost_changes.set_item("resources", PyTuple::new(py, resources)?)?; + let cost: Bound<'py, PyAny> = replace.call((record.getattr("cost")?,), Some(&cost_changes))?; + let changes: Bound<'py, PyDict> = PyDict::new(py); + changes.set_item("cost", cost)?; + replace.call((record,), Some(&changes)) +} + +fn run_detail(request: &CostRequest<'_>) -> PyResult { + let py: Python<'_> = request.request.py(); + let sort_field: String = request.sort_field(DEFAULT_DETAIL_SORT)?; + refused( + py, + validate_cost_request( + &request.selector, + (request.limit, request.no_limit), + &sort_field, + request.since.is_some() || request.until.is_some(), + ), + )?; + let record: Bound<'_, PyAny> = resolve_run(request)?; + let resources: Vec> = record + .getattr("cost")? + .getattr("resources")? + .try_iter()? + .collect::>()?; + let resources: Vec> = + sort_resources(py, resources, &sort_field, request.descending)?; + let record: Bound<'_, PyAny> = with_resources(&record, &resources)?; + let limit: Option = if request.no_limit { + None + } else { + request.limit + }; + let json_record: Bound<'_, PyAny> = match limit { + Some(limit) => { + let kept: usize = usize::try_from(limit).unwrap_or(0).min(resources.len()); + with_resources(&record, &resources[..kept])? + } + None => record.clone(), + }; + let payload: String = imported(py, NATIVE_MODULE, "format_cost_json")? + .call1((json_record,))? + .extract()?; + request.emit(&payload, || { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("record", &record)?; + options.set_item("use_color", request.use_color()?)?; + options.set_item("limit", limit)?; + imported(py, NATIVE_MODULE, "format_cost_breakdown")? + .call((), Some(&options))? + .extract() + })?; + Ok(0) +} + +/// Render persisted cost history or one run breakdown, without the delayed refresh. +fn report(request: &Bound<'_, PyAny>) -> PyResult { + let request: CostRequest<'_> = CostRequest::read(request)?; + if request.selector == HISTORY_SELECTOR { + return run_history(&request); + } + run_detail(&request) +} + +/// Refresh a delayed run, then render persisted cost history or one run breakdown. +#[pyfunction] +fn run_cost_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + refresh_pending_cost_run(py, request); + report(request) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_cost_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/mod.rs new file mode 100644 index 0000000000..d5730144e0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod bounds; +pub(crate) mod command; +pub(crate) mod refresh; +pub(crate) mod refusal; +pub(crate) mod registration; +pub(crate) mod sorting; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs new file mode 100644 index 0000000000..f979330cd9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -0,0 +1,194 @@ +//! Best-effort delayed cost refresh of one pending or partial persisted run. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cost_command::constants::{ + HISTORY_SELECTOR, LATEST_SELECTOR, REFRESHABLE_STATUSES, +}; +use sqlbuild_core::cost_command::main::accepts_cost_refresh::accepts_cost_refresh; +use sqlbuild_core::cost_command::models::CostCoverage; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const COST_TYPES: &str = "sqlbuild.cost.types"; + +fn coverage(summary: &Bound<'_, PyAny>) -> PyResult { + Ok(CostCoverage { + status: summary.getattr("status")?.getattr("value")?.extract()?, + expected_statement_count: summary.getattr("expected_statement_count")?.extract()?, + observed_statement_count: summary.getattr("observed_statement_count")?.extract()?, + query_count: summary.getattr("query_count")?.extract()?, + limitations: summary + .getattr("limitations")? + .try_iter()? + .map(|limitation| limitation?.extract()) + .collect::>()?, + }) +} + +/// The persisted record the selector names: latest, an exact run id, or a unique prefix. +fn refresh_record<'py>( + project_dir: &Bound<'py, PyAny>, + selector: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let resolve = || { + call_with( + &imported(py, NATIVE_MODULE, "resolve_cost_run")?, + &[ + ("project_dir", project_dir.clone()), + ("selector", selector.clone()), + ], + ) + }; + if selector.extract::()? == LATEST_SELECTOR { + return resolve(); + } + let exact: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "read_cost_run")?, + &[ + ("project_dir", project_dir.clone()), + ("run_id", selector.clone()), + ], + )?; + if !exact.is_none() { + return Ok(exact); + } + resolve() +} + +/// The target database the collection queries: the target's, else the connection's. +fn target_database<'py>( + target_config: &Bound<'py, PyAny>, + connection_config: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = target_config.py(); + if !target_config.is_none() { + let database: Bound<'py, PyAny> = target_config.getattr("database")?; + if !database.is_none() { + return Ok(database); + } + } + let database: Bound<'py, PyAny> = connection_config.call_method1("get", ("database",))?; + if database.is_none() { + return Ok(py.None().into_bound(py)); + } + Ok(database.str()?.into_any()) +} + +fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { + let selector: Bound<'_, PyAny> = request.getattr("selector")?; + if selector.extract::()? == HISTORY_SELECTOR { + return Ok(()); + } + let requested: Bound<'_, PyAny> = request.getattr("project_dir")?; + let project_dir: Bound<'_, PyAny> = if requested.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + requested + }; + let record: Bound<'_, PyAny> = refresh_record(&project_dir, &selector)?; + if record.is_none() { + return Ok(()); + } + let current: Bound<'_, PyAny> = record.getattr("cost")?; + let status: String = current.getattr("status")?.getattr("value")?.extract()?; + if !REFRESHABLE_STATUSES.contains(&status.as_str()) { + return Ok(()); + } + let target_name: Bound<'_, PyAny> = record.getattr("target_name")?; + let discovered: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let target_config: Bound<'_, PyAny> = call_with( + &imported(py, EFFECTIVE_TARGET_MODULE, "build_effective_target_config")?, + &[ + ("discovered_inputs", discovered.clone()), + ("selected_target", target_name.clone()), + ], + )?; + let adapter_name: Bound<'_, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'_, PyAny> = call_with( + &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name), + ("project_dir", project_dir.clone()), + ], + )?; + if !adapter.is_instance(&imported(py, COST_TYPES, "CostAwareAdapter")?)? { + return Ok(()); + } + let none_capability: Bound<'_, PyAny> = + imported(py, COST_TYPES, "CostCapability")?.getattr("NONE")?; + if adapter + .call_method0("cost_capability")? + .eq(none_capability)? + { + return Ok(()); + } + let connection_config: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", discovered), + ("project_dir", project_dir.clone()), + ("selected_target", target_name), + ], + )?; + let run_id: Bound<'_, PyAny> = record.getattr("run_id")?; + let ledger: Bound<'_, PyAny> = project_dir + .call_method1("__truediv__", ("target",))? + .call_method1("__truediv__", ("executions",))? + .call_method1("__truediv__", (&run_id,))? + .call_method1("__truediv__", ("statements.jsonl",))?; + let summary: Bound<'_, PyAny> = call_with( + &adapter.getattr("collect_run_cost")?, + &[ + ("connection_config", connection_config.clone()), + ( + "target_database", + target_database(&target_config, &connection_config)?, + ), + ("run_id", run_id), + ("started_at", record.getattr("started_at")?), + ("completed_at", record.getattr("completed_at")?), + ("statement_ledger_path", ledger), + ("usd_per_credit", current.getattr("usd_per_credit")?), + ("rate_source", current.getattr("rate_source")?), + ], + )?; + if accepts_cost_refresh(&coverage(¤t)?, &coverage(&summary)?) != Some(true) { + return Ok(()); + } + let changes: Bound<'_, PyDict> = PyDict::new(py); + changes.set_item("cost", summary)?; + let refreshed: Bound<'_, PyAny> = + imported(py, "dataclasses", "replace")?.call((record,), Some(&changes))?; + call_with( + &imported(py, NATIVE_MODULE, "write_cost_run")?, + &[("project_dir", project_dir), ("record", refreshed)], + )?; + Ok(()) +} + +/// Refresh one delayed record without making lookup depend on warehouse availability. +#[pyfunction] +pub(crate) fn refresh_pending_cost_run(py: Python<'_>, request: &Bound<'_, PyAny>) { + let _ = refresh(py, request); +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(refresh_pending_cost_run, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs new file mode 100644 index 0000000000..093784f8c8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs @@ -0,0 +1,22 @@ +//! Cost command errors raised as `CliUserError`. + +use pyo3::prelude::{Bound, PyAnyMethods, PyErr, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_core::cost_command::errors::CostCommandError; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; + +/// The `CliUserError` for a refused cost request. +pub(crate) fn cost_error(py: Python<'_>, error: &CostCommandError) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", error.code)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((error.message.as_str(),), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(failure) => failure, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/registration.rs new file mode 100644 index 0000000000..fb8534ed64 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/registration.rs @@ -0,0 +1,11 @@ +//! Register the native `sqb cost` command, its delayed refresh and `--since` parsing. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::cost_command::{bounds, command, refresh}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + command::register(module)?; + refresh::register(module)?; + bounds::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/sorting.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/sorting.rs new file mode 100644 index 0000000000..3d33b10470 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/sorting.rs @@ -0,0 +1,142 @@ +//! Python's stable `sorted` over cost records and resources, keyed by the requested metric. + +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyTuple, PyTupleMethods}; +use sqlbuild_core::cost_command::constants::{HISTORY_METRIC_SORT_FIELDS, MISSING_METRIC_STATUSES}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +type SortKey = fn(&Bound<'_, PyAny>, &str) -> PyResult>; + +/// `sorted(items, key=lambda item: key(item, field), reverse=reverse)`. +fn sorted_with<'py>( + py: Python<'py>, + items: Vec>, + (key, field): (SortKey, &str), + reverse: bool, +) -> PyResult>> { + let field: String = field.to_owned(); + let function = move |arguments: &Bound<'_, PyTuple>, + _: Option<&Bound<'_, PyDict>>| + -> PyResult> { key(&arguments.get_item(0)?, &field) }; + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("key", PyCFunction::new_closure(py, None, None, function)?)?; + options.set_item("reverse", reverse)?; + imported(py, "builtins", "sorted")? + .call((PyList::new(py, items)?,), Some(&options))? + .try_iter()? + .collect() +} + +fn attribute(item: &Bound<'_, PyAny>, path: &str) -> PyResult> { + let mut value: Bound<'_, PyAny> = item.clone(); + for name in path.split('.') { + value = value.getattr(name)?; + } + Ok(value.unbind()) +} + +fn history_value(record: &Bound<'_, PyAny>, field: &str) -> PyResult> { + let py: Python<'_> = record.py(); + match field { + "completed" => attribute(record, "completed_at"), + "cost" => attribute(record, "cost.estimated_usd"), + "credits" => attribute(record, "cost.estimated_compute_credits"), + "duration" => Ok(record + .getattr("completed_at")? + .sub(record.getattr("started_at")?)? + .unbind()), + "models" => Ok( + PyInt::new(py, record.getattr("cost")?.getattr("resources")?.len()?) + .into_any() + .unbind(), + ), + "status" => attribute(record, "cost.status.value"), + _ => Err(PyKeyError::new_err(field.to_owned())), + } +} + +fn resource_value(resource: &Bound<'_, PyAny>, field: &str) -> PyResult> { + let path: &str = match field { + "cost" => "estimated_usd", + "credits" => "estimated_compute_credits", + "busy" => "attributed_seconds", + "scanned" => "bytes_scanned", + "model" => "resource_name", + "warehouse" => "warehouse_name", + _ => return Err(PyKeyError::new_err(field.to_owned())), + }; + let py: Python<'_> = resource.py(); + Ok(PyTuple::new( + py, + [ + attribute(resource, path)?, + attribute(resource, "resource_name")?, + attribute(resource, "warehouse_name")?, + ], + )? + .into_any() + .unbind()) +} + +fn by_attribute(item: &Bound<'_, PyAny>, path: &str) -> PyResult> { + attribute(item, path) +} + +/// Newest first with run ids as the tie-break, then stably by the field. +fn sort_partition<'py>( + py: Python<'py>, + records: Vec>, + field: &str, + descending: bool, +) -> PyResult>> { + let by_run_id = sorted_with(py, records, (by_attribute, "run_id"), false)?; + let by_completion = sorted_with(py, by_run_id, (by_attribute, "completed_at"), true)?; + sorted_with(py, by_completion, (history_value, field), descending) +} + +/// History records by the field; metric sorts put runs without metrics last, newest first. +pub(crate) fn sort_history<'py>( + py: Python<'py>, + records: Vec>, + field: &str, + descending: bool, +) -> PyResult>> { + if !HISTORY_METRIC_SORT_FIELDS.contains(&field) { + return sort_partition(py, records, field, descending); + } + let mut available: Vec> = Vec::new(); + let mut missing: Vec> = Vec::new(); + for record in records { + let status: String = record + .getattr("cost")? + .getattr("status")? + .getattr("value")? + .extract()?; + if MISSING_METRIC_STATUSES.contains(&status.as_str()) { + missing.push(record); + } else { + available.push(record); + } + } + let mut sorted: Vec> = sort_partition(py, available, field, descending)?; + let missing = sorted_with(py, missing, (by_attribute, "run_id"), false)?; + sorted.extend(sorted_with( + py, + missing, + (by_attribute, "completed_at"), + true, + )?); + Ok(sorted) +} + +/// Breakdown resources by the field, then resource and warehouse name. +pub(crate) fn sort_resources<'py>( + py: Python<'py>, + resources: Vec>, + field: &str, + descending: bool, +) -> PyResult>> { + sorted_with(py, resources, (resource_value, field), descending) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs new file mode 100644 index 0000000000..c0707d7764 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs @@ -0,0 +1,311 @@ +//! `sqb format`: rewrite authored SQL files in place and report the faults that remain. + +use std::path::PathBuf; +use std::time::Instant; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::format_command::main::render_format_status::render_format_status; +use sqlbuild_core::format_command::main::render_lint_json::render_lint_json; +use sqlbuild_core::format_command::main::render_lint_text::render_lint_text; +use sqlbuild_core::format_command::models::LintReport; +use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::format_command::config::{FormatConfig, format_config, lint_config}; +use crate::bindings::_helpers::format_command::files::FormatChange; +use crate::bindings::_helpers::format_command::report::lint_report; +use crate::bindings::_helpers::format_command::run::{FormatOutcome, FormatRequest, run_format}; +use crate::bindings::_helpers::format_command::selection::{LintInputs, resolve_lint_inputs}; +use crate::bindings::_helpers::frontier::encoder::emit_error; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const STATUS_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const SQL_ANALYSIS_SETTING_KEY: &str = "sql_analysis"; +const DEFAULT_DIALECT: &str = "generic"; +const START_MESSAGE: &str = "Formatting SQL START"; +const ERROR_MESSAGE: &str = "Formatting SQL ERROR"; + +/// The command's flags. +struct FormatFlags { + check: bool, + diff: bool, + fixtures_only: bool, + fix: bool, + json_output: bool, + use_color: bool, +} + +/// The adapter, selected files and discovered inputs a run formats with. +struct Resolved<'py> { + dialect: String, + adapter: Option>, + selected: Option>, + discovered: Option>, + has_fixture_candidates: bool, +} + +fn string_tuple(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Vec::new()); + } + value.try_iter()?.map(|item| item?.extract()).collect() +} + +fn write_stdout(py: Python<'_>, text: &str) -> PyResult<()> { + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (text,))?; + Ok(()) +} + +/// Resolve selectors when the run needs the adapter, swallowing failures of the unscoped run. +fn resolve_inputs<'py>( + project_dir: &Bound<'py, PyAny>, + (select, exclude): (&[String], &[String]), + (fix, dialect): (bool, &str), +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let scoped: bool = !select.is_empty() || !exclude.is_empty(); + let mut resolved: Resolved<'py> = Resolved { + dialect: dialect.to_owned(), + adapter: None, + selected: None, + discovered: None, + has_fixture_candidates: false, + }; + if !scoped { + let directory: PathBuf = project_dir.extract()?; + resolved.has_fixture_candidates = !scan_fixture_null_candidates(&directory, None) + .map_err(value_error)? + .paths + .is_empty(); + } + if scoped || resolved.has_fixture_candidates || fix { + match resolve_lint_inputs(project_dir, (select, exclude)) { + Ok(LintInputs { + adapter, + selected, + discovered, + }) => { + resolved.has_fixture_candidates = discovered.getattr("test_files")?.is_truthy()?; + let adapter_dialect: Bound<'py, PyAny> = + adapter.getattr("sql_analysis_dialect_name")?; + resolved.dialect = if adapter_dialect.is_truthy()? { + adapter_dialect.extract()? + } else { + DEFAULT_DIALECT.to_owned() + }; + resolved.adapter = Some(adapter); + resolved.selected = selected; + resolved.discovered = Some(discovered); + } + Err(error) if scoped || fix || !error.is_instance_of::(py) => { + return Err(error); + } + Err(_) => {} + } + } + if fix { + resolved.discovered = Some(call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?); + } + Ok(resolved) +} + +/// The discovered inputs fixture-null fixes run with, when SQL analysis is enabled. +fn fixture_inputs<'py>(resolved: &Resolved<'py>) -> PyResult>> { + let Some(discovered) = &resolved.discovered else { + return Ok(None); + }; + if !resolved.has_fixture_candidates { + return Ok(None); + } + let local: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let settings_owner: Bound<'py, PyAny> = if local + .getattr("setting_overrides")? + .contains(SQL_ANALYSIS_SETTING_KEY)? + { + local + } else { + discovered.getattr("project_config")? + }; + let enabled: bool = settings_owner + .getattr("settings")? + .getattr(SQL_ANALYSIS_SETTING_KEY)? + .is_truthy()?; + Ok(enabled.then(|| discovered.clone())) +} + +/// Print stable unified diffs for the files the formatter would change. +fn render_diff(py: Python<'_>, changes: &[FormatChange<'_>]) -> PyResult<()> { + let unified_diff: Bound<'_, PyAny> = py.import("difflib")?.getattr("unified_diff")?; + for change in changes { + let lines = |text: &str| -> PyResult> { + PyString::new(py, text).call_method1("splitlines", (true,)) + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("fromfile", change.path.str()?)?; + keywords.set_item("tofile", change.path.str()?)?; + let diff: Bound<'_, PyAny> = unified_diff.call( + (lines(&change.before)?, lines(&change.after)?), + Some(&keywords), + )?; + let joined: Bound<'_, PyAny> = PyString::new(py, "").call_method1("join", (diff,))?; + write_stdout(py, &joined.extract::()?)?; + } + Ok(()) +} + +fn status_reporter<'py>(py: Python<'py>, use_color: bool) -> PyResult> { + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("stream", py.import("sys")?.getattr("stderr")?)?; + options.set_item("use_color", use_color)?; + imported(py, STATUS_MODULE, "TransientStatusReporter")?.call((), Some(&options)) +} + +fn format_with_status<'py>( + py: Python<'py>, + request: &FormatRequest<'_, 'py>, + use_color: bool, +) -> PyResult<(FormatOutcome<'py>, Bound<'py, PyAny>, f64)> { + let started: Instant = Instant::now(); + let status: Bound<'py, PyAny> = status_reporter(py, use_color)?; + status.call_method1("start", (START_MESSAGE,))?; + match run_format(request) { + Ok(outcome) => Ok((outcome, status, started.elapsed().as_secs_f64())), + Err(error) => { + status.call_method1("error", (ERROR_MESSAGE,))?; + Err(error) + } + } +} + +fn complete_status(status: &Bound<'_, PyAny>, report: &LintReport, elapsed: f64) -> PyResult<()> { + let (failed, message) = render_format_status(report, elapsed); + if failed { + status.call_method1("error", (message,))?; + } else { + call_with( + &status.getattr("complete")?, + &[("message", PyString::new(status.py(), &message).into_any())], + )?; + } + Ok(()) +} + +fn run_command( + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, + selectors: (&[String], &[String]), + flags: &FormatFlags, +) -> PyResult { + let config: FormatConfig = format_config(py, project_dir)?; + let resolved: Resolved<'_> = resolve_inputs( + project_dir, + selectors, + (flags.fix, &config.settings.dialect), + )?; + let lint: Bound<'_, PyAny> = lint_config(py, &config, &resolved.dialect)?; + let fixture_discovered: Option> = if flags.fix { + resolved.discovered.clone() + } else { + fixture_inputs(&resolved)? + }; + let request: FormatRequest<'_, '_> = FormatRequest { + project_dir, + config: &lint, + adapter: resolved.adapter.as_ref(), + selected: resolved.selected.as_ref(), + discovered: fixture_discovered.as_ref(), + fixtures_only: flags.fixtures_only, + write: !(flags.check || flags.diff), + fix: flags.fix, + }; + let (outcome, status, elapsed) = format_with_status(py, &request, flags.use_color)?; + let root: PathBuf = project_dir.extract()?; + let report: LintReport = lint_report(&outcome, &root)?; + complete_status(&status, &report, elapsed)?; + if flags.diff { + render_diff(py, &outcome.changes)?; + } + let rendered: String = if flags.json_output { + render_lint_json(&report).map_err(emit_error)? + } else { + let heading: &str = if flags.check || flags.diff { + "Would format files:" + } else { + "Formatted files:" + }; + render_lint_text( + &report, + CliStyle { + use_color: flags.use_color, + }, + heading, + ) + }; + write_stdout(py, &rendered)?; + let failed: bool = + report.fault_count() > 0 || (flags.check && !report.formatted_files.is_empty()); + Ok(i32::from(failed)) +} + +/// Run one `sqb format` invocation. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_format_command<'py>(py: Python<'py>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_format_command", + options, + ( + &["project_dir"], + &[ + "select", + "exclude", + "check", + "diff", + "fixtures_only", + "fix", + "json_output", + "no_color", + ], + ), + )?; + let raw_project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; + let project_dir: Bound<'py, PyAny> = if raw_project_dir.is_none() { + imported(py, "pathlib", "Path")?.call_method0("cwd")? + } else { + raw_project_dir + }; + let use_color: bool = !arguments.flag("no_color")? + && imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + let flags: FormatFlags = FormatFlags { + check: arguments.flag("check")?, + diff: arguments.flag("diff")?, + fixtures_only: arguments.flag("fixtures_only")?, + fix: arguments.flag("fix")?, + json_output: arguments.flag("json_output")?, + use_color, + }; + let select: Vec = string_tuple(&arguments.value("select")?)?; + let exclude: Vec = string_tuple(&arguments.value("exclude")?)?; + run_command(py, &project_dir, (&select, &exclude), &flags) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_format_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs new file mode 100644 index 0000000000..6af55e77bd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs @@ -0,0 +1,82 @@ +//! The run's `LintConfig`, resolved from the project's configuration files. + +use std::path::Path; + +use pyo3::IntoPyObject; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyString; +use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; +use sqlbuild_rules::lint_text::models::FormatSettings; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; + +/// Format settings and whether lint suppressions are allowed. +pub(crate) struct FormatConfig { + pub(crate) settings: FormatSettings, + pub(crate) allow_suppressions: bool, +} + +/// Resolve the settings, loading the rules configuration when the project file exists. +pub(crate) fn format_config( + py: Python<'_>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + let directory: std::path::PathBuf = project_dir.extract()?; + let settings: FormatSettings = + resolve_format_settings(Path::new(&directory)).map_err(PyValueError::new_err)?; + let allow_suppressions: bool = if settings.has_project_config { + call_with( + &imported(py, NATIVE_MODULE, "load_rules_config")?, + &[("project_dir", project_dir.call_method0("resolve")?)], + )? + .getattr("allow_exceptions")? + .extract()? + } else { + true + }; + Ok(FormatConfig { + settings, + allow_suppressions, + }) +} + +/// The Python `LintConfig` of `config`, linting as `dialect`. +pub(crate) fn lint_config<'py>( + py: Python<'py>, + config: &FormatConfig, + dialect: &str, +) -> PyResult> { + let settings: &FormatSettings = &config.settings; + call_with( + &imported(py, LINT_MODELS_MODULE, "LintConfig")?, + &[ + ( + "line_width", + settings.line_width.into_pyobject(py)?.into_any(), + ), + ( + "max_description_lines", + settings.max_description_lines.into_pyobject(py)?.into_any(), + ), + ("dialect", PyString::new(py, dialect).into_any()), + ( + "max_literal_length", + settings.max_literal_length.into_pyobject(py)?.into_any(), + ), + ( + "max_ranking_order_by", + settings.max_ranking_order_by.into_pyobject(py)?.into_any(), + ), + ( + "allow_suppressions", + pyo3::types::PyBool::new(py, config.allow_suppressions) + .to_owned() + .into_any(), + ), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs new file mode 100644 index 0000000000..85bc4e2fc9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs @@ -0,0 +1,131 @@ +//! Authored SQL files as `{Path: str}` dictionaries, and writing formatted files back. + +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +const LINE_FEED: &str = "\n"; +const CARRIAGE_RETURN_LINE_FEED: &str = "\r\n"; + +/// One file the formatter would change: its path and its text before and after. +pub(crate) struct FormatChange<'py> { + pub(crate) path: Bound<'py, PyAny>, + pub(crate) before: String, + pub(crate) after: String, +} + +/// The entries of a `{Path: value}` dictionary, sorted as Python sorts paths. +pub(crate) fn sorted_items<'py>( + dictionary: &Bound<'py, PyDict>, +) -> PyResult, Bound<'py, PyAny>)>> { + let mut items: Vec<(PathBuf, Bound<'py, PyAny>, Bound<'py, PyAny>)> = dictionary + .iter() + .map(|(key, value)| Ok((key.extract::()?, key, value))) + .collect::>()?; + items.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(items + .into_iter() + .map(|(_, key, value)| (key, value)) + .collect()) +} + +/// `{path: overrides.get(path, contents) for path, contents in files.items()}`. +pub(crate) fn overlaid<'py>( + files: &Bound<'py, PyDict>, + overrides: &Bound<'py, PyDict>, +) -> PyResult> { + let merged: Bound<'py, PyDict> = PyDict::new(files.py()); + for (path, contents) in files.iter() { + let value: Bound<'py, PyAny> = overrides.get_item(&path)?.unwrap_or(contents); + merged.set_item(path, value)?; + } + Ok(merged) +} + +/// Generated contents in the authored newline convention. +fn with_newline_style(contents: &str, newline: &str) -> String { + let normalized: String = contents.replace(CARRIAGE_RETURN_LINE_FEED, LINE_FEED); + if newline == LINE_FEED { + return normalized; + } + normalized.replace(LINE_FEED, newline) +} + +fn temporary_path(path: &Path) -> PathBuf { + let name: String = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + let nanos: u128 = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos()); + path.with_file_name(format!(".{name}.{}{nanos:x}", std::process::id())) +} + +fn write_replacing(path: &Path, temporary: &Path, contents: &str) -> std::io::Result<()> { + let permissions: fs::Permissions = fs::metadata(path)?.permissions(); + let mut handle: File = OpenOptions::new() + .write(true) + .create_new(true) + .open(temporary)?; + handle.set_permissions(permissions)?; + handle.write_all(contents.as_bytes())?; + handle.sync_all()?; + fs::rename(temporary, path) +} + +/// Replace a text file atomically, keeping its permissions. +fn write_atomically(path: &Path, contents: &str) -> PyResult<()> { + let temporary: PathBuf = temporary_path(path); + let written: std::io::Result<()> = write_replacing(path, &temporary, contents); + let _ = fs::remove_file(&temporary); + written.map_err(|error| PyOSError::new_err(format!("{}: {error}", path.display()))) +} + +/// The changed files in path order, written in place when `write` is set. +pub(crate) fn persist_changes<'py>( + (files, updated): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), + newlines: &Bound<'py, PyDict>, + write: bool, +) -> PyResult>> { + let mut changes: Vec> = Vec::new(); + for (path, contents) in sorted_items(updated)? { + let newline: String = newlines + .get_item(&path)? + .map_or_else(|| Ok(LINE_FEED.to_owned()), |value| value.extract())?; + let authored: String = files + .get_item(&path)? + .map_or_else(|| Ok(String::new()), |value| value.extract())?; + let after: String = with_newline_style(&contents.extract::()?, &newline); + let before: String = with_newline_style(&authored, &newline); + if after == before { + continue; + } + if write { + write_atomically(&path.extract::()?, &after)?; + } + changes.push(FormatChange { + path, + before, + after, + }); + } + Ok(changes) +} + +/// `{Path: contents}` of the collected files, in collection order. +pub(crate) fn file_dictionary<'py>( + py: Python<'py>, + files: Vec<(PathBuf, String)>, +) -> PyResult> { + let dictionary: Bound<'py, PyDict> = PyDict::new(py); + for (path, contents) in files { + dictionary.set_item(path, contents)?; + } + Ok(dictionary) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/mod.rs new file mode 100644 index 0000000000..7404b5ea9d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod command; +pub(crate) mod config; +pub(crate) mod files; +pub(crate) mod registration; +pub(crate) mod report; +pub(crate) mod run; +pub(crate) mod selection; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs new file mode 100644 index 0000000000..c2bf6cb02c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb format` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::format_command::command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/report.rs new file mode 100644 index 0000000000..73d0f2b5b8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/report.rs @@ -0,0 +1,120 @@ +//! A format run's Python violations and Rule fix reports as plain report data. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_core::format_command::models::{LintFinding, LintReport, RuleFixLine}; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; + +use crate::bindings::_helpers::format_command::run::FormatOutcome; + +const FAULT_SEVERITY: &str = "fault"; +const APPLIED_STATUS: &str = "applied"; +const REFUSED_STATUS: &str = "refused"; +const DECLINED_REASON: &str = "Final layout formatting failed; original file retained"; + +fn text(value: &Bound<'_, PyAny>) -> PyResult { + value.str()?.extract() +} + +fn display_path(path: &Path, root: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .into_owned() +} + +fn source_line( + contents: &Bound<'_, PyDict>, + path: &Bound<'_, PyAny>, + line: i64, +) -> PyResult> { + let Some(source) = contents.get_item(path)? else { + return Ok(None); + }; + let source: String = source.extract()?; + if line < 1 { + return Ok(None); + } + let lines: Vec<&str> = python_splitlines(&source); + let index: usize = usize::try_from(line - 1).unwrap_or(usize::MAX); + Ok(lines.get(index).map(|line| (*line).to_owned())) +} + +fn finding( + violation: &Bound<'_, PyAny>, + contents: &Bound<'_, PyDict>, + root: &Path, +) -> PyResult { + let path: Bound<'_, PyAny> = violation.getattr("file_path")?; + let line: i64 = violation.getattr("line")?.extract()?; + let remediation: Bound<'_, PyAny> = violation.getattr("remediation")?; + Ok(LintFinding { + file: text(&path)?, + display_path: display_path(&path.extract::()?, root), + line, + column: violation.getattr("column")?.extract()?, + end_line: violation.getattr("end_line")?.extract()?, + end_column: violation.getattr("end_column")?.extract()?, + code: violation.getattr("code")?.extract()?, + message: violation.getattr("message")?.extract()?, + fault: text(&violation.getattr("severity")?)? == FAULT_SEVERITY, + engine: violation.getattr("engine")?.extract()?, + remediation: if remediation.is_none() { + None + } else { + Some(remediation.extract()?) + }, + source_line: source_line(contents, &path, line)?, + }) +} + +fn rule_fix(report: &Bound<'_, PyAny>, declined: &BTreeSet) -> PyResult { + let path: Bound<'_, PyAny> = report.getattr("file_path")?; + let status: String = report.getattr("status")?.extract()?; + let reason: String = report.getattr("reason")?.extract()?; + let refused: bool = status == APPLIED_STATUS && declined.contains(&path.extract::()?); + Ok(RuleFixLine { + file: text(&path)?, + code: report.getattr("code")?.extract()?, + line: report.getattr("line")?.extract()?, + status: if refused { + REFUSED_STATUS.to_owned() + } else { + status + }, + reason: if refused { + DECLINED_REASON.to_owned() + } else { + reason + }, + }) +} + +/// The report of `outcome`, violations sorted by file then position, paths shown from `root`. +pub(crate) fn lint_report(outcome: &FormatOutcome<'_>, root: &Path) -> PyResult { + let mut findings: Vec = outcome + .violations + .iter() + .map(|violation| finding(violation, &outcome.final_contents, root)) + .collect::>()?; + findings.sort_by(|left, right| { + (&left.file, left.line, left.column).cmp(&(&right.file, right.line, right.column)) + }); + Ok(LintReport { + files_checked: outcome.files_checked, + formatted_files: outcome + .changes + .iter() + .map(|change| text(&change.path)) + .collect::>()?, + rule_fixes: outcome + .rule_fixes + .iter() + .map(|report| rule_fix(report, &outcome.declined)) + .collect::>()?, + findings, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs new file mode 100644 index 0000000000..82ca20dad6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -0,0 +1,329 @@ +//! Format every authored SQL file and collect the violations that remain. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::format_command::files::{ + FormatChange, file_dictionary, overlaid, persist_changes, sorted_items, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const FIXES_MODULE: &str = "sqlbuild.lint._helpers.fixes"; +const FIXTURE_AUTOFIX_MODULE: &str = "sqlbuild.compiler.planner.classes.fixture_null_autofix"; +const DESCRIPTION_MODULE: &str = "sqlbuild.compiler.compile.classes.model_description_resolution"; +const LINT_ERRORS_MODULE: &str = "sqlbuild.lint.exceptions"; +const DESCRIPTION_PRESENT: &str = "description-present"; + +/// What one format run formats and how. +pub(crate) struct FormatRequest<'a, 'py> { + pub(crate) project_dir: &'a Bound<'py, PyAny>, + pub(crate) config: &'a Bound<'py, PyAny>, + pub(crate) adapter: Option<&'a Bound<'py, PyAny>>, + pub(crate) selected: Option<&'a BTreeSet>, + pub(crate) discovered: Option<&'a Bound<'py, PyAny>>, + pub(crate) fixtures_only: bool, + pub(crate) write: bool, + pub(crate) fix: bool, +} + +/// The outcome of one format run, with Python violations and Rule fix reports. +pub(crate) struct FormatOutcome<'py> { + pub(crate) files_checked: usize, + pub(crate) violations: Vec>, + pub(crate) changes: Vec>, + pub(crate) rule_fixes: Vec>, + /// Paths whose formatting was declined, so their applied fixes were not kept. + pub(crate) declined: BTreeSet, + pub(crate) final_contents: Bound<'py, PyDict>, +} + +fn native<'py>(py: Python<'py>, name: &str) -> PyResult> { + imported(py, NATIVE_MODULE, name) +} + +/// Fixture-null rewrites proven safe by the contracts, when fixture inputs are known. +fn fixture_autofix<'py>( + request: &FormatRequest<'_, 'py>, + fixture_inputs: Option<&(Bound<'py, PyAny>, Bound<'py, PyAny>)>, + files: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = files.py(); + let Some((discovered, syntax)) = fixture_inputs else { + return Ok(PyDict::new(py)); + }; + Ok(call_with( + &imported(py, FIXTURE_AUTOFIX_MODULE, "FixtureNullAutofix")?.getattr("apply")?, + &[ + ("files", files.clone().into_any()), + ("project_dir", request.project_dir.clone()), + ("discovered_inputs", discovered.clone()), + ("sql_lexical_syntax", syntax.clone()), + ], + )? + .downcast_into::()?) +} + +/// SQL bodies of `files` formatted natively: changed texts and declined-body faults. +fn format_bodies<'py>( + request: &FormatRequest<'_, 'py>, + files: &Bound<'py, PyDict>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyAny>)> { + let py: Python<'py> = files.py(); + let items: Vec> = sorted_items(files)? + .into_iter() + .map(|(path, contents)| Ok(PyTuple::new(py, [path, contents])?.into_any())) + .collect::>()?; + let result: Bound<'py, PyAny> = native(py, "format_lint_sql_bodies")?.call1(( + PyList::new(py, items)?, + request.project_dir, + request.config, + ))?; + Ok(( + result.get_item(0)?.downcast_into::()?, + result.get_item(1)?, + )) +} + +/// Contents after header, SQL body and fixture fixes, and the faults of declined rewrites. +fn apply_fixes<'py>( + request: &FormatRequest<'_, 'py>, + files: &Bound<'py, PyDict>, + fixture_inputs: Option<&(Bound<'py, PyAny>, Bound<'py, PyAny>)>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyList>)> { + let py: Python<'py> = files.py(); + let updated: Bound<'py, PyDict> = fixture_autofix(request, fixture_inputs, files)?; + let faults: Bound<'py, PyList> = PyList::empty(py); + if request.fixtures_only { + return Ok((updated, faults)); + } + let current: Bound<'py, PyDict> = overlaid(files, &updated)?; + call_with( + &native(py, "prepare_native_header_cache")?, + &[("files", current.clone().into_any())], + )?; + for (path, contents) in sorted_items(¤t)? { + let result: Bound<'py, PyAny> = call_with( + &native(py, "format_native_headers")?, + &[ + ("contents", contents.clone()), + ("file_path", path.clone()), + ("config", request.config.clone()), + ], + )?; + for fault in result.get_item(1)?.try_iter()? { + faults.append(fault?)?; + } + let formatted: Bound<'py, PyAny> = result.get_item(0)?; + if !formatted.eq(&contents)? { + updated.set_item(path, formatted)?; + } + } + let (native_formatted, native_faults) = format_bodies(request, &overlaid(files, &updated)?)?; + for fault in native_faults.try_iter()? { + faults.append(fault?)?; + } + updated.update(native_formatted.as_mapping())?; + let fixture_formatted: Bound<'py, PyDict> = + fixture_autofix(request, fixture_inputs, &overlaid(files, &updated)?)?; + if !fixture_formatted.is_empty() { + let (final_native, final_faults) = format_bodies(request, &fixture_formatted)?; + for fault in final_faults.try_iter()? { + faults.append(fault?)?; + } + updated.update(fixture_formatted.as_mapping())?; + updated.update(final_native.as_mapping())?; + } + let result: Bound<'py, PyAny> = call_with( + &native(py, "reject_unparseable_header_rewrites")?, + &[ + ("updated", updated.into_any()), + ("config", request.config.clone()), + ("faults", faults.into_any()), + ], + )?; + Ok(( + result.get_item(0)?.downcast_into::()?, + result.get_item(1)?.downcast_into::()?, + )) +} + +/// Formatter-owned header diagnostics of the final contents, without configured Rules. +fn lint_final_contents<'py>( + request: &FormatRequest<'_, 'py>, + files: &Bound<'py, PyDict>, + updated: &Bound<'py, PyDict>, +) -> PyResult>> { + let py: Python<'py> = files.py(); + let mut violations: Vec> = Vec::new(); + for (path, contents) in sorted_items(files)? { + let final_contents: Bound<'py, PyAny> = updated.get_item(&path)?.unwrap_or(contents); + let headers: Bound<'py, PyAny> = call_with( + &native(py, "scan_headers")?, + &[("contents", final_contents.clone())], + )?; + let found: Bound<'py, PyAny> = call_with( + &native(py, "lint_native_headers")?, + &[ + ("contents", final_contents), + ("file_path", path), + ("headers", headers), + ("config", request.config.clone()), + ], + )?; + for violation in found.try_iter()? { + violations.push(violation?); + } + } + Ok(violations) +} + +fn code_is(violation: &Bound<'_, PyAny>, code: &str) -> PyResult { + Ok(violation.getattr("code")?.extract::()? == code) +} + +/// Drop `description-present` faults for models described outside their header. +fn drop_resolved_description_faults<'py>( + project_dir: &Bound<'py, PyAny>, + violations: Vec>, +) -> PyResult>> { + let mut has_description_fault: bool = false; + for violation in &violations { + has_description_fault |= code_is(violation, DESCRIPTION_PRESENT)?; + } + if !has_description_fault { + return Ok(violations); + } + let py: Python<'py> = project_dir.py(); + let described: BTreeSet = call_with( + &imported(py, DESCRIPTION_MODULE, "ModelDescriptionResolution")? + .getattr("externally_described_models")?, + &[("project_dir", project_dir.clone())], + )? + .try_iter()? + .map(|path| path?.call_method0("resolve")?.extract()) + .collect::>()?; + let mut kept: Vec> = Vec::new(); + for violation in violations { + let resolved: PathBuf = violation + .getattr("file_path")? + .call_method0("resolve")? + .extract()?; + if !code_is(&violation, DESCRIPTION_PRESENT)? || !described.contains(&resolved) { + kept.push(violation); + } + } + Ok(kept) +} + +/// Rule fixes planned in memory and verified by compilation, when `--fix` is set. +fn plan_rule_fixes<'py>( + request: &FormatRequest<'_, 'py>, + files: &Bound<'py, PyDict>, +) -> PyResult<(Bound<'py, PyDict>, Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = files.py(); + let result: Bound<'py, PyAny> = call_with( + &imported(py, FIXES_MODULE, "plan_rule_fixes")?, + &[ + ("project_dir", request.project_dir.clone()), + ("files", files.clone().into_any()), + ("config", request.config.clone()), + ( + "adapter", + request + .adapter + .map_or_else(|| py.None().into_bound(py), Clone::clone), + ), + ( + "discovered_inputs", + request + .discovered + .map_or_else(|| py.None().into_bound(py), Clone::clone), + ), + ( + "enabled", + PyBool::new(py, request.fix).to_owned().into_any(), + ), + ], + )?; + Ok(( + result.get_item(0)?.downcast_into::()?, + result.get_item(1)?, + result.get_item(2)?, + )) +} + +/// Format the project's authored SQL files and report what remains. +pub(crate) fn run_format<'py>(request: &FormatRequest<'_, 'py>) -> PyResult> { + let py: Python<'py> = request.project_dir.py(); + let fixture_inputs: Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = + match (request.discovered, request.adapter) { + (Some(discovered), Some(adapter)) => { + Some((discovered.clone(), adapter.getattr("sql_lexical_syntax")?)) + } + (Some(_), None) => { + return Err(imported(py, LINT_ERRORS_MODULE, "LintError")? + .call1(("fixture-null formatting requires the project adapter",)) + .map(pyo3::PyErr::from_value)?); + } + _ => None, + }; + let directory: PathBuf = request.project_dir.extract()?; + let collected: Vec<(PathBuf, String)> = collect_lint_files(&directory, request.selected) + .map_err(|error| pyo3::exceptions::PyOSError::new_err(error.to_string()))?; + let files_checked: usize = collected.len(); + let files: Bound<'py, PyDict> = file_dictionary(py, collected)?; + let safe: Bound<'py, PyAny> = call_with( + &native(py, "safe_format_files")?, + &[ + ("files", files.clone().into_any()), + ("config", request.config.clone()), + ], + )?; + let newlines: Bound<'py, PyDict> = safe.get_item(1)?.downcast_into::()?; + let (safe_files, rule_fixes, rule_faults) = + plan_rule_fixes(request, &safe.get_item(0)?.downcast_into::()?)?; + let (fixed, format_faults) = apply_fixes(request, &safe_files, fixture_inputs.as_ref())?; + for fault in rule_faults.try_iter()? { + format_faults.append(fault?)?; + } + let merged: Bound<'py, PyDict> = safe_files.copy()?; + merged.update(fixed.as_mapping())?; + let declined: BTreeSet = format_faults + .iter() + .map(|fault| fault.getattr("file_path")?.extract()) + .collect::>()?; + let updated: Bound<'py, PyDict> = PyDict::new(py); + for (path, contents) in merged.iter() { + if !declined.contains(&path.extract::()?) { + updated.set_item(path, contents)?; + } + } + let changes: Vec> = + persist_changes((&files, &updated), &newlines, request.write)?; + let mut violations: Vec> = if request.fixtures_only { + Vec::new() + } else { + lint_final_contents(request, &files, &updated)? + }; + violations.extend(format_faults.iter()); + let violations: Vec> = + drop_resolved_description_faults(request.project_dir, violations)?; + let final_contents: Bound<'py, PyDict> = overlaid(&files, &updated)?; + let allow_suppressions: Bound<'py, PyAny> = request.config.getattr("allow_suppressions")?; + let violations: Vec> = native(py, "apply_lint_suppressions")? + .call1((violations, &final_contents, allow_suppressions))? + .extract()?; + Ok(FormatOutcome { + files_checked, + violations, + changes, + rule_fixes: rule_fixes.try_iter()?.collect::>()?, + declined, + final_contents, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs new file mode 100644 index 0000000000..8dfbcbbaf9 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs @@ -0,0 +1,329 @@ +//! Resolve `sqb format` selectors to the adapter, discovered inputs and selected files. + +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyFrozenSet, PyString, PyStringMethods, PyTuple}; +use sqlbuild_core::format_command::errors::FormatSelectorError; +use sqlbuild_core::format_command::main::exact_selector_names::exact_selector_names; +use sqlbuild_core::format_command::main::lint_sql_paths::lint_sql_paths; +use sqlbuild_core::format_command::main::partition_lint_selectors::partition_lint_selectors; +use sqlbuild_core::format_command::main::resolve_lint_paths::resolve_lint_paths; +use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const PLANNER_ERRORS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const SELECTED_DISCOVERER_MODULE: &str = + "sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer"; +const UNIT_TEST_SELECTORS_MODULE: &str = + "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const GRAPH_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const SELECTION_CODE: &str = "S007"; + +/// The adapter, the selected files when a selector scopes the run, and the discovered inputs. +pub(crate) struct LintInputs<'py> { + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) selected: Option>, + pub(crate) discovered: Bound<'py, PyAny>, +} + +/// The project, its selectors and the inputs discovered for it. +struct Scope<'a, 'py> { + directory: PathBuf, + discovered: &'a Bound<'py, PyAny>, + adapter: &'a Bound<'py, PyAny>, +} + +fn planner_error(py: Python<'_>, message: &str, code: &str) -> PyErr { + let raised = (|| -> PyResult> { + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("code", code)?; + imported(py, PLANNER_ERRORS_MODULE, "PlannerInputError")?.call((message,), Some(&keywords)) + })(); + match raised { + Ok(instance) => PyErr::from_value(instance), + Err(failure) => failure, + } +} + +fn selector_error(py: Python<'_>, error: &FormatSelectorError) -> PyErr { + planner_error(py, &error.message, error.code) +} + +fn resolved(path: &Path) -> PathBuf { + fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +fn validate(py: Python<'_>, paths: &BTreeSet) -> PyResult<()> { + if paths.is_empty() { + return Err(planner_error( + py, + "lint selection matched no SQL files", + SELECTION_CODE, + )); + } + Ok(()) +} + +fn discovered_adapter<'py>( + project_dir: &Bound<'py, PyAny>, + discovered: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + call_with( + &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name), + ("project_dir", project_dir.clone()), + ], + ) +} + +fn selectors<'py>(py: Python<'py>, values: &[String]) -> PyResult> { + Ok(PyTuple::new(py, values)?.into_any()) +} + +/// Model names and resolved files, by configured name or file stem. +fn model_names_and_paths(scope: &Scope<'_, '_>) -> PyResult> { + let mut pairs: Vec<(String, PathBuf)> = Vec::new(); + for model_file in scope.discovered.getattr("model_files")?.try_iter()? { + let model_file: Bound<'_, PyAny> = model_file?; + let relative_path: Bound<'_, PyAny> = model_file.getattr("relative_path")?; + let configured: Bound<'_, PyAny> = model_file + .getattr("header_values")? + .call_method1("get", ("name",))?; + let name: String = match configured.downcast::() { + Ok(text) if !text.to_str()?.is_empty() => text.to_str()?.to_owned(), + _ => relative_path.getattr("stem")?.extract()?, + }; + let path: PathBuf = scope.directory.join(relative_path.extract::()?); + pairs.push((name, resolved(&path))); + } + Ok(pairs) +} + +/// Resolve unexpanded model-name selectors without compiling the project graph. +fn exact_model_paths( + py: Python<'_>, + scope: &Scope<'_, '_>, + (select, exclude): (&[String], &[String]), +) -> PyResult>> { + let (Some(selected), Some(excluded)) = + (exact_selector_names(select), exact_selector_names(exclude)) + else { + return Ok(None); + }; + let pairs: Vec<(String, PathBuf)> = model_names_and_paths(scope)?; + let available: BTreeSet = pairs.iter().map(|(name, _)| name.clone()).collect(); + if let Some(unknown) = selected + .union(&excluded) + .find(|name| !available.contains(*name)) + { + return Err(planner_error( + py, + &format!("unknown selector name '{unknown}'"), + SELECTION_CODE, + )); + } + let names: &BTreeSet = if selected.is_empty() { + &available + } else { + &selected + }; + Ok(Some( + pairs + .into_iter() + .filter(|(name, _)| names.contains(name) && !excluded.contains(name)) + .map(|(_, path)| path) + .collect(), + )) +} + +/// Resolve model selectors to authored model files. +fn model_paths( + py: Python<'_>, + scope: &Scope<'_, '_>, + (select, exclude): (&[String], &[String]), +) -> PyResult> { + if let Some(paths) = exact_model_paths(py, scope, (select, exclude))? { + return Ok(paths); + } + let graph: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_project_graph")?, + &[ + ("discovered_inputs", scope.discovered.clone()), + ("adapter", scope.adapter.clone()), + ], + )?; + let keys: Bound<'_, PyAny> = call_with( + &imported(py, GRAPH_SELECTION_MODULE, "select_project_graph")?, + &[ + ("graph", graph.clone()), + ("select", selectors(py, select)?), + ("exclude", selectors(py, exclude)?), + ], + )?; + let model_type: Bound<'_, PyAny> = + imported(py, COMPILE_TYPES_MODULE, "CompiledResourceType")?.getattr("MODEL")?; + let mut names: BTreeSet = BTreeSet::new(); + for key in keys.try_iter()? { + let key: Bound<'_, PyAny> = key?; + if key.getattr("resource_type")?.eq(&model_type)? { + names.insert(key.getattr("name")?.extract()?); + } + } + let mut paths: BTreeSet = BTreeSet::new(); + for model in graph.getattr("project")?.getattr("models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + if names.contains(&model.getattr("name")?.extract::()?) { + let relative: PathBuf = model.getattr("relative_path")?.extract()?; + paths.insert(resolved(&scope.directory.join(relative))); + } + } + Ok(paths) +} + +/// Default or path-based format scopes, resolved before graph selector expansion. +fn combined_paths( + py: Python<'_>, + scope: &Scope<'_, '_>, + (select, exclude): (&[String], &[String]), +) -> PyResult>> { + if select + .iter() + .chain(exclude) + .any(|selector| selector.trim().is_empty()) + { + return Ok(None); + } + let (selected_paths, selected_models) = + partition_lint_selectors(select).map_err(|error| selector_error(py, &error))?; + let (excluded_paths, excluded_models) = + partition_lint_selectors(exclude).map_err(|error| selector_error(py, &error))?; + if selected_paths.is_empty() && excluded_paths.is_empty() && !select.is_empty() { + return Ok(None); + } + let directory: &Path = &scope.directory; + let mut selected: BTreeSet = if select.is_empty() { + lint_sql_paths(directory, &[]) + } else if selected_paths.is_empty() { + BTreeSet::new() + } else { + lint_sql_paths(directory, &selected_paths) + }; + if !select.is_empty() && !selected_models.is_empty() { + selected.extend(model_paths(py, scope, (&selected_models, &[]))?); + } + let mut excluded: BTreeSet = if excluded_paths.is_empty() { + BTreeSet::new() + } else { + lint_sql_paths(directory, &excluded_paths) + }; + if !excluded_models.is_empty() { + excluded.extend(model_paths(py, scope, (&excluded_models, &[]))?); + } + Ok(Some(selected.difference(&excluded).cloned().collect())) +} + +/// Inputs for a run scoped by path selectors, discovering only the contracts fixtures need. +fn path_scoped_inputs<'py>( + project_dir: &Bound<'py, PyAny>, + directory: &Path, + paths: BTreeSet, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + validate(py, &paths)?; + let candidates = scan_fixture_null_candidates(directory, Some(&paths)).map_err(value_error)?; + let candidate_paths: Vec = candidates.paths.into_iter().map(PathBuf::from).collect(); + let discovered: Bound<'py, PyAny> = call_with( + &imported( + py, + SELECTED_DISCOVERER_MODULE, + "SelectedContractInputDiscoverer", + )? + .getattr("discover")?, + &[ + ("project_dir", project_dir.clone()), + ( + "selected_test_paths", + PyFrozenSet::new(py, candidate_paths)?.into_any(), + ), + ( + "referenced_model_names", + PyFrozenSet::new(py, candidates.model_names)?.into_any(), + ), + ], + )?; + Ok(LintInputs { + adapter: discovered_adapter(project_dir, &discovered)?, + selected: Some(paths), + discovered, + }) +} + +/// Resolve the adapter and optional file scope through canonical selectors. +pub(crate) fn resolve_lint_inputs<'py>( + project_dir: &Bound<'py, PyAny>, + (select, exclude): (&[String], &[String]), +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let directory: PathBuf = project_dir.extract()?; + if let Some(paths) = resolve_lint_paths(&directory, (select, exclude)) { + return path_scoped_inputs(project_dir, &directory, paths); + } + let disabled: Bound<'py, PyAny> = PyBool::new(py, false).to_owned().into_any(); + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[ + ("project_dir", project_dir.clone()), + ("sql_analysis_enabled_override", disabled.clone()), + ("extract_output_column_locations", disabled), + ], + )?; + call_with( + &imported(py, UNIT_TEST_SELECTORS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", selectors(py, select)?), + ("exclude", selectors(py, exclude)?), + ("discovered_inputs", discovered.clone()), + ], + )?; + let adapter: Bound<'py, PyAny> = discovered_adapter(project_dir, &discovered)?; + if select.is_empty() && exclude.is_empty() { + return Ok(LintInputs { + adapter, + selected: None, + discovered, + }); + } + let scope: Scope<'_, 'py> = Scope { + directory, + discovered: &discovered, + adapter: &adapter, + }; + let paths: BTreeSet = match combined_paths(py, &scope, (select, exclude))? { + Some(paths) => paths, + None => model_paths(py, &scope, (select, exclude))?, + }; + validate(py, &paths)?; + Ok(LintInputs { + adapter, + selected: Some(paths), + discovered, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs new file mode 100644 index 0000000000..464164b09f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/command.rs @@ -0,0 +1,76 @@ +//! `sqb init`: scaffold a blank SQLBuild project and its agent skills. + +use std::path::PathBuf; + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::init_command::constants::PROJECT_FILE; +use sqlbuild_core::init_command::main::init_project_name::init_project_name; +use sqlbuild_core::init_command::main::init_project_toml::init_project_toml; +use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::init_command::scaffold::scaffold_blank_project; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const EXISTS_MESSAGE: &str = "sqlbuild_project.toml already exists in this directory"; + +fn exists_error(py: Python<'_>) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", "C900")?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((EXISTS_MESSAGE,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// Create a minimal SQLBuild project in the project directory. +#[pyfunction] +#[pyo3(signature = (project_dir = None))] +fn run_init_command(py: Python<'_>, project_dir: Option>) -> PyResult { + let base_dir: Bound<'_, PyAny> = match project_dir.filter(|path| !path.is_none()) { + Some(path) => path, + None => imported(py, "pathlib", "Path")?.call_method0("cwd")?, + }; + if base_dir + .call_method1("__truediv__", (PROJECT_FILE,))? + .call_method0("exists")? + .is_truthy()? + { + return Err(exists_error(py)); + } + let project_name: String = init_project_name(&base_dir.getattr("name")?.extract::()?); + let base_path: PathBuf = base_dir.extract()?; + scaffold_blank_project(py, &base_path, &init_project_toml(&project_name))?; + call_with( + &imported(py, NATIVE_MODULE, "update_sqlbuild_skills")?, + &[("project_dir", base_dir)], + )?; + let style: CliStyle = CliStyle { + use_color: imported(py, COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?, + }; + py.import("sys")?.getattr("stdout")?.call_method1( + "write", + (PyString::new( + py, + &render_init_summary(&project_name, style), + ),), + )?; + Ok(0) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_init_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/mod.rs new file mode 100644 index 0000000000..8d7fbd85ca --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod command; +pub(crate) mod registration; +pub(crate) mod scaffold; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/registration.rs new file mode 100644 index 0000000000..57e4cf2b0a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/registration.rs @@ -0,0 +1,9 @@ +//! Register the native `sqb init` command. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::init_command::command; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs new file mode 100644 index 0000000000..4a80dff91d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs @@ -0,0 +1,84 @@ +//! Create the blank project tree, raising the `OSError` Python's `pathlib` would. + +use std::fs::{self, OpenOptions}; +use std::io::{self, ErrorKind}; +use std::path::Path; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use sqlbuild_core::init_command::constants::{GITKEEP_FILE, PROJECT_FILE, SCAFFOLD_DIRECTORIES}; + +/// `OSError(errno, strerror, filename)`, which Python narrows to its errno subclass. +fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { + let raised: PyResult> = match error.raw_os_error() { + Some(errno) => { + let described: String = io::Error::from_raw_os_error(errno).to_string(); + let strerror: &str = described + .strip_suffix(&format!(" (os error {errno})")) + .unwrap_or(&described); + py.import("builtins").and_then(|builtins| { + builtins.getattr("OSError")?.call1(( + errno, + strerror, + path.to_string_lossy().into_owned(), + )) + }) + } + None => py + .import("builtins") + .and_then(|builtins| builtins.getattr("OSError")?.call1((error.to_string(),))), + }; + match raised { + Ok(raised) => PyErr::from_value(raised), + Err(failure) => failure, + } +} + +/// `Path.mkdir(parents=True, exist_ok=True)`. +fn make_directories(path: &Path) -> Result<(), (io::Error, std::path::PathBuf)> { + match fs::create_dir(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => { + let Some(parent) = path.parent().filter(|parent| *parent != path) else { + return Err((error, path.to_path_buf())); + }; + make_directories(parent)?; + match fs::create_dir(path) { + Err(error) if !path.is_dir() => Err((error, path.to_path_buf())), + _ => Ok(()), + } + } + Err(error) if path.is_dir() => { + drop(error); + Ok(()) + } + Err(error) => Err((error, path.to_path_buf())), + } +} + +/// Create the resource directories, the project file, and `.gitkeep` in empty directories. +pub(crate) fn scaffold_blank_project( + py: Python<'_>, + base_dir: &Path, + project_toml: &str, +) -> PyResult<()> { + for directory in SCAFFOLD_DIRECTORIES { + make_directories(&base_dir.join(directory)) + .map_err(|(error, path)| os_error(py, &error, &path))?; + } + let project_file = base_dir.join(PROJECT_FILE); + fs::write(&project_file, project_toml).map_err(|error| os_error(py, &error, &project_file))?; + for directory in SCAFFOLD_DIRECTORIES { + let directory = base_dir.join(directory); + let mut entries = + fs::read_dir(&directory).map_err(|error| os_error(py, &error, &directory))?; + if entries.next().is_none() { + let gitkeep = directory.join(GITKEEP_FILE); + OpenOptions::new() + .create(true) + .append(true) + .open(&gitkeep) + .map_err(|error| os_error(py, &error, &gitkeep))?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index 8ae19635f2..931f832f8e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -28,14 +28,14 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; -const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main.effective_target"; +const EFFECTIVE_TARGET_MODULE: &str = "sqlbuild.compiler.compile.main._effective_target"; const EFFECTIVE_RUNTIME_MODULE: &str = "sqlbuild.compiler.compile.main.effective_runtime"; const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const RELATION_LOOKUP_MODULE: &str = "sqlbuild.adapter.relations.main.relation_lookup"; const CURSOR_OVERRIDES_MODULE: &str = - "sqlbuild.compiler.planner.main.cursor_window.typed_cursor_overrides"; + "sqlbuild.compiler.planner.main.cursor_window._typed_cursor_overrides"; const PLANNER_MODELS: &str = "sqlbuild.compiler.planner.models"; const PROVIDER_SESSION_MODULE: &str = "sqlbuild.provider.main.session"; const PYTHON_NODE_MODELS: &str = "sqlbuild.python_nodes.models"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 1ab5b302a1..d44788258f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,9 +1,11 @@ pub(crate) mod analysis; pub(crate) mod analysis_session; pub(crate) mod attachments; +pub(crate) mod audit_command; pub(crate) mod auditing; pub(crate) mod authored_values; pub(crate) mod boundary; +pub(crate) mod build_command; pub(crate) mod cache; pub(crate) mod check; pub(crate) mod cli; @@ -14,7 +16,9 @@ pub(crate) mod compile_command; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; +pub(crate) mod contract_command; pub(crate) mod contracts; +pub(crate) mod cost_command; pub(crate) mod cost_report; pub(crate) mod cursor; pub(crate) mod dag_artifact; @@ -24,10 +28,12 @@ pub(crate) mod diagnostics; pub(crate) mod diff_report; pub(crate) mod discovery; pub(crate) mod fingerprints; +pub(crate) mod format_command; pub(crate) mod freshness; pub(crate) mod frontier; pub(crate) mod graph; pub(crate) mod hand_built; +pub(crate) mod init_command; pub(crate) mod janitor; pub(crate) mod lineage; pub(crate) mod load; @@ -38,11 +44,13 @@ pub(crate) mod plan_output; pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; +pub(crate) mod query_command; pub(crate) mod refactor; pub(crate) mod references; pub(crate) mod render; pub(crate) mod result_output; pub(crate) mod rules; +pub(crate) mod scenario_command; pub(crate) mod scopes; pub(crate) mod seed_command; pub(crate) mod semantic_checks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs new file mode 100644 index 0000000000..7997128859 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -0,0 +1,112 @@ +//! `sqb dag`: build the project graph and print its JSON or a one-line summary. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::query_command::main::render_dag_summary::render_dag_summary; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const PYTHON_GRAPH_MODULE: &str = "sqlbuild.compiler.python_nodes.main._graph"; + +fn array_length(document: &serde_json::Value, key: &str) -> PyResult { + document + .get(key) + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .ok_or_else(|| PyValueError::new_err(format!("DAG JSON has no '{key}' list"))) +} + +/// Run one `sqb dag` invocation. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_dag_command<'py>(py: Python<'py>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_dag_command", + options, + ( + &["project_dir"], + &["no_sql_validation", "json_output", "cli_vars"], + ), + )?; + let project_dir: Option> = Some(arguments.value("project_dir")?); + let no_sql_validation: bool = arguments.flag("no_sql_validation")?; + let json_output: bool = arguments.flag("json_output")?; + let cli_vars: Bound<'py, PyAny> = arguments.value("cli_vars")?; + let ProjectAdapter { + project_dir, + discovered, + adapter, + } = project_adapter(py, project_dir)?; + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", project_dir), + ("discovered_inputs", discovered.clone()), + ], + )?; + let graph: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "build_project_graph")?, + &[ + ("discovered_inputs", discovered.clone()), + ("adapter", adapter), + ( + "no_sql_validation", + PyBool::new(py, no_sql_validation).to_owned().into_any(), + ), + ("cli_vars", cli_vars), + ("external_sql_reference_resolver", resolver), + ], + )?; + let python_graph: Bound<'py, PyAny> = call_with( + &imported( + py, + PYTHON_GRAPH_MODULE, + "build_discovered_python_node_graph", + )?, + &[("discovered_inputs", discovered.clone())], + )?; + let dag_json: String = call_with( + &imported(py, NATIVE_MODULE, "build_dag_json")?, + &[ + ("graph", graph), + ( + "project_name", + discovered.getattr("project_config")?.getattr("name")?, + ), + ("python_graph", python_graph), + ], + )? + .extract()?; + let rendered: String = if json_output { + format!("{dag_json}\n") + } else { + let document: serde_json::Value = serde_json::from_str(&dag_json) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + render_dag_summary(( + array_length(&document, "nodes")?, + array_length(&document, "edges")?, + array_length(&document, "checks")?, + )) + }; + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (rendered,))?; + Ok(0) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_dag_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs new file mode 100644 index 0000000000..7b60b819a0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod dag; +pub(crate) mod project; +pub(crate) mod query; +pub(crate) mod query_values; +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs new file mode 100644 index 0000000000..5341d58eed --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -0,0 +1,49 @@ +//! The project directory, its discovered inputs and its adapter. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const RESOLVE_ADAPTER_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; + +/// The project directory, defaulting to the working directory, with its inputs and adapter. +pub(crate) struct ProjectAdapter<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) discovered: Bound<'py, PyAny>, + pub(crate) adapter: Bound<'py, PyAny>, +} + +pub(crate) fn project_adapter<'py>( + py: Python<'py>, + project_dir: Option>, +) -> PyResult> { + let project_dir: Bound<'py, PyAny> = match project_dir.filter(|path| !path.is_none()) { + Some(path) => path, + None => imported(py, "pathlib", "Path")?.call_method0("cwd")?, + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let adapter_name: Bound<'py, PyAny> = call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ("project_config", discovered.getattr("project_config")?), + ("local_config", discovered.getattr("local_config")?), + ], + )?; + let adapter: Bound<'py, PyAny> = call_with( + &imported(py, RESOLVE_ADAPTER_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", adapter_name), + ("project_dir", project_dir.clone()), + ], + )?; + Ok(ProjectAdapter { + project_dir, + discovered, + adapter, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs new file mode 100644 index 0000000000..32ff9db1b3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -0,0 +1,345 @@ +//! `sqb query`: run ad hoc SQL on the project connection and print the rows. + +use pyo3::exceptions::PyException; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyInt, PyString, PyStringMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_core::query_command::main::render_query_csv::render_query_csv; +use sqlbuild_core::query_command::main::render_query_json::render_query_json; +use sqlbuild_core::query_command::main::render_query_long::render_query_long; +use sqlbuild_core::query_command::main::render_query_table::render_query_table; +use sqlbuild_core::query_command::models::QueryDisplay; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; +use crate::bindings::_helpers::query_command::query_values::{ + csv_text, display_text, json_cell, python_json_rows, strict_width, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const ADAPTER_ERRORS_MODULE: &str = "sqlbuild.adapter.contract.exceptions"; +const LONG_FORMAT: &str = "long"; +const TABLE_FORMAT: &str = "table"; +const JSON_FORMAT: &str = "json"; +const CSV_FORMAT: &str = "csv"; +const STATEMENT_OK: &str = "OK\n"; +const DEFAULT_LIMIT: i64 = 20; + +/// A `CliUserError` with its code and optional help. +fn user_error(py: Python<'_>, message: &str, code: Option<&str>, help: Option<&str>) -> PyErr { + let raised = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + if let Some(code) = code { + options.set_item("code", code)?; + } + if let Some(help) = help { + options.set_item("help", help)?; + } + imported(py, ERRORS_MODULE, "CliUserError")?.call((message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +fn empty_sql(py: Python<'_>) -> PyErr { + user_error(py, "query SQL must not be empty", Some("C103"), None) +} + +fn blank(text: &Bound<'_, PyAny>) -> PyResult { + Ok(!text.call_method0("strip")?.is_truthy()?) +} + +/// The SQL text of the file, refusing a missing path, a directory or non-UTF-8 text. +fn read_query_file<'py>(query_file: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = query_file.py(); + let shown: String = query_file.str()?.to_string_lossy().into_owned(); + if !query_file.call_method0("exists")?.is_truthy()? { + return Err(user_error( + py, + &format!("query file does not exist: {shown}"), + Some("C105"), + None, + )); + } + if !query_file.call_method0("is_file")?.is_truthy()? { + return Err(user_error( + py, + &format!("query file path is not a file: {shown}"), + Some("C106"), + None, + )); + } + let options: Bound<'py, PyDict> = PyDict::new(py); + options.set_item("encoding", "utf-8")?; + query_file + .call_method("read_text", (), Some(&options)) + .map_err(|error| { + let builtins = py.import("builtins"); + let readable: bool = builtins.is_ok_and(|builtins| { + ["OSError", "UnicodeError"].iter().any(|name| { + builtins + .getattr(*name) + .is_ok_and(|class| error.value(py).is_instance(&class).unwrap_or(false)) + }) + }); + if !readable { + return error; + } + let wrapped: PyErr = user_error( + py, + &format!("query file could not be read as UTF-8: {shown}"), + Some("C107"), + None, + ); + wrapped.set_cause(py, Some(error)); + wrapped + }) +} + +/// The SQL to run: the positional SQL or the file's text, never both and never blank. +fn query_sql<'py>( + py: Python<'py>, + sql: Option>, + query_file: Option>, +) -> PyResult> { + let sql: Option> = sql.filter(|value| !value.is_none()); + let query_file: Option> = query_file.filter(|value| !value.is_none()); + let text: Bound<'py, PyAny> = match (sql, query_file) { + (Some(_), Some(_)) => { + return Err(user_error( + py, + "query accepts either positional SQL or --file, not both", + Some("C104"), + None, + )); + } + (None, Some(path)) => read_query_file(&path)?, + (Some(sql), None) => sql, + (None, None) => { + return Err(user_error( + py, + "query requires SQL", + Some("C102"), + Some("pass SQL as the query argument or use --file PATH"), + )); + } + }; + if blank(&text)? { + return Err(empty_sql(py)); + } + Ok(text) +} + +/// Connect, run the SQL and close; warehouse failures become `QueryExecutionError`. +fn run_sql<'py>( + project: &ProjectAdapter<'py>, + selected_target: Option>, + sql: Bound<'py, PyAny>, + limit: Option, +) -> PyResult> { + let py: Python<'py> = sql.py(); + let connection_config: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", project.discovered.clone()), + ("project_dir", project.project_dir.clone()), + ( + "selected_target", + selected_target.unwrap_or_else(|| py.None().into_bound(py)), + ), + ], + )?; + let connection: Bound<'py, PyAny> = project + .adapter + .call_method1("connect", (connection_config,))?; + let limit_value: Bound<'py, PyAny> = match limit { + Some(limit) => PyInt::new(py, limit).into_any(), + None => py.None().into_bound(py), + }; + let queried: PyResult> = call_with( + &project.adapter.getattr("query")?, + &[ + ("connection", connection.clone()), + ("sql", sql), + ("limit", limit_value), + ], + ); + let queried: PyResult> = queried.map_err(|error| { + let adapter_error: bool = imported(py, ADAPTER_ERRORS_MODULE, "AdapterUserError") + .is_ok_and(|class| error.value(py).is_instance(&class).unwrap_or(false)); + if adapter_error || !error.is_instance_of::(py) { + return error; + } + let value: Bound<'_, PyAny> = error.value(py).clone().into_any(); + let message: PyResult> = match value.getattr("message") { + Ok(message) => message.str().map(Bound::into_any), + Err(_) => value.str().map(Bound::into_any), + }; + let wrapped: PyErr = match message.and_then(|message| { + imported(py, ERRORS_MODULE, "QueryExecutionError")?.call1((message,)) + }) { + Ok(raised) => PyErr::from_value(raised), + Err(failure) => failure, + }; + wrapped.set_cause(py, Some(error)); + wrapped + }); + let closed: PyResult> = project.adapter.call_method1("close", (connection,)); + let result: Bound<'py, PyAny> = queried?; + closed?; + Ok(result) +} + +fn rows<'py>(result: &Bound<'py, PyAny>) -> PyResult>>> { + result + .getattr("rows")? + .try_iter()? + .map(|row| row?.try_iter()?.collect::>>()) + .collect() +} + +fn texts( + rows: &[Vec>], + convert: fn(&Bound<'_, PyAny>) -> PyResult, +) -> PyResult>> { + rows.iter() + .map(|row| row.iter().map(convert).collect()) + .collect() +} + +fn render_json( + result: &Bound<'_, PyAny>, + columns: &[String], + rows: &[Vec>], +) -> PyResult { + let mut values: Vec> = Vec::new(); + for row in rows { + strict_width(columns.len(), row.len())?; + let mut cells: Vec = Vec::new(); + for value in row { + match json_cell(value)? { + Some(cell) => cells.push(cell), + None => return python_json_rows(result.py(), &result.getattr("columns")?, rows), + } + } + values.push(cells); + } + match render_query_json(columns, values) { + Ok(rendered) => Ok(rendered), + Err(_) => python_json_rows(result.py(), &result.getattr("columns")?, rows), + } +} + +/// The result in the requested format; a statement without columns prints `OK`. +fn render(result: &Bound<'_, PyAny>, output_format: &str, limit: Option) -> PyResult { + let py: Python<'_> = result.py(); + let columns: Vec = result + .getattr("columns")? + .try_iter()? + .map(|column| column?.extract()) + .collect::>()?; + if columns.is_empty() { + return Ok(STATEMENT_OK.to_owned()); + } + let rows: Vec>> = rows(result)?; + let display = |strict: bool| -> PyResult { + if strict { + for row in &rows { + strict_width(columns.len(), row.len())?; + } + } + Ok(QueryDisplay { + columns: columns.clone(), + rows: texts(&rows, display_text)?, + truncated: result.getattr("truncated")?.is_truthy()?, + limit, + }) + }; + match output_format { + LONG_FORMAT => Ok(render_query_long(&display(true)?)), + TABLE_FORMAT => Ok(render_query_table(&display(false)?)), + JSON_FORMAT => render_json(result, &columns, &rows), + CSV_FORMAT => Ok(render_query_csv(&columns, &texts(&rows, csv_text)?)), + _ => Err(user_error( + py, + &format!("unsupported query output format '{output_format}'"), + None, + None, + )), + } +} + +/// Execute ad hoc SQL against the active project connection. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_query_command(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "run_query_command", + options, + ( + &["project_dir", "sql"], + &["query_file", "selected_target", "output_format", "limit"], + ), + )?; + let output_format: String = if arguments.passed("output_format")? { + arguments.value("output_format")?.extract()? + } else { + LONG_FORMAT.to_owned() + }; + let limit: Option = if arguments.passed("limit")? { + arguments.value("limit")?.extract()? + } else { + Some(DEFAULT_LIMIT) + }; + let sql: Bound<'_, PyAny> = query_sql( + py, + Some(arguments.value("sql")?), + Some(arguments.value("query_file")?), + )?; + if limit.is_some_and(|limit| limit < 0) { + return Err(user_error( + py, + "query --limit must be greater than or equal to 0", + Some("C101"), + Some("pass a non-negative integer or use --no-limit"), + )); + } + let project: ProjectAdapter<'_> = project_adapter(py, Some(arguments.value("project_dir")?))?; + let result: Bound<'_, PyAny> = run_sql( + &project, + Some(arguments.value("selected_target")?), + sql, + limit, + )?; + let rendered: String = render(&result, &output_format, limit)?; + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + stdout.call_method1("write", (PyString::new(py, &rendered),))?; + stdout.call_method0("flush")?; + Ok(0) +} + +/// Render a query result using the requested output format. +#[pyfunction] +#[pyo3(signature = (*, result, output_format, limit))] +fn render_query_result( + result: &Bound<'_, PyAny>, + output_format: &str, + limit: Option, +) -> PyResult { + render(result, output_format, limit) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_query_command, module)?)?; + module.add_function(wrap_pyfunction!(render_query_result, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs new file mode 100644 index 0000000000..5bde968c4c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs @@ -0,0 +1,187 @@ +//! Query cell values as display text, CSV fields and JSON values. + +use pyo3::exceptions::{PyUnicodeEncodeError, PyValueError}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyListMethods, PyString, + PyStringMethods, PyTuple, +}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const NULL_TEXT: &str = "NULL"; +const FIXED_POINT_FORMAT: &str = "f"; + +fn python_text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string_lossy().into_owned()) +} + +fn decimal_instance(value: &Bound<'_, PyAny>) -> PyResult { + value.is_instance(&imported(value.py(), "decimal", "Decimal")?) +} + +/// `format(value, "f")`, Decimal's fixed-point text. +fn fixed_point(value: &Bound<'_, PyAny>) -> PyResult { + imported(value.py(), "builtins", "format")? + .call1((value, FIXED_POINT_FORMAT))? + .extract() +} + +/// `NULL`, a Decimal in fixed-point notation, or the value's `str`. +pub(crate) fn display_text(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(NULL_TEXT.to_owned()); + } + if decimal_instance(value)? { + return fixed_point(value); + } + python_text(value) +} + +/// The text `csv.writer` writes: empty for `None`, `repr` for floats, else `str`. +pub(crate) fn csv_text(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(String::new()); + } + if value.is_instance_of::() { + return Ok(value.repr()?.to_string_lossy().into_owned()); + } + python_text(value) +} + +/// The JSON `default` hook: Decimal in fixed-point notation, anything else as its `str`. +#[pyfunction] +pub(crate) fn query_json_default(value: &Bound<'_, PyAny>) -> PyResult { + if decimal_instance(value)? { + return fixed_point(value); + } + python_text(value) +} + +/// The text, or `None` for a `str` holding lone surrogates. +fn exact_text(text: &Bound<'_, PyString>) -> PyResult> { + match text.to_str() { + Ok(text) => Ok(Some(text.to_owned())), + Err(error) if error.is_instance_of::(text.py()) => Ok(None), + Err(error) => Err(error), + } +} + +/// The decimal integer, or `None` beyond Python's integer string conversion limit. +fn exact_integer(value: &Bound<'_, PyAny>) -> PyResult> { + match value.str() { + Ok(text) => Ok(JsonInteger::parse(&text.to_string_lossy()).map(JsonValue::Integer)), + Err(error) if error.is_instance_of::(value.py()) => Ok(None), + Err(error) => Err(error), + } +} + +fn object_key(key: &Bound<'_, PyAny>) -> PyResult> { + if key.is_exact_instance_of::() { + return exact_text(key.downcast::()?); + } + if key.is_none() { + return Ok(Some("null".to_owned())); + } + if let Ok(flag) = key.downcast_exact::() { + return Ok(Some(flag.is_true().to_string())); + } + if key.is_exact_instance_of::() { + return Ok(Some(python_text(key)?)); + } + Ok(None) +} + +/// The value as the query JSON encoder writes it, or `None` when only Python's encoder can. +pub(crate) fn json_cell(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Some(JsonValue::Null)); + } + if let Ok(flag) = value.downcast_exact::() { + return Ok(Some(JsonValue::Bool(flag.is_true()))); + } + if value.is_exact_instance_of::() { + return exact_integer(value); + } + if value.is_exact_instance_of::() { + return Ok(Some(JsonValue::Float(value.extract()?))); + } + if value.is_exact_instance_of::() { + return Ok(exact_text(value.downcast::()?)?.map(JsonValue::String)); + } + if value.is_exact_instance_of::() || value.is_exact_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + let Some(item) = json_cell(&item?)? else { + return Ok(None); + }; + items.push(item); + } + return Ok(Some(JsonValue::Array(items))); + } + if let Ok(mapping) = value.downcast_exact::() { + let mut entries: Vec<(String, JsonValue)> = Vec::new(); + for (key, item) in mapping.iter() { + let Some(key) = object_key(&key)? else { + return Ok(None); + }; + if entries.iter().any(|(existing, _)| *existing == key) { + return Ok(None); + } + let Some(item) = json_cell(&item)? else { + return Ok(None); + }; + entries.push((key, item)); + } + return Ok(Some(JsonValue::Object(entries))); + } + if value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + { + return Ok(None); + } + Ok(Some(JsonValue::String(query_json_default(value)?))) +} + +/// `json.dumps([dict(zip(columns, row)) ...], default=query_json_default) + "\n"`. +pub(crate) fn python_json_rows( + py: Python<'_>, + columns: &Bound<'_, PyAny>, + rows: &[Vec>], +) -> PyResult { + let names: Vec> = columns.try_iter()?.collect::>()?; + let objects: Bound<'_, PyList> = PyList::empty(py); + for row in rows { + let object: Bound<'_, PyDict> = PyDict::new(py); + for (name, value) in names.iter().zip(row) { + object.set_item(name, value)?; + } + objects.append(object)?; + } + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("default", wrap_pyfunction!(query_json_default, py)?)?; + let rendered: String = py + .import("json")? + .call_method("dumps", (objects,), Some(&options))? + .extract()?; + Ok(format!("{rendered}\n")) +} + +/// The `ValueError` strict `zip` raises when a row and the columns differ in length. +pub(crate) fn strict_width(columns: usize, row: usize) -> PyResult<()> { + match row.cmp(&columns) { + std::cmp::Ordering::Less => Err(PyValueError::new_err( + "zip() argument 2 is shorter than argument 1", + )), + std::cmp::Ordering::Greater => Err(PyValueError::new_err( + "zip() argument 2 is longer than argument 1", + )), + std::cmp::Ordering::Equal => Ok(()), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/registration.rs new file mode 100644 index 0000000000..e42f61f8aa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb query` and `sqb dag` commands. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::query_command::{dag, query}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + query::register(module)?; + dag::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs new file mode 100644 index 0000000000..921104c611 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs @@ -0,0 +1,200 @@ +//! Native SQL lint of prepared lint bodies, as Python lint violations. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::lint_text::main::lint_sql_bodies::lint_sql_bodies; +use sqlbuild_rules::lint_text::models::{ + AuthoredDiagnostic, ExpansionSpan, LintSqlBody, LintSqlOptions, +}; + +use crate::bindings::_helpers::boundary::panics::value_error; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::types::CompilerDetach; + +const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; +const LINT_TYPES_MODULE: &str = "sqlbuild.lint.types"; +const NATIVE_ENGINE: &str = "native"; + +fn spans(passes: &Bound<'_, PyAny>) -> PyResult>> { + passes.try_iter()?.map(|pass| pass_spans(&pass?)).collect() +} + +fn pass_spans(pass: &Bound<'_, PyAny>) -> PyResult> { + pass.try_iter()? + .map(|span| { + let span: Bound<'_, PyAny> = span?; + Ok(ExpansionSpan { + source_start: span.getattr("source_start")?.extract()?, + source_end: span.getattr("source_end")?.extract()?, + output_start: span.getattr("output_start")?.extract()?, + output_end: span.getattr("output_end")?.extract()?, + }) + }) + .collect() +} + +/// Declared keys of the relations one body reads, by their neutralized sentinel. +fn relation_keys( + body: &Bound<'_, PyAny>, + declared: &Bound<'_, PyAny>, +) -> PyResult>)>> { + let mut keys: Vec<(String, Vec>)> = Vec::new(); + for relation in body.getattr("dependency_relations")?.try_iter()? { + let (sentinel, name): (String, Bound<'_, PyAny>) = relation?.extract()?; + let found: Bound<'_, PyAny> = declared.call_method1("get", (name,))?; + if found.is_truthy()? { + keys.push((sentinel, found.extract()?)); + } + } + Ok(keys) +} + +fn lint_body( + body: &Bound<'_, PyAny>, + index_by_path: &Bound<'_, PyDict>, + declared: &Bound<'_, PyAny>, +) -> PyResult { + let file: usize = index_by_path + .as_any() + .get_item(body.getattr("file_path")?)? + .extract()?; + Ok(LintSqlBody { + file, + body_start: body.getattr("body_start")?.extract()?, + lint_text: body.getattr("lint_text")?.extract()?, + passes: spans(&body.getattr("passes")?)?, + external_identifiers: body.getattr("external_identifiers")?.extract()?, + dependency_identifiers: body.getattr("dependency_identifiers")?.extract()?, + externally_referenced_ctes: body.getattr("externally_referenced_ctes")?.extract()?, + allows_ceremonial_select: body.getattr("allows_ceremonial_select")?.extract()?, + allows_dynamic_output_star: body.getattr("allows_dynamic_output_star")?.extract()?, + allows_empty_fixture_star: body.getattr("allows_empty_fixture_star")?.extract()?, + relation_keys: relation_keys(body, declared)?, + }) +} + +fn options(config: &Bound<'_, PyAny>) -> PyResult { + Ok(LintSqlOptions { + dialect: config.getattr("dialect")?.extract()?, + enabled_rules: config.getattr("enabled_native_rules")?.extract()?, + ignored_rules: config.getattr("ignored_native_rules")?.extract()?, + max_literal_length: config.getattr("max_literal_length")?.extract()?, + max_ranking_order_by: config.getattr("max_ranking_order_by")?.extract()?, + }) +} + +/// The lint model classes native SQL diagnostics are built as. +struct ViolationClasses<'py> { + violation: Bound<'py, PyAny>, + edit: Bound<'py, PyAny>, + severity: Bound<'py, PyAny>, +} + +impl<'py> ViolationClasses<'py> { + fn new(py: Python<'py>) -> PyResult { + Ok(Self { + violation: imported(py, LINT_MODELS_MODULE, "LintViolation")?, + edit: imported(py, LINT_MODELS_MODULE, "LintEdit")?, + severity: imported(py, LINT_TYPES_MODULE, "LintSeverity")?, + }) + } + + fn violation( + &self, + file_path: &Bound<'py, PyAny>, + diagnostic: AuthoredDiagnostic, + ) -> PyResult> { + let py: Python<'py> = file_path.py(); + let none = || py.None().into_bound(py); + let code: Bound<'py, PyAny> = PyString::new(py, &diagnostic.code).into_any(); + let fix: Bound<'py, PyAny> = match diagnostic.fix { + Some(edit) => call_with( + &self.edit, + &[ + ("file_path", file_path.clone()), + ("code", code.clone()), + ("start", edit.start.into_pyobject(py)?.into_any()), + ("end", edit.end.into_pyobject(py)?.into_any()), + ( + "replacement", + PyString::new(py, &edit.replacement).into_any(), + ), + ], + )?, + None => none(), + }; + let (end_line, end_column) = match diagnostic.end { + Some((line, column)) => ( + line.into_pyobject(py)?.into_any(), + column.into_pyobject(py)?.into_any(), + ), + None => (none(), none()), + }; + let severity: &str = if diagnostic.fault { "FAULT" } else { "WARNING" }; + call_with( + &self.violation, + &[ + ("file_path", file_path.clone()), + ("line", diagnostic.line.into_pyobject(py)?.into_any()), + ("column", diagnostic.column.into_pyobject(py)?.into_any()), + ("code", code), + ("message", PyString::new(py, &diagnostic.message).into_any()), + ("severity", self.severity.getattr(severity)?), + ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), + ("end_line", end_line), + ("end_column", end_column), + ( + "remediation", + PyString::new(py, &diagnostic.remediation).into_any(), + ), + ("fix", fix), + ( + "fix_unavailable_reason", + diagnostic + .fix_unavailable_reason + .map_or_else(none, |reason| PyString::new(py, &reason).into_any()), + ), + ], + ) + } +} + +/// Lint expanded bodies natively and map every diagnostic onto authored source. +#[pyfunction] +fn lint_native_sql_bodies<'py>( + py: Python<'py>, + bodies: Vec>, + contents_by_path: Bound<'py, PyDict>, + config: Bound<'py, PyAny>, +) -> PyResult>> { + let index_by_path: Bound<'py, PyDict> = PyDict::new(py); + let mut paths: Vec> = Vec::new(); + let mut texts: Vec = Vec::new(); + for (path, text) in contents_by_path.iter() { + index_by_path.set_item(&path, paths.len())?; + paths.push(path); + texts.push(text.extract()?); + } + let declared: Bound<'py, PyAny> = config.getattr("relation_keys")?; + let inputs: Vec = bodies + .iter() + .map(|body| lint_body(body, &index_by_path, &declared)) + .collect::>()?; + let options: LintSqlOptions = options(&config)?; + let diagnostics: Vec = py + .compiler_detach(|| { + let files: Vec<&str> = texts.iter().map(String::as_str).collect(); + Ok(lint_sql_bodies(&inputs, &files, &options)) + }) + .map_err(value_error)?; + let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; + diagnostics + .into_iter() + .map(|diagnostic| classes.violation(&paths[diagnostic.file], diagnostic)) + .collect() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lint_native_sql_bodies, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs new file mode 100644 index 0000000000..e637d4bf30 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -0,0 +1,331 @@ +//! Interpolation sentinels, suppressions and fixture-null candidates over Python lint models. + +use std::collections::BTreeSet; +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::lint_text::constants::SUPPRESSION_CODE; +use sqlbuild_rules::lint_text::errors::LintTextError; +use sqlbuild_rules::lint_text::main::apply_suppressions::apply_suppressions; +use sqlbuild_rules::lint_text::main::format_sql_bodies::format_sql_bodies; +use sqlbuild_rules::lint_text::main::interpolation_text_at::interpolation_text_at; +use sqlbuild_rules::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; +use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; +use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use sqlbuild_rules::lint_text::main::restore_interpolation::restore_interpolation; +use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; +use sqlbuild_rules::lint_text::models::{ + FormatFile, FormattedBodies, InterpolationSite, Neutralized, RetainedViolation, + SuppressibleViolation, SuppressionFinding, +}; + +use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::types::CompilerDetach; + +const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; +const LINT_TYPES_MODULE: &str = "sqlbuild.lint.types"; +const NATIVE_ENGINE: &str = "native"; +const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; +const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; + +/// One interpolation site as `InterpolationSite` fields. +type SiteTuple = (String, usize, usize, usize, usize, String); + +fn site_tuples(neutralized: Neutralized) -> (String, Vec) { + let sites: Vec = neutralized + .sites + .into_iter() + .map(|site| { + ( + site.sentinel, + site.neutralized_start, + site.neutralized_end, + site.original_start, + site.original_end, + site.original_text, + ) + }) + .collect(); + (neutralized.text, sites) +} + +#[pyfunction] +#[pyo3(signature = (body, dialect, for_formatting = false))] +fn neutralize_lint_interpolation( + body: &str, + dialect: &str, + for_formatting: bool, +) -> PyResult<(String, Vec)> { + compiler_guard(|| { + neutralize_interpolation(body, dialect, for_formatting) + .map(site_tuples) + .map_err(value_error) + }) +} + +#[pyfunction] +fn neutralize_lint_context_interpolation(body: &str) -> PyResult<(String, Vec)> { + compiler_guard(|| Ok(site_tuples(neutralize_context_interpolation(body)))) +} + +#[pyfunction] +fn neutralize_lint_audit_parameters(body: &str) -> PyResult<(String, Vec)> { + compiler_guard(|| Ok(site_tuples(neutralize_audit_parameters(body)))) +} + +/// Restored text, or `(False, reason)` when a sentinel is missing or repeated. +#[pyfunction] +fn restore_lint_interpolation(fixed: &str, sites: Vec) -> PyResult<(bool, String)> { + let sites: Vec = sites + .into_iter() + .map(|site| InterpolationSite { + sentinel: site.0, + neutralized_start: site.1, + neutralized_end: site.2, + original_start: site.3, + original_end: site.4, + original_text: site.5, + }) + .collect(); + match compiler_guard(|| Ok(restore_interpolation(fixed, &sites)))? { + Ok(restored) => Ok((true, restored)), + Err(LintTextError::Restoration(message)) => Ok((false, message)), + Err(error) => Err(value_error(error)), + } +} + +#[pyfunction] +fn lint_interpolation_text_at(body: &str, start: usize, dialect: &str) -> PyResult> { + compiler_guard(|| Ok(interpolation_text_at(body, start, dialect))) +} + +#[pyfunction] +#[pyo3(signature = (project_dir, selected_paths = None))] +fn scan_fixture_typed_null_candidates<'py>( + py: Python<'py>, + project_dir: PathBuf, + selected_paths: Option>, +) -> PyResult<(Bound<'py, PyFrozenSet>, Bound<'py, PyFrozenSet>)> { + let candidates = compiler_guard(|| { + scan_fixture_null_candidates(&project_dir, selected_paths.as_ref()).map_err(value_error) + })?; + let paths: Vec = candidates.paths.into_iter().map(PathBuf::from).collect(); + Ok(( + PyFrozenSet::new(py, paths)?, + PyFrozenSet::new(py, candidates.model_names)?, + )) +} + +/// Remove violations matched by suppressions and add the suppression findings, as lint models. +#[pyfunction] +fn apply_lint_suppressions<'py>( + py: Python<'py>, + violations: Vec>, + contents_by_path: Bound<'py, PyDict>, + allow_suppressions: bool, +) -> PyResult>> { + let index_by_path: Bound<'py, PyDict> = PyDict::new(py); + let mut paths: Vec> = Vec::new(); + let mut contents: Vec = Vec::new(); + for (path, text) in contents_by_path.iter() { + index_by_path.set_item(&path, paths.len())?; + paths.push(path); + contents.push(text.extract()?); + } + let keys: Vec = violations + .iter() + .map(|violation| suppressible(violation, &index_by_path)) + .collect::>()?; + let texts: Vec<&str> = contents.iter().map(String::as_str).collect(); + let retained: Vec = compiler_guard(|| { + apply_suppressions(&texts, &keys, allow_suppressions).map_err(value_error) + })?; + let classes: FindingClasses<'py> = FindingClasses::new(py)?; + retained + .into_iter() + .map(|item| match item { + RetainedViolation::Existing(index) => Ok(violations[index].clone()), + RetainedViolation::Added(finding) => classes.violation(&paths[finding.file], finding), + }) + .collect() +} + +/// The suppression identity of one Python lint violation. +fn suppressible( + violation: &Bound<'_, PyAny>, + index_by_path: &Bound<'_, PyDict>, +) -> PyResult { + let file: Option = match index_by_path.get_item(violation.getattr("file_path")?)? { + Some(index) => Some(index.extract()?), + None => None, + }; + Ok(SuppressibleViolation { + file: file.unwrap_or(usize::MAX), + line: violation.getattr("line")?.extract()?, + code: violation.getattr("code")?.extract()?, + native: violation.getattr("engine")?.extract::()? == NATIVE_ENGINE, + }) +} + +/// The lint model classes suppression findings are built as. +struct FindingClasses<'py> { + violation: Bound<'py, PyAny>, + edit: Bound<'py, PyAny>, + severity: Bound<'py, PyAny>, +} + +impl<'py> FindingClasses<'py> { + fn new(py: Python<'py>) -> PyResult { + Ok(Self { + violation: imported(py, LINT_MODELS_MODULE, "LintViolation")?, + edit: imported(py, LINT_MODELS_MODULE, "LintEdit")?, + severity: imported(py, LINT_TYPES_MODULE, "LintSeverity")?, + }) + } + + fn violation( + &self, + file_path: &Bound<'py, PyAny>, + finding: SuppressionFinding, + ) -> PyResult> { + let py: Python<'py> = file_path.py(); + let code: Bound<'py, PyAny> = PyString::new(py, SUPPRESSION_CODE).into_any(); + let fix: Bound<'py, PyAny> = match finding.fix { + Some((start, end)) => call_with( + &self.edit, + &[ + ("file_path", file_path.clone()), + ("code", code.clone()), + ("start", start.into_pyobject(py)?.into_any()), + ("end", end.into_pyobject(py)?.into_any()), + ("replacement", PyString::new(py, "").into_any()), + ], + )?, + None => py.None().into_bound(py), + }; + let severity: &str = if finding.fault { "FAULT" } else { "WARNING" }; + call_with( + &self.violation, + &[ + ("file_path", file_path.clone()), + ("line", finding.line.into_pyobject(py)?.into_any()), + ("column", 1_usize.into_pyobject(py)?.into_any()), + ("code", code), + ("message", PyString::new(py, &finding.message).into_any()), + ("severity", self.severity.getattr(severity)?), + ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), + ( + "remediation", + PyString::new(py, finding.remediation).into_any(), + ), + ("fix", fix), + ( + "fix_unavailable_reason", + finding.fix_unavailable_reason.map_or_else( + || py.None().into_bound(py), + |reason| PyString::new(py, reason).into_any(), + ), + ), + ], + ) + } +} + +/// The parts of `path` below `project_dir`, or `None` outside the project. +fn relative_parts(path: &std::path::Path, project_dir: &std::path::Path) -> Option> { + match path.strip_prefix(project_dir) { + Ok(relative) => Some( + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect(), + ), + Err(_) => None, + } +} + +/// Format the SQL bodies of `files`; return changed texts by path and declined-body faults. +#[pyfunction] +fn format_lint_sql_bodies<'py>( + py: Python<'py>, + files: Vec<(Bound<'py, PyAny>, String)>, + project_dir: PathBuf, + config: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyDict>, Vec>)> { + let inputs: Vec = files + .iter() + .map(|(path, contents)| { + Ok(FormatFile { + relative_parts: relative_parts(&path.extract::()?, &project_dir), + contents: contents.clone(), + }) + }) + .collect::>()?; + let dialect: String = config.getattr("dialect")?.extract()?; + let line_width: usize = config.getattr("line_width")?.extract()?; + let result: FormattedBodies = py + .compiler_detach(|| { + format_sql_bodies(&inputs, &dialect, line_width).map_err(|error| error.to_string()) + }) + .map_err(value_error)?; + let formatted: Bound<'py, PyDict> = PyDict::new(py); + for (index, text) in result.formatted { + formatted.set_item(&files[index].0, text)?; + } + let violation: Bound<'py, PyAny> = imported(py, LINT_MODELS_MODULE, "LintViolation")?; + let fault: Bound<'py, PyAny> = + imported(py, LINT_TYPES_MODULE, "LintSeverity")?.getattr("FAULT")?; + let faults: Vec> = result + .faults + .into_iter() + .map(|item| { + call_with( + &violation, + &[ + ("file_path", files[item.file].0.clone()), + ("line", item.line.into_pyobject(py)?.into_any()), + ("column", 1_usize.into_pyobject(py)?.into_any()), + ("code", PyString::new(py, FORMAT_UNSAFE_CODE).into_any()), + ( + "message", + PyString::new( + py, + &format!( + "File left unchanged because its SQL body could not be formatted: {}", + item.reason + ), + ) + .into_any(), + ), + ("severity", fault.clone()), + ("engine", PyString::new(py, NATIVE_ENGINE).into_any()), + ( + "remediation", + PyString::new(py, FORMAT_UNSAFE_REMEDIATION).into_any(), + ), + ], + ) + }) + .collect::>()?; + Ok((formatted, faults)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(neutralize_lint_interpolation, module)?)?; + module.add_function(wrap_pyfunction!( + neutralize_lint_context_interpolation, + module + )?)?; + module.add_function(wrap_pyfunction!(neutralize_lint_audit_parameters, module)?)?; + module.add_function(wrap_pyfunction!(restore_lint_interpolation, module)?)?; + module.add_function(wrap_pyfunction!(lint_interpolation_text_at, module)?)?; + module.add_function(wrap_pyfunction!( + scan_fixture_typed_null_candidates, + module + )?)?; + module.add_function(wrap_pyfunction!(format_lint_sql_bodies, module)?)?; + module.add_function(wrap_pyfunction!(apply_lint_suppressions, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs index 57a21d4b9a..3958390e03 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/mod.rs @@ -5,11 +5,14 @@ pub(crate) mod dsl_headers; pub(crate) mod evaluation; pub(crate) mod findings; pub(crate) mod lint; +pub(crate) mod lint_sql_bodies; +pub(crate) mod lint_text; pub(crate) mod registration; pub(crate) mod relation_keys; pub(crate) mod results; pub(crate) mod rows; pub(crate) mod rule_cache; pub(crate) mod rules_config; +pub(crate) mod run_lint; pub(crate) mod skills; pub(crate) mod sql_test_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs index f07045325b..b57c779d18 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/registration.rs @@ -4,18 +4,21 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::rules::{ builtin_catalogue, custom_rule_cache, custom_rule_evidence, dsl_headers, evaluation, lint, - results, rule_cache, rules_config, skills, + lint_sql_bodies, lint_text, results, rule_cache, rules_config, run_lint, skills, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { evaluation::register(module)?; lint::register(module)?; + lint_text::register(module)?; + lint_sql_bodies::register(module)?; dsl_headers::register(module)?; custom_rule_evidence::register(module)?; rule_cache::register(module)?; custom_rule_cache::register(module)?; builtin_catalogue::register(module)?; rules_config::register(module)?; + run_lint::register(module)?; results::register(module)?; skills::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs new file mode 100644 index 0000000000..0dfbbdf000 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/run_lint.rs @@ -0,0 +1,330 @@ +//! Lint authored SQL files: header rules, then expanded bodies through the native SQL engine. + +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::{PyException, PyOSError}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyTuple}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_core::format_command::main::collect_lint_files::collect_lint_files; +use sqlbuild_rules::lint_text::main::first_header_only::first_header_only; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const EXPANSION_MODULE: &str = "sqlbuild.lint._helpers.expansion"; +const STOP_CONTEXT_MODULE: &str = "sqlbuild.lint.classes.stop_context"; +const LINT_MODELS_MODULE: &str = "sqlbuild.lint.models"; + +fn resolved(path: &Path) -> PathBuf { + fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +fn selected_set(selected_paths: &Bound<'_, PyAny>) -> PyResult>> { + if selected_paths.is_none() { + return Ok(None); + } + selected_paths + .try_iter()? + .map(|path| path?.extract()) + .collect::>() + .map(Some) +} + +/// Authored SQL files keyed by path: `source_files` filtered to the selection, or read from disk. +fn project_files<'py>( + project_dir: &Bound<'py, PyAny>, + selected_paths: &Bound<'py, PyAny>, + source_files: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let selected: Option> = selected_set(selected_paths)?; + let files: Bound<'py, PyDict> = PyDict::new(py); + if !source_files.is_none() { + for (path, contents) in source_files.downcast::()?.iter() { + let keep: bool = match &selected { + None => true, + Some(paths) => paths.contains(&resolved(&path.extract::()?)), + }; + if keep { + files.set_item(path, contents)?; + } + } + return Ok(files); + } + let directory: PathBuf = project_dir.extract()?; + let collected: Vec<(PathBuf, String)> = collect_lint_files(&directory, selected.as_ref()) + .map_err(|error| PyOSError::new_err(error.to_string()))?; + for (path, contents) in collected { + files.set_item(path, contents)?; + } + Ok(files) +} + +/// Collect supported SQL-analysis inputs of a project. +#[pyfunction] +#[pyo3(signature = (*, project_dir, selected_paths = None))] +fn collect_lint_project_files<'py>( + project_dir: &Bound<'py, PyAny>, + selected_paths: Option>, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let none: Bound<'py, PyAny> = py.None().into_bound(py); + project_files( + project_dir, + &selected_paths.unwrap_or_else(|| none.clone()), + &none, + ) +} + +/// `LintStopContext.check()`: abandon a speculative run its owner no longer needs. +fn check_stop(py: Python<'_>) -> PyResult<()> { + imported(py, STOP_CONTEXT_MODULE, "LintStopContext")?.call_method0("check")?; + Ok(()) +} + +/// The keyword arguments of one lint run. +struct LintRun<'py> { + project_dir: Bound<'py, PyAny>, + config: Bound<'py, PyAny>, + dynamic_output_paths: Bound<'py, PyAny>, + compiled_expansions: Bound<'py, PyAny>, + skip_unexpandable: bool, +} + +/// One file's header violations and, when expanding, its expanded bodies. +fn scan_file<'py>( + run: &LintRun<'py>, + (file_path, contents): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + context: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Option>)> { + let py: Python<'py> = file_path.py(); + let relative_path: Bound<'py, PyAny> = + file_path.call_method1("relative_to", (&run.project_dir,))?; + let relative_parts: Vec = relative_path.getattr("parts")?.extract()?; + let headers: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "scan_headers")?, + &[ + ("contents", contents.clone()), + ( + "first_only", + PyBool::new(py, first_header_only(&relative_parts)) + .to_owned() + .into_any(), + ), + ], + )?; + let header_violations: Bound<'py, PyAny> = + if run.config.getattr("header_rules_enabled")?.is_truthy()? { + call_with( + &imported(py, NATIVE_MODULE, "lint_native_headers")?, + &[ + ("contents", contents.clone()), + ("file_path", file_path.clone()), + ("headers", headers.clone()), + ("config", run.config.clone()), + ], + )? + } else { + PyTuple::empty(py).into_any() + }; + if context.is_none() { + return Ok((header_violations, None)); + } + let expanded: Bound<'py, PyAny> = call_with( + &imported(py, EXPANSION_MODULE, "expand_file_bodies")?, + &[ + ("file_path", file_path.clone()), + ("contents", contents.clone()), + ("headers", headers), + ("context", context.clone()), + ("dialect", run.config.getattr("dialect")?), + ("project_dir", run.project_dir.clone()), + ("relative_path", relative_path), + ("dynamic_output_paths", run.dynamic_output_paths.clone()), + ("compiled_expansions", run.compiled_expansions.clone()), + ], + )?; + Ok((header_violations, Some(expanded))) +} + +/// The expansion context: the caller's, or one built when native lint is enabled. +fn expansion_context<'py>( + run: &LintRun<'py>, + arguments: &KeywordArguments<'py>, +) -> PyResult> { + let py: Python<'py> = run.config.py(); + let given: Bound<'py, PyAny> = arguments.value("expansion_context")?; + if given.is_truthy()? { + return Ok(given); + } + if !run.config.getattr("native_enabled")?.is_truthy()? { + return Ok(py.None().into_bound(py)); + } + call_with( + &imported(py, EXPANSION_MODULE, "build_lint_expansion_context")?, + &[ + ("project_dir", run.project_dir.clone()), + ("value_renderer", arguments.value("value_renderer")?), + ("discovered_inputs", arguments.value("discovered_inputs")?), + ], + ) +} + +/// Violations sorted by file path text, then line and column. +fn sorted_violations<'py>(violations: Vec>) -> PyResult>> { + let mut keyed: Vec<((String, i64, i64), Bound<'py, PyAny>)> = violations + .into_iter() + .map(|violation| { + let key: (String, i64, i64) = ( + violation.getattr("file_path")?.str()?.extract()?, + violation.getattr("line")?.extract()?, + violation.getattr("column")?.extract()?, + ); + Ok((key, violation)) + }) + .collect::>()?; + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(keyed.into_iter().map(|(_, violation)| violation).collect()) +} + +/// Header and expanded-body violations of every file, and the files expansion skipped. +fn lint_files<'py>( + run: &LintRun<'py>, + files: &Bound<'py, PyDict>, + context: &Bound<'py, PyAny>, +) -> PyResult<(Vec>, Bound<'py, PyAny>)> { + let py: Python<'py> = files.py(); + let mut items: Vec<(PathBuf, Bound<'py, PyAny>, Bound<'py, PyAny>)> = files + .iter() + .map(|(path, contents)| Ok((path.extract::()?, path, contents))) + .collect::>()?; + items.sort_by(|left, right| left.0.cmp(&right.0)); + let mut violations: Vec> = Vec::new(); + let mut expanded_files: Vec> = Vec::new(); + let mut failure: Option = None; + for (_, file_path, contents) in &items { + let (header_violations, expanded) = match scan_file(run, (file_path, contents), context) { + Ok(scanned) => scanned, + Err(error) if error.is_instance_of::(py) => { + failure = Some(error); + break; + } + Err(error) => return Err(error), + }; + for violation in header_violations.try_iter()? { + violations.push(violation?); + } + let Some(expanded) = expanded else { + continue; + }; + let failed: bool = !expanded.getattr("failure")?.is_none(); + expanded_files.push(expanded); + if failed && !run.skip_unexpandable { + break; + } + } + let skip: Bound<'py, PyAny> = PyBool::new(py, run.skip_unexpandable).to_owned().into_any(); + let prepared: Bound<'py, PyAny> = call_with( + &imported(py, EXPANSION_MODULE, "prepare_expanded_files")?, + &[ + ( + "expanded_files", + PyTuple::new(py, expanded_files)?.into_any(), + ), + ("dialect", run.config.getattr("dialect")?), + ("skip_unexpandable", skip), + ], + )?; + if let Some(error) = failure { + return Err(error); + } + let bodies: Bound<'py, PyAny> = prepared.getattr("bodies")?; + if bodies.is_truthy()? && run.config.getattr("native_enabled")?.is_truthy()? { + check_stop(py)?; + let body_list: Vec> = bodies.extract()?; + let found: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "lint_native_sql_bodies")? + .call1((body_list, files, &run.config))?; + for violation in found.try_iter()? { + violations.push(violation?); + } + check_stop(py)?; + } + Ok((violations, prepared.getattr("unexpandable")?)) +} + +/// Lint all DSL files without modifying anything; optionally skip unexpandable SQL. +#[pyfunction] +#[pyo3(signature = (**options))] +fn run_lint<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "run_lint", + options, + ( + &["project_dir", "config"], + &[ + "value_renderer", + "selected_paths", + "discovered_inputs", + "dynamic_output_paths", + "compiled_expansions", + "expansion_context", + "source_files", + "skip_unexpandable", + ], + ), + )?; + let project_dir: Bound<'py, PyAny> = arguments.value("project_dir")?; + let dynamic_output_paths: Bound<'py, PyAny> = if arguments.passed("dynamic_output_paths")? { + arguments.value("dynamic_output_paths")? + } else { + PyFrozenSet::empty(py)?.into_any() + }; + let run: LintRun<'py> = LintRun { + project_dir, + config: arguments.value("config")?, + dynamic_output_paths, + compiled_expansions: arguments.value("compiled_expansions")?, + skip_unexpandable: arguments.flag("skip_unexpandable")?, + }; + let files: Bound<'py, PyDict> = project_files( + &run.project_dir, + &arguments.value("selected_paths")?, + &arguments.value("source_files")?, + )?; + let context: Bound<'py, PyAny> = expansion_context(&run, &arguments)?; + let (violations, unexpandable) = lint_files(&run, &files, &context)?; + let allow_suppressions: Bound<'py, PyAny> = run.config.getattr("allow_suppressions")?; + let retained: Vec> = imported(py, NATIVE_MODULE, "apply_lint_suppressions")? + .call1((violations, &files, allow_suppressions))? + .extract()?; + call_with( + &imported(py, LINT_MODELS_MODULE, "LintRunResult")?, + &[ + ("files_checked", files.len().into_pyobject(py)?.into_any()), + ( + "violations", + PyTuple::new(py, sorted_violations(retained)?)?.into_any(), + ), + ("formatted_files", PyTuple::empty(py).into_any()), + ("source_texts", files.into_any()), + ("unexpandable", unexpandable), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_lint, module)?)?; + module.add_function(wrap_pyfunction!(collect_lint_project_files, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs new file mode 100644 index 0000000000..27b0c40a09 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs @@ -0,0 +1,165 @@ +//! `sqb scenario capture`: capture the selected scenarios' inputs to durable local snapshots. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyTuple, PyTupleMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::format_summary_footer::format_summary_footer; +use sqlbuild_core::cli_text::models::{CliStyle, SummaryCount}; +use sqlbuild_core::scenario_command::main::snapshot_capture_warning::snapshot_capture_warning; +use sqlbuild_core::scenario_command::main::sql_analysis_required_error::sql_analysis_required_error; +use sqlbuild_core::scenario_command::models::SqlAnalysisCommand; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::scenario_command::conversions::succeeded; +use crate::bindings::_helpers::scenario_command::presentation::{ + CompletionKind, SectionStatus, activity, begin_section, connection_hooks, progress_relay, + write_completion, +}; +use crate::bindings::_helpers::scenario_command::setup::{ + CompileInputs, ScenarioProject, capture_limits, capture_settings, cli_user_error, compile, + discover_project, flush, project_adapter_name, project_connection_config, + require_capture_dialect, require_sql_analysis, resolve_adapter, selected_scenarios, style, + write, write_headers, +}; + +const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; +const EXECUTION_COMMAND: &str = "sqb scenario capture"; + +fn count(label: &str, value: usize) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value: i64::try_from(value).unwrap_or(i64::MAX), + } +} + +/// Run one `sqb scenario capture` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_scenario_capture_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let no_sql_validation: Bound<'_, PyAny> = request.getattr("no_sql_validation")?; + if let Some(error) = sql_analysis_required_error( + SqlAnalysisCommand::Capture, + no_sql_validation.is_truthy()?, + None, + ) { + return Err(cli_user_error(py, error)); + } + let project: ScenarioProject<'_> = discover_project(py, request)?; + require_sql_analysis(&project, (SqlAnalysisCommand::Capture, false))?; + let adapter_name: String = project_adapter_name(&project)?; + let adapter: Bound<'_, PyAny> = resolve_adapter(&project, &adapter_name)?; + let capture_dialect: String = require_capture_dialect(&adapter, &adapter_name)?; + let connection_config: Bound<'_, PyAny> = project_connection_config(&project)?; + let style: CliStyle = style(request)?; + let stream: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + let planning: Bound<'_, PyAny> = + write_headers(&stream, &project, (EXECUTION_COMMAND, 1, style))?; + let force: bool = request + .getattr("limit_inputs")? + .getattr("force")? + .is_truthy()?; + write(&stream, &format!("{}\n\n", snapshot_capture_warning(force)))?; + flush(&stream)?; + let pipeline: Bound<'_, PyAny> = compile( + &project, + CompileInputs { + adapter: &adapter, + options: &[ + ("no_sql_validation", no_sql_validation), + ("connection_config", connection_config.clone()), + ], + planning: &planning, + }, + )?; + let selectors: Vec = project.selectors.extract()?; + let exclude: Vec = project.exclude.extract()?; + let scenarios: Bound<'_, PyTuple> = + selected_scenarios(&pipeline, (&selectors, &exclude, &project.project_dir))?; + let limits: Bound<'_, PyAny> = capture_limits(request, &project)?; + let settings: Bound<'_, PyAny> = capture_settings( + py, + ( + &adapter_name, + &capture_dialect, + request.getattr("retain")?.is_truthy()?, + ), + limits, + )?; + let section: SectionStatus<'_> = begin_section( + &stream, + ( + &format!("Scenario Capture ({} selected)", scenarios.len()), + "", + activity("Capturing scenarios...", &project.namespace), + ), + style, + )?; + let hooks: Bound<'_, PyAny> = connection_hooks(&stream, (&adapter_name, style))?; + let relay = progress_relay( + &stream, + §ion, + ( + &project.project_dir, + &adapter, + CompletionKind::Capture, + style, + ), + )?; + let results: Vec> = call_with( + &imported(py, EXECUTOR_RUN_MODULE, "run_scenario_capture_pipeline")?, + &[ + ("project_dir", project.project_dir.clone()), + ("pipeline_result", pipeline), + ("scenarios", scenarios.into_any()), + ("connection_config", connection_config), + ("adapter", adapter.clone()), + ( + "project_name", + project + .discovered + .getattr("project_config")? + .getattr("name")?, + ), + ("settings", settings), + ("connection_hooks", hooks), + ("on_scenario_start", relay.getattr("on_scenario_start")?), + ( + "on_scenario_complete", + relay.getattr("on_scenario_complete")?, + ), + ], + )? + .try_iter()? + .collect::>()?; + section.status.call_method0("close")?; + let mut passed: usize = 0; + for result in &results { + if succeeded(result)? { + passed += 1; + } + } + let failed: usize = results.len() - passed; + write( + &stream, + &format!( + "\n{}\n", + format_summary_footer( + style, + &[ + count("PASS", passed), + count("FAIL", failed), + count("TOTAL", results.len()) + ], + None + ) + ), + )?; + flush(&stream)?; + write_completion(&stream, (&project.namespace, failed == 0))?; + Ok(i32::from(failed != 0)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_scenario_capture_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs new file mode 100644 index 0000000000..43e3e99094 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/conversions.rs @@ -0,0 +1,440 @@ +//! Scenario executor results read into plain rows and runtime artifact inputs. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_core::scenario_command::main::build_local_runtime_artifacts::build_local_runtime_artifacts; +use sqlbuild_core::scenario_command::main::build_runtime_artifacts::build_runtime_artifacts; +use sqlbuild_core::scenario_command::main::write_runtime_artifacts::write_runtime_artifacts; +use sqlbuild_core::scenario_command::models::{ + AssertionRow, CaptureRelationRow, CaptureRow, CodedText, ExpectationSql, ExpectedRow, + FixtureStatements, FunctionStatements, LoadedRelation, LocalRuntimeInputs, ModelArtifactEntry, + NamedStatements, RetainedInputs, RetainedRelation, ScenarioResultRow, WarehouseRuntimeInputs, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::scenario_command::setup::text; + +const SUCCESS_STATUS: &str = "success"; +const FAILED_STATUS: &str = "failed"; +const SQL_EVENT_KIND: &str = "sql"; +const PASS_STATUS: &str = "PASS"; +const FAIL_STATUS: &str = "FAIL"; +const RUN_DIR: &str = "run"; +const SCENARIOS_DIR: &str = "scenarios"; +const LOCAL_DIR: &str = "local"; +const PLANNER_TYPES: &str = "sqlbuild.compiler.planner.types"; +const QUALIFIED_NAME_MODULE: &str = + "sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"; +const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.scenario.main.expected_comparison_sql"; + +fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + value.try_iter()?.collect() +} + +fn enum_value(value: &Bound<'_, PyAny>) -> PyResult { + value.getattr("value")?.extract() +} + +/// `result.status` is the given `ExecutionStatus` value. +pub(crate) fn matching_status(result: &Bound<'_, PyAny>, status: &str) -> PyResult { + Ok(enum_value(&result.getattr("status")?)? == status) +} + +pub(crate) fn succeeded(result: &Bound<'_, PyAny>) -> PyResult { + matching_status(result, SUCCESS_STATUS) +} + +/// `path.as_posix()`. +fn posix(path: &Bound<'_, PyAny>) -> PyResult { + path.call_method0("as_posix")?.extract() +} + +/// The result's coded error, when its message is set. +fn coded_error(result: &Bound<'_, PyAny>) -> PyResult> { + let message: Option = result.getattr("error_message")?.extract()?; + let Some(message) = message else { + return Ok(None); + }; + Ok(Some(CodedText { + code: result.getattr("error_code")?.extract()?, + message, + help: result.getattr("error_help")?.extract()?, + })) +} + +fn expected_row(expected: &Bound<'_, PyAny>) -> PyResult { + Ok(ExpectedRow { + model_name: expected.getattr("model_name")?.extract()?, + failed: matching_status(expected, FAILED_STATUS)?, + actual_row_count: expected.getattr("actual_row_count")?.extract()?, + expected_row_count: expected.getattr("expected_row_count")?.extract()?, + mismatched_row_count: expected.getattr("mismatched_row_count")?.extract()?, + error: coded_error(expected)?, + }) +} + +fn assertion_row(assertion: &Bound<'_, PyAny>) -> PyResult { + let samples: Vec> = items(&assertion.getattr("sample_rows")?)?; + Ok(AssertionRow { + name: assertion.getattr("name")?.extract()?, + failed: matching_status(assertion, FAILED_STATUS)?, + failing_row_count: assertion.getattr("failing_row_count")?.extract()?, + first_sample: samples + .first() + .map(|sample| sample.str()?.extract()) + .transpose()?, + error: coded_error(assertion)?, + }) +} + +fn retained_relations(relation_map: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = relation_map.py(); + let kinds: Bound<'_, PyAny> = imported(py, PLANNER_TYPES, "ScenarioArtifactKind")?; + items(&relation_map.getattr("artifacts")?)? + .iter() + .map(|artifact| { + let identity: Bound<'_, PyAny> = artifact.getattr("identity")?; + Ok(RetainedRelation { + kind: enum_value(&kinds.call1((identity.getattr("kind")?,))?)?, + logical_name: identity.getattr("logical_name")?.extract()?, + physical_name: artifact.getattr("physical_name")?.extract()?, + }) + }) + .collect() +} + +/// One `ScenarioRunResult` as its result row shows it. +pub(crate) fn scenario_result_row(result: &Bound<'_, PyAny>) -> PyResult { + let local_status: Bound<'_, PyAny> = result.getattr("local_status")?; + let local: bool = !local_status.is_none(); + let status: String = if local { + enum_value(&local_status)? + } else if succeeded(result)? { + PASS_STATUS.to_owned() + } else { + FAIL_STATUS.to_owned() + }; + let relation_map: Bound<'_, PyAny> = result.getattr("relation_map")?; + let duckdb_path: Bound<'_, PyAny> = result.getattr("local_duckdb_path")?; + Ok(ScenarioResultRow { + name: result.getattr("scenario_name")?.extract()?, + status, + local, + duration_ms: result.getattr("duration_ms")?.extract()?, + error: coded_error(result)?, + retained: result.getattr("retained")?.is_truthy()?, + retained_relations: if relation_map.is_none() { + None + } else { + Some(retained_relations(&relation_map)?) + }, + local_duckdb_path: if duckdb_path.is_none() { + None + } else { + Some(posix(&duckdb_path)?) + }, + expected: items(&result.getattr("expected_results")?)? + .iter() + .map(expected_row) + .collect::>()?, + assertions: items(&result.getattr("assertion_results")?)? + .iter() + .map(assertion_row) + .collect::>()?, + }) +} + +/// `manifest_path` relative to the project when it is inside it, as a POSIX path. +fn display_path( + manifest_path: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult { + match manifest_path.call_method1("relative_to", (project_dir,)) { + Ok(relative) => posix(&relative), + Err(error) + if error.is_instance_of::(manifest_path.py()) => + { + posix(manifest_path) + } + Err(error) => Err(error), + } +} + +/// One `ScenarioSnapshotCaptureRunResult` as capture and sync rows show it. +pub(crate) fn capture_row( + result: &Bound<'_, PyAny>, + (project_dir, scenario_plan): (&Bound<'_, PyAny>, Option<&Bound<'_, PyAny>>), +) -> PyResult { + let retained: bool = result.getattr("retained")?.is_truthy()?; + let capture: Bound<'_, PyAny> = result.getattr("capture_result")?; + let (relations, snapshot_path) = if capture.is_none() { + (None, None) + } else { + let relations: Vec = items(&capture.getattr("relation_results")?)? + .iter() + .map(|relation| { + Ok(CaptureRelationRow { + kind: enum_value(&relation.getattr("kind")?)?, + logical_name: relation.getattr("logical_name")?.extract()?, + passed: succeeded(relation)?, + row_count: relation.getattr("row_count")?.extract()?, + byte_count: relation.getattr("byte_count")?.extract()?, + }) + }) + .collect::>()?; + let manifest_path: Bound<'_, PyAny> = capture.getattr("manifest_path")?; + let snapshot_path: Option = if manifest_path.is_none() { + None + } else { + Some(display_path(&manifest_path, project_dir)?) + }; + (Some(relations), snapshot_path) + }; + let retained_inputs: Option = if retained && scenario_plan.is_some() { + Some(RetainedInputs { + fixtures: items(&result.getattr("fixture_results")?)? + .iter() + .map(|fixture| { + Ok(RetainedRelation { + kind: enum_value(&fixture.getattr("kind")?)?, + logical_name: fixture.getattr("logical_name")?.extract()?, + physical_name: fixture.getattr("target_relation")?.extract()?, + }) + }) + .collect::>()?, + seeds: items(&result.getattr("seed_results")?)? + .iter() + .map(|seed| seed.getattr("seed_name")?.extract()) + .collect::>()?, + }) + } else { + None + }; + Ok(CaptureRow { + name: result.getattr("scenario_name")?.extract()?, + passed: succeeded(result)?, + error: coded_error(result)?, + retained, + relations, + snapshot_path, + retained_inputs, + }) +} + +/// The SQL contents of an executed result's lifecycle events. +fn sql_statements(result: &Bound<'_, PyAny>) -> PyResult> { + let mut statements: Vec = Vec::new(); + for event in items(&result.getattr("lifecycle_events")?)? { + if enum_value(&event.getattr("kind")?)? == SQL_EVENT_KIND { + statements.push(event.getattr("content")?.extract()?); + } + } + Ok(statements) +} + +fn cleanup_statements(cleanup: &Bound<'_, PyAny>) -> PyResult>> { + if cleanup.is_none() { + return Ok(None); + } + sql_statements(cleanup).map(Some) +} + +/// Each expected comparison and assertion of a plan, rendered for `adapter`. +fn expectation_sql( + scenario_plan: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = adapter.py(); + let mut expectations: ExpectationSql = ExpectationSql::default(); + for expected in items(&scenario_plan.getattr("expected_expectations")?)? { + let actual_relation: String = call_with( + &imported( + py, + QUALIFIED_NAME_MODULE, + "resolve_relation_location_qualified_name", + )?, + &[ + ("adapter", adapter.clone()), + ("location", expected.getattr("actual_destination")?), + ], + )? + .extract()?; + let sql: String = call_with( + &imported( + py, + COMPARISON_SQL_MODULE, + "build_scenario_expected_comparison_sql", + )?, + &[ + ( + "actual_sql", + text(py, &format!("SELECT * FROM {actual_relation}")), + ), + ("expected_sql", expected.getattr("expected_sql")?), + ( + "set_difference_operator", + adapter.call_method0("render_set_difference_operator")?, + ), + ], + )? + .extract()?; + expectations + .expected + .push((expected.getattr("model_name")?.extract()?, sql)); + } + for assertion in items(&scenario_plan.getattr("assertion_expectations")?)? { + expectations.assertions.push(( + assertion.getattr("name")?.extract()?, + assertion.getattr("sql")?.extract()?, + )); + } + Ok(expectations) +} + +fn scenario_run_dir(target_dir: &Path, scenario_plan: &Bound<'_, PyAny>) -> PyResult { + let name: String = scenario_plan.getattr("name")?.extract()?; + Ok(target_dir.join(RUN_DIR).join(SCENARIOS_DIR).join(name)) +} + +fn written(root: &Path, result: std::io::Result<()>) -> PyResult<()> { + result.map_err(|error| PyOSError::new_err(format!("{error}: {}", root.display()))) +} + +/// Write executed warehouse scenario lifecycle SQL under `target/run/scenarios//`. +pub(crate) fn write_warehouse_runtime_target( + target_dir: &Path, + (adapter, scenario_plan, result): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + let named = |values: Bound<'_, PyAny>, field: &str| -> PyResult> { + items(&values)? + .iter() + .map(|value| { + Ok(NamedStatements { + name: value.getattr(field)?.extract()?, + statements: sql_statements(value)?, + }) + }) + .collect() + }; + let inputs: WarehouseRuntimeInputs = WarehouseRuntimeInputs { + prepare_cleanup: cleanup_statements(&result.getattr("prepare_cleanup_result")?)?, + fixtures: items(&result.getattr("fixture_results")?)? + .iter() + .map(|fixture| { + Ok(FixtureStatements { + kind: enum_value(&fixture.getattr("kind")?)?, + logical_name: fixture.getattr("logical_name")?.extract()?, + statements: sql_statements(fixture)?, + }) + }) + .collect::>()?, + seeds: named(result.getattr("seed_results")?, "seed_name")?, + models: named(result.getattr("model_results")?, "model_name")?, + model_entries: items(&scenario_plan.getattr("model_entries")?)? + .iter() + .map(|entry| { + Ok(ModelArtifactEntry { + name: entry.getattr("name")?.extract()?, + relative_path: entry.getattr("relative_path")?.extract()?, + }) + }) + .collect::>()?, + expectations: expectation_sql(scenario_plan, adapter)?, + final_cleanup: cleanup_statements(&result.getattr("cleanup_result")?)?, + }; + let root: PathBuf = scenario_run_dir(target_dir, scenario_plan)?; + written( + &root, + write_runtime_artifacts(&root, &build_runtime_artifacts(&inputs)), + ) +} + +fn function_statements( + entry: &Bound<'_, PyAny>, + adapter: &Bound<'_, PyAny>, +) -> PyResult { + let destination: Bound<'_, PyAny> = entry.getattr("destination")?; + let qualified: Bound<'_, PyAny> = destination.getattr("qualified_name")?; + let destination_name: Bound<'_, PyAny> = if qualified.is_truthy()? { + qualified + } else { + destination.getattr("name")? + }; + let language: Bound<'_, PyAny> = entry.getattr("language")?; + let rendered: Bound<'_, PyAny> = call_with( + &adapter.getattr("render_create_function")?, + &[ + ("destination", destination_name), + ("arguments", entry.getattr("arguments")?), + ("returns", entry.getattr("returns")?), + ("body_sql", entry.getattr("body_sql")?), + ("return_columns", entry.getattr("return_columns")?), + ("language", language.clone()), + ("runtime_version", entry.getattr("runtime_version")?), + ("entry_point", entry.getattr("entry_point")?), + ("packages", entry.getattr("packages")?), + ], + )?; + Ok(FunctionStatements { + relative_path: entry.getattr("relative_path")?.extract()?, + language: enum_value(&language)?, + statements: items(&rendered)? + .iter() + .map(|statement| statement.extract()) + .collect::>()?, + }) +} + +/// Write local replay SQL under `target/run/scenarios//local/`. +pub(crate) fn write_local_runtime_target( + target_dir: &Path, + (adapter, scenario_plan, result): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult<()> { + let local_plan: Bound<'_, PyAny> = match result.getattr("local_execution_plan")? { + plan if plan.is_truthy()? => plan, + _ => scenario_plan.clone(), + }; + let executed: HashSet = items(&result.getattr("function_results")?)? + .iter() + .map(|function| function.getattr("function_name")?.extract()) + .collect::>()?; + let mut functions: Vec = Vec::new(); + for entry in items(&local_plan.getattr("function_entries")?)? { + if executed.contains(&entry.getattr("name")?.extract::()?) { + functions.push(function_statements(&entry, adapter)?); + } + } + let inputs: LocalRuntimeInputs = LocalRuntimeInputs { + loaded_relations: items(&result.getattr("local_snapshot_relations")?)? + .iter() + .map(|relation| { + Ok(LoadedRelation { + kind: enum_value(&relation.getattr("kind")?)?, + logical_name: relation.getattr("logical_name")?.extract()?, + file_path: posix(&relation.getattr("file_path")?)?, + table_name: relation.getattr("table_name")?.extract()?, + row_count: relation.getattr("row_count")?.extract()?, + }) + }) + .collect::>()?, + functions, + models: items(&local_plan.getattr("model_entries")?)? + .iter() + .map(|entry| { + Ok(( + entry.getattr("relative_path")?.extract()?, + entry.getattr("resolved_sql")?.extract()?, + )) + }) + .collect::>()?, + expectations: expectation_sql(&local_plan, adapter)?, + }; + let root: PathBuf = scenario_run_dir(target_dir, scenario_plan)?.join(LOCAL_DIR); + written( + &root, + write_runtime_artifacts(&root, &build_local_runtime_artifacts(&inputs)), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/mod.rs new file mode 100644 index 0000000000..b1a5080091 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod capture_command; +pub(crate) mod conversions; +pub(crate) mod presentation; +pub(crate) mod registration; +pub(crate) mod setup; +pub(crate) mod snapshot_sync; +pub(crate) mod test_command; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs new file mode 100644 index 0000000000..5c8be6460d --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs @@ -0,0 +1,273 @@ +//! Scenario progress: section headers, live status, per-scenario rows and connection timing. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::scenario_command::main::render_capture_result::render_capture_result; +use sqlbuild_core::scenario_command::main::render_scenario_result::render_scenario_result; +use sqlbuild_core::scenario_command::main::render_snapshot_sync_result::render_snapshot_sync_result; +use sqlbuild_core::scenario_command::main::scenario_activity_message::scenario_activity_message; +use sqlbuild_core::scenario_command::models::ScenarioNamespace; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::scenario_command::conversions::{ + capture_row, scenario_result_row, write_local_runtime_target, write_warehouse_runtime_target, +}; +use crate::bindings::_helpers::scenario_command::setup::{ + NATIVE_MODULE, boolean, flush, stream_isatty, text, write, +}; + +const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; +const RUNTIME_CONTRACT_MODELS: &str = "sqlbuild.runtime.contracts.models"; + +/// How a finished scenario is reported. +#[derive(Clone, Debug)] +pub(crate) enum CompletionKind { + /// A test run: write its runtime SQL under `target_dir` and its result row. + Run { target_dir: PathBuf }, + /// A snapshot capture: its capture row. + Capture, + /// A local snapshot sync or refresh: its sync row. + Sync { refresh: bool }, +} + +/// Starts the live status per scenario and reports each finished scenario. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ScenarioProgressRelay { + stream: Py, + status: Py, + status_is_tty: bool, + activity: String, + use_color: bool, + project_dir: Py, + adapter: Py, + kind: CompletionKind, +} + +#[pymethods] +impl ScenarioProgressRelay { + fn on_scenario_start(&self, py: Python<'_>, _scenario: Bound<'_, PyAny>) -> PyResult<()> { + if self.status_is_tty { + self.status + .bind(py) + .call_method1("start", (self.activity.as_str(),))?; + } + Ok(()) + } + + fn on_scenario_complete( + &self, + py: Python<'_>, + _scenario: Bound<'_, PyAny>, + scenario_plan: Bound<'_, PyAny>, + result: Bound<'_, PyAny>, + ) -> PyResult<()> { + if self.status_is_tty { + self.status.bind(py).call_method0("close")?; + } + let style: CliStyle = CliStyle { + use_color: self.use_color, + }; + let plan: Option<&Bound<'_, PyAny>> = (!scenario_plan.is_none()).then_some(&scenario_plan); + let project_dir: &Bound<'_, PyAny> = self.project_dir.bind(py); + let rendered: String = match &self.kind { + CompletionKind::Run { target_dir } => { + if let Some(plan) = plan { + let adapter: &Bound<'_, PyAny> = self.adapter.bind(py); + if result.getattr("local_status")?.is_none() { + write_warehouse_runtime_target(target_dir, (adapter, plan, &result))?; + } else { + write_local_runtime_target(target_dir, (adapter, plan, &result))?; + } + } + render_scenario_result(&scenario_result_row(&result)?, style) + } + CompletionKind::Capture => { + render_capture_result(&capture_row(&result, (project_dir, plan))?, style) + } + CompletionKind::Sync { refresh } => render_snapshot_sync_result( + &capture_row(&result, (project_dir, plan))?, + *refresh, + style, + ), + }; + let stream: &Bound<'_, PyAny> = self.stream.bind(py); + write(stream, &rendered)?; + flush(stream) + } +} + +/// Forward connection timing to a reporter that takes keywords. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct ScenarioConnectionRelay { + reporter: Py, +} + +#[pymethods] +impl ScenarioConnectionRelay { + fn on_connection_complete( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.forward( + py, + "on_connection_complete", + (connection_count, elapsed_seconds), + ) + } + + fn on_connection_error( + &self, + py: Python<'_>, + connection_count: i64, + elapsed_seconds: f64, + ) -> PyResult<()> { + self.forward( + py, + "on_connection_error", + (connection_count, elapsed_seconds), + ) + } +} + +impl ScenarioConnectionRelay { + fn forward( + &self, + py: Python<'_>, + method: &str, + (connection_count, elapsed_seconds): (i64, f64), + ) -> PyResult<()> { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("connection_count", connection_count)?; + options.set_item("elapsed_seconds", elapsed_seconds)?; + self.reporter + .bind(py) + .call_method(method, (), Some(&options))?; + Ok(()) + } +} + +/// `ConnectionHooks` reporting connection progress for `adapter_name` on `stream`. +pub(crate) fn connection_hooks<'py>( + stream: &Bound<'py, PyAny>, + (adapter_name, style): (&str, CliStyle), +) -> PyResult> { + let py: Python<'py> = stream.py(); + let reporter: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "ConnectionProgressReporter")?, + &[ + ("adapter_name", text(py, adapter_name)), + ("blank_line_after_complete", boolean(py, true)), + ("stream", stream.clone()), + ("use_color", boolean(py, style.use_color)), + ], + )?; + let relay: Bound<'py, ScenarioConnectionRelay> = Bound::new( + py, + ScenarioConnectionRelay { + reporter: reporter.clone().unbind(), + }, + )?; + call_with( + &imported(py, RUNTIME_CONTRACT_MODELS, "ConnectionHooks")?, + &[ + ( + "on_connection_start", + reporter.getattr("on_connection_start")?, + ), + ( + "on_connection_complete", + relay.getattr("on_connection_complete")?, + ), + ("on_connection_error", relay.getattr("on_connection_error")?), + ], + ) +} + +/// The live status of one scenario section and whether it animates on a terminal. +pub(crate) struct SectionStatus<'py> { + pub(crate) status: Bound<'py, PyAny>, + pub(crate) status_is_tty: bool, + pub(crate) activity: String, +} + +/// Write the section header and preamble, then the activity line when not on a terminal. +pub(crate) fn begin_section<'py>( + stream: &Bound<'py, PyAny>, + (header, preamble, activity): (&str, &str, String), + style: CliStyle, +) -> PyResult> { + let py: Python<'py> = stream.py(); + write( + stream, + &format!("\n{}\n\n{preamble}", style.success_strong(header)), + )?; + flush(stream)?; + let status: Bound<'py, PyAny> = call_with( + &imported(py, STATUS_REPORTER_MODULE, "TransientStatusReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", boolean(py, style.use_color)), + ], + )?; + let status_is_tty: bool = stream_isatty(stream)?; + if !status_is_tty { + write(stream, &format!("{activity}\n\n"))?; + flush(stream)?; + } + Ok(SectionStatus { + status, + status_is_tty, + activity, + }) +} + +/// The relay reporting one section's scenarios. +pub(crate) fn progress_relay<'py>( + stream: &Bound<'py, PyAny>, + section: &SectionStatus<'py>, + (project_dir, adapter, kind, style): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + CompletionKind, + CliStyle, + ), +) -> PyResult> { + Bound::new( + stream.py(), + ScenarioProgressRelay { + stream: stream.clone().unbind(), + status: section.status.clone().unbind(), + status_is_tty: section.status_is_tty, + activity: section.activity.clone(), + use_color: style.use_color, + project_dir: project_dir.clone().unbind(), + adapter: adapter.clone().unbind(), + kind, + }, + ) +} + +/// ` (namespace: ...; source: ...)`. +pub(crate) fn activity(activity: &str, namespace: &ScenarioNamespace) -> String { + scenario_activity_message(activity, namespace) +} + +/// The explicit terminal state of the namespaced run. +pub(crate) fn write_completion( + stream: &Bound<'_, PyAny>, + (namespace, succeeded): (&ScenarioNamespace, bool), +) -> PyResult<()> { + let state: &str = if succeeded { + "Scenario execution complete" + } else { + "Scenario execution failed" + }; + write(stream, &format!("{}\n", activity(state, namespace)))?; + flush(stream) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/registration.rs new file mode 100644 index 0000000000..30bbbf8e11 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/registration.rs @@ -0,0 +1,10 @@ +//! Register the native `sqb scenario test` and `sqb scenario capture` commands. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::scenario_command::{capture_command, test_command}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + test_command::register(module)?; + capture_command::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs new file mode 100644 index 0000000000..512834423f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs @@ -0,0 +1,479 @@ +//! Shared `sqb scenario` preparation: project, namespace, adapters, compile and selection. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::scenario_command::constants::SCENARIO_NAMESPACE_ENV; +use sqlbuild_core::scenario_command::errors::ScenarioCommandError; +use sqlbuild_core::scenario_command::main::capture_dialect_required_error::capture_dialect_required_error; +use sqlbuild_core::scenario_command::main::format_captured_at::format_captured_at; +use sqlbuild_core::scenario_command::main::render_namespace_header::render_namespace_header; +use sqlbuild_core::scenario_command::main::resolve_scenario_namespace::resolve_scenario_namespace; +use sqlbuild_core::scenario_command::main::resolve_snapshot_limits::resolve_snapshot_limits; +use sqlbuild_core::scenario_command::main::select_scenarios::select_scenarios; +use sqlbuild_core::scenario_command::main::sql_analysis_purpose::sql_analysis_purpose; +use sqlbuild_core::scenario_command::main::sql_analysis_required_error::sql_analysis_required_error; +use sqlbuild_core::scenario_command::models::{ + NamespaceCandidates, ScenarioFile, ScenarioNamespace, ScenarioSelection, SnapshotLimitConfig, + SnapshotLimitInputs, SnapshotLimits, SqlAnalysisCommand, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, within}; + +pub(crate) const NATIVE_MODULE: &str = "sqlbuild._native"; +const ERRORS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const REJECT_UNIT_TESTS_MODULE: &str = "sqlbuild.compiler.pipeline.main.reject_unit_test_selectors"; +const GUIDANCE_MODULE: &str = "sqlbuild.compiler.discovery.main._sql_analysis_off_guidance"; +const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; +const SCENARIO_CONFIG_MODULE: &str = + "sqlbuild.spec.contracts.main.resolve_effective_scenario_config"; +const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; +const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; +const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; +const SCENARIO_MODELS: &str = "sqlbuild.executor.scenario.models"; +const METADATA_MODULE: &str = "importlib.metadata"; +const DISTRIBUTION_NAME: &str = "sqlbuild"; +const UNKNOWN_VERSION: &str = "unknown"; + +pub(crate) fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { + PyString::new(py, value).into_any() +} + +pub(crate) fn optional_text<'py>(py: Python<'py>, value: Option<&str>) -> Bound<'py, PyAny> { + value.map_or_else(|| py.None().into_bound(py), |value| text(py, value)) +} + +/// `CliUserError(message, code=..., help=...)`. +pub(crate) fn cli_user_error(py: Python<'_>, error: ScenarioCommandError) -> PyErr { + let raised: PyResult> = (|| { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("code", error.code)?; + options.set_item("help", error.help)?; + imported(py, ERRORS_MODULE, "CliUserError")?.call((error.message,), Some(&options)) + })(); + match raised { + Ok(error) => PyErr::from_value(error), + Err(error) => error, + } +} + +/// The progress stream: `write` then `flush`. +pub(crate) fn write(stream: &Bound<'_, PyAny>, value: &str) -> PyResult<()> { + stream.call_method1("write", (value,))?; + Ok(()) +} + +pub(crate) fn flush(stream: &Bound<'_, PyAny>) -> PyResult<()> { + stream.call_method0("flush")?; + Ok(()) +} + +/// `hasattr(stream, "isatty") and stream.isatty()`. +pub(crate) fn stream_isatty(stream: &Bound<'_, PyAny>) -> PyResult { + if !stream.hasattr("isatty")? { + return Ok(false); + } + stream.call_method0("isatty")?.is_truthy() +} + +/// The project, its discovered inputs and the effective scenario namespace. +pub(crate) struct ScenarioProject<'py> { + pub(crate) project_dir: Bound<'py, PyAny>, + pub(crate) discovered: Bound<'py, PyAny>, + pub(crate) namespace: ScenarioNamespace, + pub(crate) selectors: Bound<'py, PyAny>, + pub(crate) exclude: Bound<'py, PyAny>, +} + +fn config_namespace(config: &Bound<'_, PyAny>) -> PyResult> { + config + .getattr("scenario")? + .getattr("run_namespace")? + .extract() +} + +/// `dataclasses.replace(value, **{name: replacement})`. +fn replaced<'py>( + value: &Bound<'py, PyAny>, + (name, replacement): (&str, Bound<'py, PyAny>), +) -> PyResult> { + let changes: Bound<'py, PyDict> = PyDict::new(value.py()); + changes.set_item(name, replacement)?; + value + .py() + .import("dataclasses")? + .getattr("replace")? + .call((value,), Some(&changes)) +} + +/// Discover the project, reject unit-test selectors and apply the effective namespace. +pub(crate) fn discover_project<'py>( + py: Python<'py>, + request: &Bound<'py, PyAny>, +) -> PyResult> { + let project_dir: Bound<'py, PyAny> = match request.getattr("project_dir")? { + directory if !directory.is_none() => directory, + _ => py.import("pathlib")?.getattr("Path")?.call_method0("cwd")?, + }; + let discovered: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "discover_project_inputs")?, + &[("project_dir", project_dir.clone())], + )?; + let selectors: Bound<'py, PyAny> = request.getattr("selectors")?; + let exclude: Bound<'py, PyAny> = request.getattr("exclude")?; + call_with( + &imported(py, REJECT_UNIT_TESTS_MODULE, "reject_unit_test_selectors")?, + &[ + ("select", selectors.clone()), + ("exclude", exclude.clone()), + ("discovered_inputs", discovered.clone()), + ], + )?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let candidates: NamespaceCandidates = NamespaceCandidates { + cli: request.getattr("scenario_namespace")?.extract()?, + env: py + .import("os")? + .getattr("environ")? + .call_method1("get", (SCENARIO_NAMESPACE_ENV,))? + .extract()?, + local_config: config_namespace(&local_config)?, + project_config: config_namespace(&discovered.getattr("project_config")?)?, + }; + let namespace: ScenarioNamespace = + resolve_scenario_namespace(candidates).map_err(|error| cli_user_error(py, error))?; + let discovered: Bound<'py, PyAny> = match &namespace.value { + Some(value) => { + let scenario: Bound<'py, PyAny> = replaced( + &local_config.getattr("scenario")?, + ("run_namespace", text(py, value)), + )?; + replaced( + &discovered, + ( + "local_config", + replaced(&local_config, ("scenario", scenario))?, + ), + )? + } + None => discovered, + }; + Ok(ScenarioProject { + project_dir, + discovered, + namespace, + selectors, + exclude, + }) +} + +/// Raise when SQL analysis is off by flag or by a config file. +pub(crate) fn require_sql_analysis( + project: &ScenarioProject<'_>, + (command, disabled_by_flag): (SqlAnalysisCommand, bool), +) -> PyResult<()> { + let py: Python<'_> = project.discovered.py(); + let guidance: Option<(String, String)> = if disabled_by_flag { + None + } else { + call_with( + &imported(py, GUIDANCE_MODULE, "sql_analysis_off_guidance")?, + &[ + ("discovered_inputs", project.discovered.clone()), + ("purpose", text(py, sql_analysis_purpose(command))), + ], + )? + .extract()? + }; + match sql_analysis_required_error(command, disabled_by_flag, guidance) { + Some(error) => Err(cli_user_error(py, error)), + None => Ok(()), + } +} + +/// The adapter named by the project's effective configuration. +pub(crate) fn project_adapter_name(project: &ScenarioProject<'_>) -> PyResult { + let py: Python<'_> = project.discovered.py(); + call_with( + &imported(py, ADAPTER_NAME_MODULE, "resolve_effective_adapter_name")?, + &[ + ( + "project_config", + project.discovered.getattr("project_config")?, + ), + ("local_config", project.discovered.getattr("local_config")?), + ], + )? + .extract() +} + +pub(crate) fn resolve_adapter<'py>( + project: &ScenarioProject<'py>, + adapter_name: &str, +) -> PyResult> { + let py: Python<'py> = project.discovered.py(); + call_with( + &imported(py, ADAPTERS_MODULE, "resolve_adapter")?, + &[ + ("adapter_name", text(py, adapter_name)), + ("project_dir", project.project_dir.clone()), + ], + ) +} + +/// The adapter's SQL analysis dialect, which snapshots require. +pub(crate) fn require_capture_dialect( + adapter: &Bound<'_, PyAny>, + adapter_name: &str, +) -> PyResult { + let dialect: Option = adapter.call_method0("sql_analysis_dialect")?.extract()?; + dialect + .ok_or_else(|| cli_user_error(adapter.py(), capture_dialect_required_error(adapter_name))) +} + +pub(crate) fn project_connection_config<'py>( + project: &ScenarioProject<'py>, +) -> PyResult> { + let py: Python<'py> = project.discovered.py(); + call_with( + &imported(py, NATIVE_MODULE, "resolve_project_connection_config")?, + &[ + ("discovered_inputs", project.discovered.clone()), + ("project_dir", project.project_dir.clone()), + ], + ) +} + +/// `not no_color and supports_color()`. +pub(crate) fn style(request: &Bound<'_, PyAny>) -> PyResult { + let use_color: bool = !request.getattr("no_color")?.is_truthy()? + && imported(request.py(), COLOR_MODULE, "supports_color")? + .call0()? + .is_truthy()?; + Ok(CliStyle { use_color }) +} + +/// Write the namespace line, then the execution header with the selected target. +pub(crate) fn write_headers<'py>( + stream: &Bound<'py, PyAny>, + project: &ScenarioProject<'py>, + (command, concurrency, style): (&str, i64, CliStyle), +) -> PyResult> { + let py: Python<'py> = stream.py(); + write(stream, &render_namespace_header(&project.namespace))?; + let selectors: Vec = project.selectors.extract()?; + let planning: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", stream.clone()), + ("use_color", boolean(py, style.use_color)), + ], + )?; + write(stream, "\n")?; + let target: Option = (!selectors.is_empty()).then(|| selectors.join(" ")); + call_with( + &imported(py, NATIVE_MODULE, "write_execution_header")?, + &[ + ("stream", stream.clone()), + ("command", text(py, command)), + ("target", optional_text(py, target.as_deref())), + ("concurrency", int(py, concurrency)), + ("use_color", boolean(py, style.use_color)), + ], + )?; + Ok(planning) +} + +/// The adapter, compile options and planning progress of one scenario compile. +pub(crate) struct CompileInputs<'a, 'py> { + pub(crate) adapter: &'a Bound<'py, PyAny>, + pub(crate) options: &'a [(&'a str, Bound<'py, PyAny>)], + pub(crate) planning: &'a Bound<'py, PyAny>, +} + +/// Compile the project inside the project compile lifecycle. +pub(crate) fn compile<'py>( + project: &ScenarioProject<'py>, + inputs: CompileInputs<'_, 'py>, +) -> PyResult> { + let CompileInputs { + adapter, + options, + planning, + } = inputs; + let py: Python<'py> = project.discovered.py(); + let resolver: Bound<'py, PyAny> = call_with( + &imported( + py, + EXTERNAL_REFS_MODULE, + "resolve_external_sql_reference_resolver", + )?, + &[ + ("project_dir", project.project_dir.clone()), + ("discovered_inputs", project.discovered.clone()), + ], + )?; + let mut pairs: Vec<(&str, Bound<'py, PyAny>)> = options.to_vec(); + pairs.push(("external_sql_reference_resolver", resolver)); + let options: Bound<'py, PyAny> = call_with( + &imported(py, PIPELINE_MODELS, "CompilePipelineOptions")?, + &pairs, + )?; + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", text(py, "project")), + ("operation_name", text(py, "project_compile")), + ], + )?; + within(&lifecycle, |_| { + call_with( + &imported(py, NATIVE_MODULE, "run_compile_only_pipeline")?, + &[ + ("discovered_inputs", project.discovered.clone()), + ("adapter", adapter.clone()), + ("options", options.clone()), + ("on_progress", planning.getattr("on_progress")?), + ], + ) + }) +} + +/// The compiled scenarios the selectors and exclusions select, in selection order. +pub(crate) fn selected_scenarios<'py>( + pipeline: &Bound<'py, PyAny>, + (selectors, exclude, project_dir): (&[String], &[String], &Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = pipeline.py(); + let compiled: Vec> = pipeline + .getattr("project")? + .getattr("sql_scenarios")? + .try_iter()? + .collect::>()?; + let mut files: Vec = Vec::with_capacity(compiled.len()); + for scenario in &compiled { + let file: Bound<'py, PyAny> = scenario.getattr("scenario_file")?; + files.push(ScenarioFile { + name: scenario.getattr("name")?.extract()?, + file_path: file.getattr("file_path")?.extract()?, + relative_path: file.getattr("relative_path")?.extract()?, + }); + } + let project_dir: PathBuf = project_dir.extract()?; + let indices: Vec = select_scenarios( + &files, + ScenarioSelection { + selectors, + exclude, + project_dir: &project_dir, + }, + ) + .map_err(|error| cli_user_error(py, error))?; + PyTuple::new(py, indices.into_iter().map(|index| compiled[index].clone())) +} + +fn optional_int(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + value.getattr(name)?.extract() +} + +/// The snapshot capture limits from the request, the scenario config and the defaults. +pub(crate) fn capture_limits<'py>( + request: &Bound<'py, PyAny>, + project: &ScenarioProject<'py>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let inputs: Bound<'py, PyAny> = request.getattr("limit_inputs")?; + let configured: Bound<'py, PyAny> = call_with( + &imported( + py, + SCENARIO_CONFIG_MODULE, + "resolve_effective_scenario_config", + )?, + &[ + ( + "project_config", + project.discovered.getattr("project_config")?, + ), + ("local_config", project.discovered.getattr("local_config")?), + ], + )? + .getattr("snapshot_limits")?; + let limits: SnapshotLimits = resolve_snapshot_limits( + SnapshotLimitInputs { + max_rows_per_relation: optional_int(&inputs, "max_snapshot_rows")?, + max_total_rows: optional_int(&inputs, "max_snapshot_total_rows")?, + max_bytes_per_relation: optional_int(&inputs, "max_snapshot_bytes")?, + max_total_bytes: optional_int(&inputs, "max_snapshot_total_bytes")?, + force: inputs.getattr("force")?.is_truthy()?, + }, + SnapshotLimitConfig { + max_rows_per_relation: optional_int(&configured, "max_rows_per_relation")?, + max_total_rows: optional_int(&configured, "max_total_rows")?, + max_bytes_per_relation: optional_int(&configured, "max_bytes_per_relation")?, + max_total_bytes: optional_int(&configured, "max_total_bytes")?, + }, + ); + call_with( + &imported(py, SCENARIO_MODELS, "ScenarioSnapshotCaptureLimits")?, + &[ + ( + "max_rows_per_relation", + int(py, limits.max_rows_per_relation), + ), + ("max_total_rows", int(py, limits.max_total_rows)), + ( + "max_bytes_per_relation", + int(py, limits.max_bytes_per_relation), + ), + ("max_total_bytes", int(py, limits.max_total_bytes)), + ("force", boolean(py, limits.force)), + ], + ) +} + +fn sqlbuild_version(py: Python<'_>) -> PyResult { + let metadata: Bound<'_, PyAny> = py.import(METADATA_MODULE)?.into_any(); + match metadata.getattr("version")?.call1((DISTRIBUTION_NAME,)) { + Ok(version) => version.extract(), + Err(error) if error.is_instance(py, &metadata.getattr("PackageNotFoundError")?) => { + Ok(UNKNOWN_VERSION.to_owned()) + } + Err(error) => Err(error), + } +} + +/// Capture settings with the current capture time and SQLBuild version. +pub(crate) fn capture_settings<'py>( + py: Python<'py>, + (capture_adapter, capture_dialect, retain): (&str, &str, bool), + limits: Bound<'py, PyAny>, +) -> PyResult> { + let epoch_seconds: i64 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + call_with( + &imported(py, SCENARIO_MODELS, "ScenarioCaptureSettings")?, + &[ + ("captured_at", text(py, &format_captured_at(epoch_seconds))), + ("capture_adapter", text(py, capture_adapter)), + ("capture_dialect", text(py, capture_dialect)), + ("sqlbuild_version", text(py, &sqlbuild_version(py)?)), + ("retain", boolean(py, retain)), + ("limits", limits), + ], + ) +} + +pub(crate) fn int(py: Python<'_>, value: i64) -> Bound<'_, PyAny> { + PyInt::new(py, value).into_any() +} + +pub(crate) fn boolean(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { + PyBool::new(py, value).to_owned().into_any() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs new file mode 100644 index 0000000000..c58c3c8270 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs @@ -0,0 +1,221 @@ +//! `sqb scenario test --local --sync-snapshots` and `--refresh`: capture stale snapshots first. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyTuple, PyTupleMethods}; +use sqlbuild_core::cli_text::main::format_summary_footer::format_summary_footer; +use sqlbuild_core::cli_text::models::SummaryCount; +use sqlbuild_core::scenario_command::main::snapshot_capture_warning::snapshot_capture_warning; +use sqlbuild_core::scenario_command::models::ScenarioNamespace; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::scenario_command::conversions::succeeded; +use crate::bindings::_helpers::scenario_command::presentation::{ + CompletionKind, SectionStatus, begin_section, connection_hooks, progress_relay, +}; +use crate::bindings::_helpers::scenario_command::setup::{ + CompileInputs, NATIVE_MODULE, boolean, capture_limits, capture_settings, compile, flush, + optional_text, project_connection_config, selected_scenarios, text, write, +}; +use crate::bindings::_helpers::scenario_command::test_command::TestRun; + +const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; + +/// The outcome of syncing local snapshots: its exit code and capture results. +pub(crate) struct LocalSync<'py> { + pub(crate) exit_code: i32, + results: Bound<'py, PyTuple>, + refresh: bool, +} + +impl LocalSync<'_> { + /// Write the snapshot capture JSON document and return the failing exit code. + pub(crate) fn write_failure_json( + &self, + request: &Bound<'_, PyAny>, + namespace: &ScenarioNamespace, + ) -> PyResult { + let py: Python<'_> = request.py(); + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_scenario_snapshot_execution_json")?, + &[ + ( + "run_namespace", + optional_text(py, namespace.value.as_deref()), + ), + ("namespace_source", text(py, &namespace.source)), + ("results", self.results.clone().into_any()), + ("refresh", boolean(py, self.refresh)), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(self.exit_code) + } +} + +fn label(prefix: &str, suffix: &str, value: usize) -> SummaryCount { + SummaryCount { + label: format!("{prefix}_{suffix}"), + value: i64::try_from(value).unwrap_or(i64::MAX), + } +} + +/// Capture the selected scenarios whose local snapshots are missing or stale. +pub(crate) fn sync_local_snapshots<'py>( + request: &Bound<'py, PyAny>, + run: &TestRun<'py>, + (local_pipeline, local_scenarios): (&Bound<'py, PyAny>, &Bound<'py, PyTuple>), +) -> PyResult> { + let py: Python<'py> = request.py(); + let refresh: bool = request.getattr("refresh")?.is_truthy()?; + let project_connection: Bound<'py, PyAny> = project_connection_config(&run.project)?; + let project_name: Bound<'py, PyAny> = run + .project + .discovered + .getattr("project_config")? + .getattr("name")?; + let names: Vec = call_with( + &imported( + py, + EXECUTOR_RUN_MODULE, + "select_scenario_snapshot_capture_candidates", + )?, + &[ + ("project_dir", run.project.project_dir.clone()), + ("pipeline_result", local_pipeline.clone()), + ("scenarios", local_scenarios.clone().into_any()), + ("adapter", run.adapter.clone()), + ("project_name", project_name.clone()), + ("capture_adapter", text(py, &run.project_adapter_name)), + ("capture_dialect", text(py, &run.capture_dialect)), + ("refresh", boolean(py, refresh)), + ( + "source_lexical_syntax", + run.project_adapter.getattr("sql_lexical_syntax")?, + ), + ], + )? + .extract()?; + if names.is_empty() { + write(&run.stream, "\nSnapshots are fresh.\n")?; + flush(&run.stream)?; + return Ok(LocalSync { + exit_code: 0, + results: PyTuple::empty(py), + refresh, + }); + } + let planning: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PlanningProgressReporter")?, + &[ + ("stream", run.stream.clone()), + ("use_color", boolean(py, run.style.use_color)), + ], + )?; + let compiled: PyResult> = compile( + &run.project, + CompileInputs { + adapter: &run.project_adapter, + options: &[ + ("no_sql_validation", request.getattr("no_sql_validation")?), + ("source_deferral_enabled", boolean(py, false)), + ("connection_config", project_connection.clone()), + ], + planning: &planning, + }, + ); + let finished: PyResult> = planning.call_method0("finish"); + let pipeline: Bound<'py, PyAny> = compiled?; + finished?; + let scenarios: Bound<'py, PyTuple> = + selected_scenarios(&pipeline, (&names, &[], &run.project.project_dir))?; + let force: bool = request + .getattr("limit_inputs")? + .getattr("force")? + .is_truthy()?; + let section: SectionStatus<'py> = begin_section( + &run.stream, + ( + &format!( + "Snapshot {} ({} selected)", + if refresh { "Refresh" } else { "Sync" }, + scenarios.len() + ), + &format!("{}\n\n", snapshot_capture_warning(force)), + "Capturing snapshots...".to_owned(), + ), + run.style, + )?; + let hooks: Bound<'py, PyAny> = + connection_hooks(&run.stream, (&run.project_adapter_name, run.style))?; + let relay = progress_relay( + &run.stream, + §ion, + ( + &run.project.project_dir, + &run.project_adapter, + CompletionKind::Sync { refresh }, + run.style, + ), + )?; + let settings: Bound<'py, PyAny> = capture_settings( + py, + (&run.project_adapter_name, &run.capture_dialect, false), + capture_limits(request, &run.project)?, + )?; + let results: Bound<'py, PyTuple> = PyTuple::new( + py, + call_with( + &imported(py, EXECUTOR_RUN_MODULE, "run_scenario_capture_pipeline")?, + &[ + ("project_dir", run.project.project_dir.clone()), + ("pipeline_result", pipeline), + ("scenarios", scenarios.into_any()), + ("connection_config", project_connection), + ("adapter", run.project_adapter.clone()), + ("project_name", project_name), + ("settings", settings), + ("connection_hooks", hooks), + ("on_scenario_start", relay.getattr("on_scenario_start")?), + ( + "on_scenario_complete", + relay.getattr("on_scenario_complete")?, + ), + ], + )? + .try_iter()? + .collect::>>()?, + )?; + section.status.call_method0("close")?; + let mut passed: usize = 0; + for result in results.iter() { + if succeeded(&result)? { + passed += 1; + } + } + let failed: usize = results.len() - passed; + let prefix: &str = if refresh { "REFRESH" } else { "SYNC" }; + write( + &run.stream, + &format!( + "\n{}\n", + format_summary_footer( + run.style, + &[label(prefix, "PASS", passed), label(prefix, "FAIL", failed)], + None + ) + ), + )?; + flush(&run.stream)?; + Ok(LocalSync { + exit_code: i32::from(failed != 0), + results, + refresh, + }) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs new file mode 100644 index 0000000000..004178139b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -0,0 +1,360 @@ +//! `sqb scenario test`: run the selected scenarios warehouse-direct or replay them locally. + +use std::path::PathBuf; +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::format_summary_footer::format_summary_footer; +use sqlbuild_core::cli_text::models::{CliStyle, SummaryCount}; +use sqlbuild_core::scenario_command::main::validate_scenario_test_flags::validate_scenario_test_flags; +use sqlbuild_core::scenario_command::models::{ + ScenarioNamespace, ScenarioTestFlags, SqlAnalysisCommand, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::scenario_command::conversions::succeeded; +use crate::bindings::_helpers::scenario_command::presentation::{ + CompletionKind, SectionStatus, activity, begin_section, connection_hooks, progress_relay, + write_completion, +}; +use crate::bindings::_helpers::scenario_command::setup::{ + CompileInputs, NATIVE_MODULE, ScenarioProject, boolean, cli_user_error, compile, + discover_project, flush, int, optional_text, project_adapter_name, project_connection_config, + require_capture_dialect, require_sql_analysis, resolve_adapter, selected_scenarios, style, + text, write, write_headers, +}; +use crate::bindings::_helpers::scenario_command::snapshot_sync::{LocalSync, sync_local_snapshots}; + +const EFFECTIVE_SETTINGS_MODULE: &str = "sqlbuild.compiler.compile.main.effective_settings"; +const EXECUTOR_RUN_MODULE: &str = "sqlbuild.executor.pipeline.main.run"; +const SCENARIO_MODELS: &str = "sqlbuild.executor.scenario.models"; +const LOCAL_ADAPTER: &str = "duckdb"; +const LOCAL_STATUSES: [&str; 4] = ["PASS", "FAIL", "ERROR", "SKIP"]; +const TARGET_DIR: &str = "target"; + +/// The resolved adapters, connection and output settings of one scenario test run. +pub(crate) struct TestRun<'py> { + pub(crate) project: ScenarioProject<'py>, + pub(crate) local: bool, + pub(crate) adapter_name: String, + pub(crate) adapter: Bound<'py, PyAny>, + pub(crate) project_adapter_name: String, + pub(crate) project_adapter: Bound<'py, PyAny>, + pub(crate) capture_dialect: String, + pub(crate) connection_config: Bound<'py, PyAny>, + pub(crate) stream: Bound<'py, PyAny>, + pub(crate) style: CliStyle, + pub(crate) concurrency: i64, +} + +fn flag(request: &Bound<'_, PyAny>, name: &str) -> PyResult { + request.getattr(name)?.is_truthy() +} + +fn prepare<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult> { + let local: bool = flag(request, "local")?; + validate_scenario_test_flags(ScenarioTestFlags { + local, + retain: flag(request, "retain")?, + sync_snapshots: flag(request, "sync_snapshots")?, + refresh: flag(request, "refresh")?, + }) + .map_err(|error| cli_user_error(py, error))?; + let project: ScenarioProject<'py> = discover_project(py, request)?; + if local { + require_sql_analysis( + &project, + ( + SqlAnalysisCommand::LocalTest, + flag(request, "no_sql_validation")?, + ), + )?; + } + let project_adapter_name: String = project_adapter_name(&project)?; + let adapter_name: String = if local { + LOCAL_ADAPTER.to_owned() + } else { + project_adapter_name.clone() + }; + let adapter: Bound<'py, PyAny> = resolve_adapter(&project, &adapter_name)?; + let project_adapter: Bound<'py, PyAny> = resolve_adapter(&project, &project_adapter_name)?; + let capture_dialect: String = if local { + require_capture_dialect(&project_adapter, &project_adapter_name)? + } else { + String::new() + }; + let connection_config: Bound<'py, PyAny> = if local { + let memory: Bound<'py, PyDict> = PyDict::new(py); + memory.set_item("database", ":memory:")?; + memory.into_any() + } else { + project_connection_config(&project)? + }; + let style: CliStyle = style(request)?; + let requested: Bound<'py, PyAny> = request.getattr("concurrency")?; + let concurrency: i64 = if requested.is_none() { + call_with( + &imported( + py, + EFFECTIVE_SETTINGS_MODULE, + "build_effective_settings_config", + )?, + &[("discovered_inputs", project.discovered.clone())], + )? + .getattr("concurrency")? + .extract()? + } else { + requested.extract()? + }; + let sys: Bound<'py, PyModule> = py.import("sys")?; + let stream: Bound<'py, PyAny> = sys.getattr(if flag(request, "json_output")? { + "stderr" + } else { + "stdout" + })?; + Ok(TestRun { + project, + local, + adapter_name, + adapter, + project_adapter_name, + project_adapter, + capture_dialect, + connection_config, + stream, + style, + concurrency: concurrency.max(1), + }) +} + +/// The run's elapsed summary, completion line and JSON document. +fn finish( + request: &Bound<'_, PyAny>, + run: &TestRun<'_>, + (section, started, results): (&SectionStatus<'_>, Instant, &Bound<'_, PyTuple>), + (counts, succeeded): (Vec, bool), +) -> PyResult<()> { + let py: Python<'_> = request.py(); + section.status.call_method0("close")?; + let elapsed_seconds: f64 = started.elapsed().as_secs_f64(); + write( + &run.stream, + &format!( + "\n{}\n", + format_summary_footer(run.style, &counts, Some(&format!("{elapsed_seconds:.2}s"))) + ), + )?; + flush(&run.stream)?; + write_completion(&run.stream, (&run.project.namespace, succeeded))?; + let namespace: &ScenarioNamespace = &run.project.namespace; + let payload: Bound<'_, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "format_scenario_execution_json")?, + &[ + ("results", results.clone().into_any()), + ("local", boolean(py, run.local)), + ( + "run_namespace", + optional_text(py, namespace.value.as_deref()), + ), + ("namespace_source", text(py, &namespace.source)), + ( + "duration_ms", + pyo3::types::PyFloat::new(py, elapsed_seconds * 1000.0).into_any(), + ), + ], + )?; + call_with( + &imported(py, NATIVE_MODULE, "write_execution_json_output")?, + &[ + ("payload", payload), + ("json_output", request.getattr("json_output")?), + ("json_output_path", request.getattr("json_output_path")?), + ], + )?; + Ok(()) +} + +fn count(label: &str, value: usize) -> SummaryCount { + SummaryCount { + label: label.to_owned(), + value: i64::try_from(value).unwrap_or(i64::MAX), + } +} + +/// Run the selected scenarios warehouse-direct, or replay them on run-scoped DuckDB. +fn run_scenarios( + request: &Bound<'_, PyAny>, + run: &TestRun<'_>, + (pipeline, scenarios): (&Bound<'_, PyAny>, &Bound<'_, PyTuple>), +) -> PyResult { + let py: Python<'_> = request.py(); + let section: SectionStatus<'_> = begin_section( + &run.stream, + ( + &format!("Scenario ({} selected)", scenarios.len()), + "", + activity("Running scenarios...", &run.project.namespace), + ), + run.style, + )?; + let started: Instant = Instant::now(); + let target_dir: PathBuf = run + .project + .project_dir + .extract::()? + .join(TARGET_DIR); + let relay = progress_relay( + &run.stream, + §ion, + ( + &run.project.project_dir, + &run.adapter, + CompletionKind::Run { target_dir }, + run.style, + ), + )?; + let project_name: Bound<'_, PyAny> = run + .project + .discovered + .getattr("project_config")? + .getattr("name")?; + let mut arguments: Vec<(&str, Bound<'_, PyAny>)> = vec![ + ("pipeline_result", pipeline.clone()), + ("scenarios", scenarios.clone().into_any()), + ("adapter", run.adapter.clone()), + ("project_name", project_name), + ("on_scenario_start", relay.getattr("on_scenario_start")?), + ( + "on_scenario_complete", + relay.getattr("on_scenario_complete")?, + ), + ]; + let runner: &str = if run.local { + let replay_source: Bound<'_, PyAny> = call_with( + &imported(py, SCENARIO_MODELS, "ScenarioLocalReplaySource")?, + &[ + ( + "lexical_syntax", + run.project_adapter.getattr("sql_lexical_syntax")?, + ), + ("capture_adapter", text(py, &run.project_adapter_name)), + ("capture_dialect", text(py, &run.capture_dialect)), + ], + )?; + arguments.extend([ + ("project_dir", run.project.project_dir.clone()), + ("strict", request.getattr("strict")?), + ("replay_source", replay_source), + ]); + "run_scenario_local_test_pipeline" + } else { + arguments.extend([ + ("connection_config", run.connection_config.clone()), + ("retain", request.getattr("retain")?), + ("concurrency", int(py, run.concurrency)), + ( + "connection_hooks", + connection_hooks(&run.stream, (&run.adapter_name, run.style))?, + ), + ]); + "run_scenario_test_pipeline" + }; + let results: Bound<'_, PyTuple> = PyTuple::new( + py, + call_with(&imported(py, EXECUTOR_RUN_MODULE, runner)?, &arguments)? + .try_iter()? + .collect::>>()?, + )?; + let total: usize = results.len(); + let (counts, ok): (Vec, bool) = if run.local { + let mut tallies: [usize; 4] = [0; 4]; + for result in results.iter() { + let status: Bound<'_, PyAny> = result.getattr("local_status")?; + if status.is_none() { + continue; + } + let value: String = status.getattr("value")?.extract()?; + if let Some(index) = LOCAL_STATUSES.iter().position(|label| *label == value) { + tallies[index] += 1; + } + } + let mut counts: Vec = LOCAL_STATUSES + .iter() + .zip(tallies) + .map(|(label, value)| count(label, value)) + .collect(); + counts.push(count("TOTAL", total)); + (counts, tallies[1] == 0 && tallies[2] == 0) + } else { + let mut passed: usize = 0; + for result in results.iter() { + if succeeded(&result)? { + passed += 1; + } + } + ( + vec![ + count("PASS", passed), + count("FAIL", total - passed), + count("TOTAL", total), + ], + passed == total, + ) + }; + finish(request, run, (§ion, started, &results), (counts, ok))?; + Ok(if ok { 0 } else { 1 }) +} + +/// Run one `sqb scenario test` request. +#[pyfunction] +#[pyo3(signature = (*, request))] +fn run_scenario_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { + let run: TestRun<'_> = prepare(py, request)?; + let planning: Bound<'_, PyAny> = write_headers( + &run.stream, + &run.project, + ( + if run.local { + "sqb scenario test --local" + } else { + "sqb scenario test" + }, + if run.local { 1 } else { run.concurrency }, + run.style, + ), + )?; + let no_sql_validation: bool = run.local || flag(request, "no_sql_validation")?; + let compiled: PyResult> = compile( + &run.project, + CompileInputs { + adapter: &run.adapter, + options: &[ + ("no_sql_validation", boolean(py, no_sql_validation)), + ("source_deferral_enabled", boolean(py, false)), + ("connection_config", run.connection_config.clone()), + ], + planning: &planning, + }, + ); + let finished: PyResult> = planning.call_method0("finish"); + let pipeline: Bound<'_, PyAny> = compiled?; + finished?; + let selectors: Vec = run.project.selectors.extract()?; + let exclude: Vec = run.project.exclude.extract()?; + let scenarios: Bound<'_, PyTuple> = + selected_scenarios(&pipeline, (&selectors, &exclude, &run.project.project_dir))?; + if run.local && (flag(request, "sync_snapshots")? || flag(request, "refresh")?) { + let synced: LocalSync<'_> = sync_local_snapshots(request, &run, (&pipeline, &scenarios))?; + if synced.exit_code != 0 { + return synced.write_failure_json(request, &run.project.namespace); + } + } + run_scenarios(request, &run, (&pipeline, &scenarios)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_scenario_test_command, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index 1b087534ea..61e4d53156 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -20,7 +20,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const SEED_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed"; const SEED_STATE_MODULE: &str = "sqlbuild.compiler.planner.main.commands._seed_state"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index fc9b4685a8..1a0a4d7d90 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -32,7 +32,7 @@ const NATIVE_MODULE: &str = "sqlbuild._native"; const COMMAND_MODELS: &str = "sqlbuild.cli.commands.models"; const INVOCATION_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.command_invocation"; const EXTERNAL_REFS_MODULE: &str = "sqlbuild.cli.commands._helpers.planning.external_refs"; -const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main.static_command"; +const STATIC_COMMAND_MODULE: &str = "sqlbuild.compiler.pipeline.main._static_command"; const PIPELINE_MODELS: &str = "sqlbuild.compiler.pipeline.models"; const TEST_PLAN_MODULE: &str = "sqlbuild.compiler.planner.main.commands._sql_test"; const PLANNER_ERRORS: &str = "sqlbuild.compiler.planner.exceptions"; diff --git a/crates/sqlbuild-rules/Cargo.toml b/crates/sqlbuild-rules/Cargo.toml index 29ac27d0b5..51945d206b 100644 --- a/crates/sqlbuild-rules/Cargo.toml +++ b/crates/sqlbuild-rules/Cargo.toml @@ -17,6 +17,7 @@ serde.workspace = true serde_json.workspace = true sha2.workspace = true sqlbuild-analysis.workspace = true +sqlbuild-attachments.workspace = true sqlbuild-core.workspace = true sqlbuild-sqltext.workspace = true stacker.workspace = true diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index a624e65ed6..359cfce2a2 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -203,7 +203,7 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann expected_snowflake_comma: "SELECT 'é)ü' ñ a FROM t", expected_compact: "a='é)ü'ñandb", expected_snowflake_compact: "a='é)ü'ñandb", - expected_lint_site: None, + expected_lint_site: Some("@m('é)ü' ñ x)"), }, ]; diff --git a/crates/sqlbuild-rules/src/lib.rs b/crates/sqlbuild-rules/src/lib.rs index 11d313a627..ae2bbcee68 100644 --- a/crates/sqlbuild-rules/src/lib.rs +++ b/crates/sqlbuild-rules/src/lib.rs @@ -8,6 +8,7 @@ pub mod custom_rule_cache; pub mod dsl_headers; pub mod engine; pub mod errors; +pub mod lint_text; pub mod models; pub mod rule_cache; pub mod rule_results; diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/fixtures.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/fixtures.rs new file mode 100644 index 0000000000..9500ba69a6 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/fixtures.rs @@ -0,0 +1,72 @@ +//! Fixture-null candidate file listing and matching. + +use std::path::{Component, Path, PathBuf}; +use std::sync::LazyLock; + +use regex::Regex; + +use crate::lint_text::_helpers::sites::compiled; +use crate::lint_text::constants::{ + FIXTURE_CTE_MARKERS, FIXTURE_TYPED_NULL_PATTERN, IGNORED_SEARCH_DIRECTORIES, + MODEL_FIXTURE_PATTERN, SQL_SUFFIX, +}; +use crate::lint_text::errors::LintTextError; + +static TYPED_NULL: LazyLock> = + LazyLock::new(|| Regex::new(FIXTURE_TYPED_NULL_PATTERN)); +static MODEL_FIXTURE: LazyLock> = + LazyLock::new(|| Regex::new(MODEL_FIXTURE_PATTERN)); + +/// Every `*.sql` file below `directory`, without following directory symlinks. +pub(crate) fn sql_files(directory: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(directory) else { + return Vec::new(); + }; + let mut files: Vec = Vec::new(); + for entry in entries.flatten() { + let path: PathBuf = entry.path(); + let is_directory: bool = entry.file_type().is_ok_and(|kind| kind.is_dir()); + if is_directory { + files.extend(sql_files(&path)); + } else if is_sql_file(&path) { + files.push(path); + } + } + files +} + +fn is_sql_file(path: &Path) -> bool { + path.file_name() + .is_some_and(|name| name.to_string_lossy().ends_with(SQL_SUFFIX)) + && path.is_file() +} + +/// Whether any path component is a directory the scan never searches. +pub(crate) fn in_ignored_directory(path: &Path) -> bool { + path.components().any(is_ignored_component) +} + +fn is_ignored_component(component: Component<'_>) -> bool { + match component { + Component::Normal(part) => IGNORED_SEARCH_DIRECTORIES.contains(&&*part.to_string_lossy()), + _ => false, + } +} + +/// The models a fixture file stands in for, or `None` when it holds no typed-null fixture. +pub(crate) fn fixture_models(contents: &str) -> Result>, LintTextError> { + let folded: String = contents.to_lowercase(); + if !FIXTURE_CTE_MARKERS + .iter() + .any(|marker| folded.contains(marker)) + || !compiled(&TYPED_NULL)?.is_match(contents) + { + return Ok(None); + } + Ok(Some( + compiled(&MODEL_FIXTURE)? + .captures_iter(contents) + .map(|captures| captures["name"].to_string()) + .collect(), + )) +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs new file mode 100644 index 0000000000..21082ff7d9 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/format_bodies.rs @@ -0,0 +1,250 @@ +//! Body preparation, request building and result splicing for SQL body formatting. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::LazyLock; + +use regex::Regex; +use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; +use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::dsl_headers::main::measurement_body_ranges::measurement_body_ranges; +use crate::dsl_headers::main::scan_headers::scan_headers; +use crate::dsl_headers::main::sql_body_ranges::sql_body_ranges; +use crate::dsl_headers::models::HeaderSpan; +use crate::lint_text::_helpers::sites::compiled; +use crate::lint_text::constants::{ + DECLARATION_GROUP_DIRECTORY, DECLINED_WITHOUT_REASON, FIXTURE_HEADER_KINDS, FORMAT_API_VERSION, + GLOBAL_NAMED_ROOTS, GROUPED_NAMED_ROLES, LINE_COMMENT_PREFIX, MAX_FUNCTION_CALL_DEPTH, + MOVED_OMITTED_SELECT, OMITTED_CEREMONIAL_SELECT_SQL, OMITTED_SELECT_SUFFIX_PATTERN, +}; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use crate::lint_text::main::restore_interpolation::restore_interpolation; +use crate::lint_text::models::{FormatFault, FormatFile, InterpolationSite, PreparedBody}; +use crate::sql_lint::models::{FormatRequest, FormatResponse}; + +static OMITTED_SELECT_SUFFIX: LazyLock> = + LazyLock::new(|| Regex::new(OMITTED_SELECT_SUFFIX_PATTERN)); + +/// Deduplicated format requests, keyed by neutralized SQL and sentinel widths. +#[derive(Default)] +pub(crate) struct RequestTable { + pub(crate) requests: Vec, + index: HashMap<(String, Vec<(String, usize)>), usize>, +} + +impl RequestTable { + pub(crate) fn insert( + &mut self, + sql: String, + widths: BTreeMap, + options: (&str, usize), + ) -> usize { + let key: (String, Vec<(String, usize)>) = ( + sql.clone(), + widths + .iter() + .map(|(name, width)| (name.clone(), *width)) + .collect(), + ); + if let Some(index) = self.index.get(&key) { + return *index; + } + self.requests.push(FormatRequest { + version: FORMAT_API_VERSION, + sql, + dialect: options.0.to_string(), + max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + line_width: Some(options.1), + token_widths: widths.into_iter().collect(), + }); + let _ = self.index.insert(key, self.requests.len() - 1); + self.requests.len() - 1 + } +} + +/// Whether a project-relative path lies in a generic or singular audit role. +fn is_audit_path(parts: &[String]) -> bool { + let directories: &[String] = &parts[..parts.len().saturating_sub(1)]; + for (root, audit) in GLOBAL_NAMED_ROOTS { + if starts_with(directories, root) { + return audit; + } + } + for (index, part) in directories.iter().enumerate() { + if part != DECLARATION_GROUP_DIRECTORY { + continue; + } + for (role, audit) in GROUPED_NAMED_ROLES { + if starts_with(&directories[index + 1..], role) { + return audit; + } + } + } + false +} + +fn starts_with(parts: &[String], prefix: &[&str]) -> bool { + parts.len() >= prefix.len() && parts.iter().zip(prefix).all(|(part, name)| part == name) +} + +/// The lintable body ranges of one file, measurement blocks first for audits. +fn body_ranges(file: &FormatFile, text: &[char], headers: &[HeaderSpan]) -> Vec<(usize, usize)> { + if file.relative_parts.as_deref().is_some_and(is_audit_path) { + let ranges: Vec<(usize, usize)> = measurement_body_ranges(text); + if !ranges.is_empty() { + return ranges; + } + } + sql_body_ranges(text, headers) +} + +/// One prepared body with its neutralized SQL and sentinel widths, awaiting its request index. +pub(crate) type PendingBody = (PreparedBody, String, BTreeMap); + +/// Prepare every body of one file for formatting. +pub(crate) fn prepare_file( + file: &FormatFile, + dialect: &str, +) -> Result, LintTextError> { + let text: Vec = file.contents.chars().collect(); + let headers: Vec = scan_headers(&text, false); + let omits: bool = headers + .iter() + .any(|header| FIXTURE_HEADER_KINDS.contains(&header.kind.as_str())); + let syntax: LexicalSyntax = LexicalSyntax { + line_comment_prefixes: vec![LINE_COMMENT_PREFIX.to_string()], + ..LexicalSyntax::default() + }; + let mut prepared: Vec = Vec::new(); + for (start, end) in body_ranges(file, &text, &headers) { + let body: &[char] = &text[start..end]; + let core_end: usize = body + .iter() + .rposition(|character| !python_isspace(*character)) + .map_or(0, |index| index + 1); + let mut core: String = body[..core_end].iter().collect(); + let mut trailing: String = body[core_end..].iter().collect(); + let omitted: Option = if omits { + match omitted_select_offset(&core, &syntax) { + OmittedSelect::At(offset) => Some(offset), + OmittedSelect::Absent => None, + } + } else { + None + }; + if let Some(offset) = omitted { + let characters: Vec = core.chars().collect(); + trailing = characters[offset..].iter().collect::() + &trailing; + core = characters[..offset].iter().collect(); + } + let neutralized = neutralize_interpolation(&core, dialect, true)?; + let mut sql: String = neutralized.text; + if omitted.is_some() { + sql.push_str(OMITTED_CEREMONIAL_SELECT_SQL); + } + let widths: BTreeMap = sentinel_widths(&neutralized.sites); + let body: PreparedBody = PreparedBody { + start, + end, + trailing, + request: 0, + sites: neutralized.sites, + omits_select: omitted.is_some(), + }; + prepared.push((body, sql, widths)); + } + Ok(prepared) +} + +/// The printed width of the SQLBuild syntax each sentinel stands in for, where it differs. +fn sentinel_widths(sites: &[InterpolationSite]) -> BTreeMap { + sites + .iter() + .filter(|site| { + site.original_text.chars().count() != site.sentinel.chars().count() + || site.original_text.contains('\n') + }) + .map(|site| { + ( + site.sentinel.to_uppercase(), + printed_width(&site.original_text), + ) + }) + .collect() +} + +/// The widest line of `text`, in characters. +fn printed_width(text: &str) -> usize { + let lines: Vec<&str> = python_splitlines(text); + lines + .iter() + .map(|line| line.chars().count()) + .max() + .unwrap_or(0) +} + +/// The formatted text replacing one body, or why the body was left unchanged. +pub(crate) fn formatted_body( + prepared: &PreparedBody, + response: &FormatResponse, +) -> Result, String> { + if !response.formatted { + return Err(response + .reason + .clone() + .filter(|reason| !reason.is_empty()) + .unwrap_or_else(|| DECLINED_WITHOUT_REASON.to_string())); + } + let mut sql: &str = &response.sql; + let mut changed: bool = response.changed; + if prepared.omits_select { + let pattern: &Regex = + compiled(&OMITTED_SELECT_SUFFIX).map_err(|error| error.to_string())?; + let Some(suffix) = pattern.find(sql) else { + return Err(MOVED_OMITTED_SELECT.to_string()); + }; + sql = &sql[..suffix.start()]; + changed = true; + } + if !changed { + return Ok(None); + } + restore_interpolation(sql, &prepared.sites) + .map(|restored| Some(restored + &prepared.trailing)) + .map_err(|error| error.to_string()) +} + +/// One file's text with every formatted body spliced in, if it changed, and its declined bodies. +pub(crate) fn splice_file( + index: usize, + contents: &str, + bodies: &[PreparedBody], + responses: &[FormatResponse], +) -> (Option, Vec) { + let original: Vec = contents.chars().collect(); + let mut updated: Vec = original.clone(); + let mut faults: Vec = Vec::new(); + for body in bodies.iter().rev() { + match formatted_body(body, &responses[body.request]) { + Ok(Some(replacement)) => { + let _ = updated.splice(body.start..body.end, replacement.chars()); + } + Ok(None) => {} + Err(reason) => faults.push(FormatFault { + file: index, + line: original[..body.start] + .iter() + .filter(|character| **character == '\n') + .count() + + 1, + reason, + }), + } + } + let formatted: Option = (updated != original).then(|| updated.into_iter().collect()); + (formatted, faults) +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/lines.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/lines.rs new file mode 100644 index 0000000000..979b3f26e5 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/lines.rs @@ -0,0 +1,59 @@ +//! Python `str.splitlines(keepends=True)` lines with character offsets. + +/// One line: its text without the ending, and its character range including the ending. +pub(crate) struct LineSpan<'a> { + pub(crate) content: &'a str, + pub(crate) start: usize, + pub(crate) end: usize, +} + +fn is_line_boundary(character: char) -> bool { + matches!( + character, + '\n' | '\r' + | '\u{0b}' + | '\u{0c}' + | '\u{1c}' + | '\u{1d}' + | '\u{1e}' + | '\u{85}' + | '\u{2028}' + | '\u{2029}' + ) +} + +/// Lines as Python's `splitlines` finds them; `\r\n` ends one line. +pub(crate) fn line_spans(text: &str) -> Vec> { + let mut spans: Vec> = Vec::new(); + let mut byte_start: usize = 0; + let mut char_start: usize = 0; + let mut position: usize = 0; + let mut characters = text.char_indices().peekable(); + while let Some((index, character)) = characters.next() { + position += 1; + if !is_line_boundary(character) { + continue; + } + let content: &str = &text[byte_start..index]; + byte_start = index + character.len_utf8(); + if character == '\r' && characters.peek().is_some_and(|(_, next)| *next == '\n') { + let _ = characters.next(); + byte_start += 1; + position += 1; + } + spans.push(LineSpan { + content, + start: char_start, + end: position, + }); + char_start = position; + } + if byte_start < text.len() { + spans.push(LineSpan { + content: &text[byte_start..], + start: char_start, + end: position, + }); + } + spans +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs new file mode 100644 index 0000000000..5726c1c9b0 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod fixtures; +pub(crate) mod format_bodies; +pub(crate) mod lines; +pub(crate) mod settings; +pub(crate) mod sites; +pub(crate) mod sql_diagnostics; +pub(crate) mod sql_requests; +pub(crate) mod suppressions; diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/settings.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/settings.rs new file mode 100644 index 0000000000..d7a061599b --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/settings.rs @@ -0,0 +1,37 @@ +//! Read format settings from parsed configuration tables. + +use std::fs; +use std::path::Path; + +use crate::lint_text::constants::ADAPTER_DIALECT_TRANSLATIONS; + +/// The parsed TOML file at `path`, or `None` when it is not a file. +pub(crate) fn read_table(path: &Path) -> Result, String> { + if !path.is_file() { + return Ok(None); + } + let text: String = fs::read_to_string(path).map_err(|error| error.to_string())?; + let text: &str = text.strip_prefix('\u{feff}').unwrap_or(&text); + toml::from_str(text) + .map(Some) + .map_err(|error| error.to_string()) +} + +/// The dialect a configured adapter name selects. +pub(crate) fn adapter_dialect(table: &toml::Table) -> Option { + let adapter: &str = table.get("adapter")?.as_str()?; + let dialect: &str = ADAPTER_DIALECT_TRANSLATIONS + .iter() + .find(|(name, _)| *name == adapter) + .map_or(adapter, |(_, dialect)| dialect); + Some(dialect.to_owned()) +} + +/// A positive integer `key` of `section`, or `current`. +pub(crate) fn positive_int(section: Option<&toml::Table>, key: &str, current: i64) -> i64 { + section + .and_then(|section| section.get(key)) + .and_then(toml::Value::as_integer) + .filter(|value| *value > 0) + .unwrap_or(current) +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs new file mode 100644 index 0000000000..3911e70e3e --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sites.rs @@ -0,0 +1,197 @@ +//! Interpolation site scanning and sentinel assembly over byte offsets. + +use std::sync::LazyLock; + +use regex::Regex; +use sqlbuild_core::text::main::python_isalnum::python_isalnum; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::models::QuotePolicy; + +use crate::lint_text::constants::{ + FORMAT_SITE_PATTERN, FUNCTION_NAMES, LINT_SITE_PATTERN, SENTINEL_SUFFIX, +}; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::{InterpolationSite, Neutralized}; + +static LINT_SITE: LazyLock> = + LazyLock::new(|| Regex::new(LINT_SITE_PATTERN)); +static FORMAT_SITE: LazyLock> = + LazyLock::new(|| Regex::new(FORMAT_SITE_PATTERN)); + +/// The site-start pattern for lint, or the case-insensitive one for formatting. +pub(crate) fn site_pattern(formatting: bool) -> Result<&'static Regex, LintTextError> { + let pattern: &'static LazyLock> = + if formatting { &FORMAT_SITE } else { &LINT_SITE }; + compiled(pattern) +} + +/// A lazily compiled built-in pattern, or its compile error. +pub(crate) fn compiled( + pattern: &'static LazyLock>, +) -> Result<&'static Regex, LintTextError> { + pattern + .as_ref() + .map_err(|error| LintTextError::Pattern(error.to_string())) +} + +/// One replaced byte range and the sentinel that replaces it. +pub(crate) type ByteSite = (usize, usize, String); + +/// The lexical policy lint and format scan authored SQL with. +pub(crate) fn lint_policy(backtick_identifiers: bool) -> QuotePolicy { + QuotePolicy::SQL_LINT.with_backtick_identifiers(backtick_identifiers) +} + +/// Offsets of candidate sites outside comments and quoted text, as Python's `finditer` yields them. +pub(crate) fn site_starts(text: &str, pattern: &Regex, policy: QuotePolicy) -> Vec { + let bytes: &[u8] = text.as_bytes(); + let mut starts: Vec = Vec::new(); + let mut index: usize = 0; + while index < bytes.len() { + match non_code_end(bytes, index, policy) { + Ok(Some(end)) => { + index = end; + continue; + } + Err(_) => break, + Ok(None) => {} + } + if let Some(found) = text.get(index..).and_then(|rest| pattern.find(rest)) { + starts.push(index); + index += found.end(); + } else { + index += 1; + } + } + starts +} + +/// The end of the interpolation site starting at `start`, if one starts there. +pub(crate) fn site_end(text: &str, start: usize, policy: QuotePolicy) -> Option { + let bytes: &[u8] = text.as_bytes(); + match *bytes.get(start)? { + b'@' if bytes.get(start + 1) == Some(&b'@') => Some(name_end(text, start + 2, true)), + b'@' => macro_site_end(text, start, policy), + b'$' => bytes + .get(start + 2..)? + .iter() + .position(|byte| *byte == b'}') + .map(|index| start + 2 + index + 1), + b'_' => { + let end: usize = name_end(text, start, false); + let name: String = text[start..end].to_lowercase(); + if !FUNCTION_NAMES.contains(&name.as_str()) { + return None; + } + let open: usize = skip_whitespace(text, end); + (bytes.get(open) == Some(&b'(')) + .then(|| paren_end(bytes, open, policy)) + .flatten() + } + _ => None, + } +} + +fn macro_site_end(text: &str, start: usize, policy: QuotePolicy) -> Option { + let name_start: usize = start + 1; + let first: char = text.get(name_start..)?.chars().next()?; + if first == '\'' { + return text[name_start + 1..] + .find('\'') + .map(|index| name_start + 1 + index + 1); + } + if !(first.is_alphabetic() || first == '_') { + return None; + } + let end: usize = name_end(text, name_start, false); + if text.as_bytes().get(end) == Some(&b'(') { + paren_end(text.as_bytes(), end, policy) + } else { + Some(end) + } +} + +/// The end of a name: alphanumerics and `_`, plus `:` and `.` inside `@@` interpolation. +pub(crate) fn name_end(text: &str, start: usize, interpolation: bool) -> usize { + let mut end: usize = start; + for character in text[start..].chars() { + if python_isalnum(character) + || character == '_' + || interpolation && matches!(character, ':' | '.') + { + end += character.len_utf8(); + } else { + break; + } + } + end +} + +fn skip_whitespace(text: &str, start: usize) -> usize { + let mut end: usize = start; + for character in text[start..].chars() { + if !python_isspace(character) { + break; + } + end += character.len_utf8(); + } + end +} + +fn paren_end(bytes: &[u8], open: usize, policy: QuotePolicy) -> Option { + match matching_paren(bytes, open, policy) { + Ok(close) => Some(close + 1), + Err(_) => None, + } +} + +/// A numbered sentinel such as `__sqb_lint_0__`. +pub(crate) fn sentinel(prefix: &str, index: usize) -> String { + format!("{prefix}{index}{SENTINEL_SUFFIX}") +} + +/// Replace ascending, non-overlapping byte ranges by their sentinels, with character offsets. +pub(crate) fn assemble(body: &str, byte_sites: Vec) -> Neutralized { + let mut text: String = String::with_capacity(body.len()); + let mut sites: Vec = Vec::with_capacity(byte_sites.len()); + let mut copied_to: usize = 0; + let mut original_chars: usize = 0; + let mut output_chars: usize = 0; + for (start, end, sentinel) in byte_sites { + let literal: &str = &body[copied_to..start]; + let literal_chars: usize = literal.chars().count(); + text.push_str(literal); + original_chars += literal_chars; + output_chars += literal_chars; + let original: &str = &body[start..end]; + let original_len: usize = original.chars().count(); + let sentinel_len: usize = sentinel.chars().count(); + text.push_str(&sentinel); + sites.push(InterpolationSite { + sentinel, + neutralized_start: output_chars, + neutralized_end: output_chars + sentinel_len, + original_start: original_chars, + original_end: original_chars + original_len, + original_text: original.to_string(), + }); + original_chars += original_len; + output_chars += sentinel_len; + copied_to = end; + } + text.push_str(&body[copied_to..]); + Neutralized { text, sites } +} + +/// The byte offset of character position `position`, or `None` past the end. +pub(crate) fn byte_offset(text: &str, position: usize) -> Option { + if text.is_ascii() { + return (position <= text.len()).then_some(position); + } + text.char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(text.len())) + .nth(position) +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs new file mode 100644 index 0000000000..9599fb1e2b --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -0,0 +1,375 @@ +//! Native SQL lint responses mapped back onto authored source through expansion passes. + +use std::sync::LazyLock; + +use regex::Regex; +use serde_json::{Map, Value}; + +use crate::lint_text::_helpers::sites::compiled; +use crate::lint_text::constants::{ + GENERATED_FIX, GENERATED_SQL_SUFFIX, HARNESS_CTE_PREFIXES, INVALID_DIAGNOSTIC, + INVALID_FIX_EDIT, INVALID_FIX_OBJECT, INVALID_FIX_REASON, LINT_API_VERSION, LONG_LITERAL_CODE, + MISSING_DIAGNOSTICS, NATIVE_LINT_ERROR_CODE, NON_CONTIGUOUS_FIX, NON_OBJECT_RESPONSE, + NOT_AN_OBJECT, PARSE_ERROR_POSITION_PATTERN, PARSE_FAILURE_PREFIX, PARSE_FAILURE_REMEDIATION, + UNSUPPORTED_VERSION, UNUSED_CTE_CODE, +}; +use crate::lint_text::main::interpolation_text_at::interpolation_text_at; +use crate::lint_text::models::{AuthoredDiagnostic, AuthoredEdit, ExpansionSpan, LintSqlBody}; + +static PARSE_ERROR_POSITION: LazyLock> = + LazyLock::new(|| Regex::new(PARSE_ERROR_POSITION_PATTERN)); + +/// One file's text as characters and line starts, and the dialect tokens are read in. +pub(crate) struct AuthoredFile<'a> { + pub(crate) text: &'a str, + pub(crate) characters: Vec, + pub(crate) line_starts: Vec, + pub(crate) dialect: &'a str, +} + +impl<'a> AuthoredFile<'a> { + pub(crate) fn new(text: &'a str, dialect: &'a str) -> Self { + let characters: Vec = text.chars().collect(); + let mut line_starts: Vec = vec![0]; + line_starts.extend( + characters + .iter() + .enumerate() + .filter(|(_, character)| **character == '\n') + .map(|(index, _)| index as i64 + 1), + ); + Self { + text, + characters, + line_starts, + dialect, + } + } + + /// One-based line and column of a character offset. + fn position(&self, offset: i64) -> (i64, i64) { + let index: usize = self + .line_starts + .partition_point(|start| *start <= offset) + .saturating_sub(1); + (index as i64 + 1, offset - self.line_starts[index] + 1) + } + + fn token_at(&self, offset: i64) -> Option { + match usize::try_from(offset) { + Ok(start) => interpolation_text_at(self.text, start, self.dialect), + Err(_) => None, + } + } + + fn starts_with_harness_cte(&self, offset: i64) -> bool { + let Ok(start) = usize::try_from(offset) else { + return false; + }; + let rest: &[char] = self.characters.get(start..).unwrap_or_default(); + HARNESS_CTE_PREFIXES + .iter() + .any(|prefix| has_prefix(rest, prefix)) + } +} + +fn has_prefix(rest: &[char], prefix: &str) -> bool { + let expected: Vec = prefix.chars().collect(); + rest.starts_with(&expected) +} + +/// Map an expanded offset through one pass: the authored offset, and whether it was generated. +fn map_pass(offset: i64, spans: &[ExpansionSpan]) -> (i64, bool) { + let mut mapped: i64 = offset; + for span in spans { + if offset < span.output_start { + break; + } + if offset < span.output_end { + return (span.source_start, true); + } + mapped += (span.source_end - span.source_start) - (span.output_end - span.output_start); + } + (mapped, false) +} + +/// Map an expanded offset back through every pass, last pass first. +pub(crate) fn map_offset(offset: i64, passes: &[Vec]) -> (i64, bool) { + let mut current: i64 = offset; + let mut generated: bool = false; + for spans in passes.iter().rev() { + let (mapped, inside) = map_pass(current, spans); + current = mapped; + generated = generated || inside; + } + (current, generated) +} + +/// The diagnostics list of one engine response, or the error it reports. +pub(crate) fn response_diagnostics(response: &Value) -> Result, String> { + let Some(payload) = response.as_object() else { + return Err(NON_OBJECT_RESPONSE.to_string()); + }; + if let Some(Value::String(message)) = payload.get("Err") { + return Err(message.clone()); + } + let Some(result) = payload.get("Ok").and_then(Value::as_object) else { + return Err(NON_OBJECT_RESPONSE.to_string()); + }; + if integer(result.get("version")) != Some(LINT_API_VERSION) { + return Err(UNSUPPORTED_VERSION.to_string()); + } + match result.get("diagnostics") { + Some(Value::Array(diagnostics)) => Ok(diagnostics.clone()), + _ => Err(MISSING_DIAGNOSTICS.to_string()), + } +} + +/// A JSON integer, excluding floats and booleans. +fn integer(value: Option<&Value>) -> Option { + match value { + Some(Value::Number(number)) if number.is_i64() || number.is_u64() => number.as_i64(), + _ => None, + } +} + +/// Keep diagnostics before an added `SELECT 1`, clamped to end there. +pub(crate) fn authored_prefix(diagnostics: Vec, offset: i64) -> Vec { + let mut kept: Vec = Vec::new(); + for raw in diagnostics { + let Value::Object(mut diagnostic) = raw else { + kept.push(raw); + continue; + }; + if integer(diagnostic.get("start")).is_some_and(|start| start >= offset) { + continue; + } + if integer(diagnostic.get("end")).is_some_and(|end| end > offset) { + let _ = diagnostic.insert("end".to_string(), Value::from(offset)); + } + let fix_end: Option = diagnostic + .get("fix") + .and_then(Value::as_object) + .and_then(|fix| integer(fix.get("end"))); + if fix_end.is_some_and(|end| end > offset) { + let _ = diagnostic.insert("fix".to_string(), Value::Null); + let _ = diagnostic.insert( + "fix_unavailable_reason".to_string(), + Value::from(GENERATED_FIX), + ); + } + kept.push(Value::Object(diagnostic)); + } + kept +} + +/// The authored fault for a body the native parser rejected. +pub(crate) fn parse_failure( + message: &str, + body: &LintSqlBody, + file: &AuthoredFile<'_>, +) -> AuthoredDiagnostic { + let (line, column): (i64, i64) = parser_position(message); + let lines: Vec = keepends_lengths(&body.lint_text); + let mut expanded: i64 = 0; + if line >= 1 && line as usize <= lines.len() { + let index: usize = line as usize - 1; + expanded = lines[..index].iter().sum::() + (column - 1).min(lines[index]); + } + let (mapped, _) = map_offset(expanded, &body.passes); + let (line, column) = file.position(body.body_start + mapped); + AuthoredDiagnostic { + file: body.file, + line, + column, + code: NATIVE_LINT_ERROR_CODE.to_string(), + message: format!("{PARSE_FAILURE_PREFIX}{message}"), + fault: true, + end: None, + remediation: PARSE_FAILURE_REMEDIATION.to_string(), + fix: None, + fix_unavailable_reason: None, + } +} + +/// The one-based line and column a parser error names, or `(1, 1)`. +fn parser_position(message: &str) -> (i64, i64) { + let Ok(pattern) = compiled(&PARSE_ERROR_POSITION) else { + return (1, 1); + }; + let Some(captures) = pattern.captures(message) else { + return (1, 1); + }; + match ( + captures["line"].parse::(), + captures["column"].parse::(), + ) { + (Ok(line), Ok(column)) => (line, column), + _ => (1, 1), + } +} + +/// Character lengths of Python's `splitlines(keepends=True)` lines. +fn keepends_lengths(text: &str) -> Vec { + crate::lint_text::_helpers::lines::line_spans(text) + .iter() + .map(|span| (span.end - span.start) as i64) + .collect() +} + +/// The fields of one diagnostic: code, message, remediation, start and end. +fn diagnostic_fields( + diagnostic: &Map, + body_length: i64, +) -> Result<(String, String, String, i64, i64), String> { + let text = |key: &str| { + diagnostic + .get(key) + .and_then(Value::as_str) + .map(str::to_string) + }; + let (Some(code), Some(message), Some(remediation)) = + (text("code"), text("message"), text("remediation")) + else { + return Err(INVALID_DIAGNOSTIC.to_string()); + }; + let (Some(start), Some(end)) = ( + integer(diagnostic.get("start")), + integer(diagnostic.get("end")), + ) else { + return Err(INVALID_DIAGNOSTIC.to_string()); + }; + if start < 0 || end < start || end > body_length { + return Err(INVALID_DIAGNOSTIC.to_string()); + } + Ok((code, message, remediation, start, end)) +} + +/// Map one native diagnostic onto authored source, or drop it when it belongs to generated SQL. +pub(crate) fn authored_diagnostic( + raw: &Value, + body: &LintSqlBody, + file: &AuthoredFile<'_>, +) -> Result, String> { + let Some(diagnostic) = raw.as_object() else { + return Err(NOT_AN_OBJECT.to_string()); + }; + let body_length: i64 = body.lint_text.chars().count() as i64; + let (code, message, remediation, start, end) = diagnostic_fields(diagnostic, body_length)?; + let reason: Option = match diagnostic.get("fix_unavailable_reason") { + None | Some(Value::Null) => None, + Some(Value::String(reason)) => Some(reason.clone()), + Some(_) => return Err(INVALID_FIX_REASON.to_string()), + }; + let (mapped, generated) = map_offset(start, &body.passes); + let absolute: i64 = body.body_start + mapped; + if code == LONG_LITERAL_CODE && generated + || code == UNUSED_CTE_CODE && file.starts_with_harness_cte(absolute) + { + return Ok(None); + } + let (line, column) = file.position(absolute); + let end_position: Option<(i64, i64)> = + authored_end((start, end), (mapped, generated), body, file); + let raw_fix: Option<&Value> = diagnostic.get("fix").filter(|fix| !fix.is_null()); + let fix: Option = match raw_fix { + Some(raw_fix) => authored_fix(raw_fix, body_length, body)?, + None => None, + }; + let fix_unavailable_reason: Option = if raw_fix.is_some() && fix.is_none() { + Some(NON_CONTIGUOUS_FIX.to_string()) + } else { + reason + }; + let message: String = if generated { + match file.token_at(absolute) { + Some(token) => format!("{message} (in SQL generated by {token})"), + None => format!("{message} {GENERATED_SQL_SUFFIX}"), + } + } else { + message + }; + Ok(Some(AuthoredDiagnostic { + file: body.file, + line, + column, + code, + message, + fault: false, + end: end_position, + remediation, + fix, + fix_unavailable_reason, + })) +} + +/// The authored exclusive end position, only when authored continuity is provable. +fn authored_end( + (start, end): (i64, i64), + mapped_start: (i64, bool), + body: &LintSqlBody, + file: &AuthoredFile<'_>, +) -> Option<(i64, i64)> { + let absolute: i64 = body.body_start + mapped_start.0; + if mapped_start.1 { + let token: String = file.token_at(absolute)?; + return Some(file.position(absolute + token.chars().count() as i64)); + } + if end <= start { + return None; + } + let (last, generated) = map_offset(end - 1, &body.passes); + if generated || last - mapped_start.0 != end - start - 1 { + return None; + } + Some(file.position(body.body_start + last + 1)) +} + +/// Map a native edit only when both boundaries remain contiguous authored source. +fn authored_fix( + raw_fix: &Value, + body_length: i64, + body: &LintSqlBody, +) -> Result, String> { + let Some(payload) = raw_fix.as_object() else { + return Err(INVALID_FIX_OBJECT.to_string()); + }; + let (Some(start), Some(end), Some(replacement)) = ( + integer(payload.get("start")), + integer(payload.get("end")), + payload.get("replacement").and_then(Value::as_str), + ) else { + return Err(INVALID_FIX_EDIT.to_string()); + }; + if start < 0 || end <= start || end > body_length { + return Err(INVALID_FIX_EDIT.to_string()); + } + let (mut region_start, mut region_end): (i64, i64) = (start, end); + for spans in body.passes.iter().rev() { + if spans + .iter() + .any(|span| overlaps(span, region_start, region_end)) + { + return Ok(None); + } + region_start = map_pass(region_start, spans).0; + region_end = map_pass(region_end, spans).0; + } + let (first, first_generated) = map_offset(start, &body.passes); + let (last, last_generated) = map_offset(end - 1, &body.passes); + if first_generated || last_generated || last - first != end - start - 1 { + return Ok(None); + } + Ok(Some(AuthoredEdit { + start: body.body_start + first, + end: body.body_start + last + 1, + replacement: replacement.to_string(), + })) +} + +fn overlaps(span: &ExpansionSpan, start: i64, end: i64) -> bool { + start < span.output_end && span.output_start < end + || span.output_start == span.output_end + && start <= span.output_start + && span.output_start < end +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs new file mode 100644 index 0000000000..b5909ebcde --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -0,0 +1,111 @@ +//! Native SQL lint request payloads and the batch engine call. + +use serde_json::{Map, Value}; +use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitted_select_offset; +use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; + +use crate::lint_text::constants::{ + INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, OMITTED_CEREMONIAL_SELECT_SQL, +}; +use crate::lint_text::models::{LintSqlBody, LintSqlOptions}; +use crate::sql_lint::main::batch_engine::lint_batch_json; + +/// Where a fixture body omits its ceremonial `SELECT 1`, so the parser sees one statement. +pub(crate) fn omitted_select(body: &LintSqlBody) -> Option { + if !body.allows_ceremonial_select { + return None; + } + let syntax: LexicalSyntax = LexicalSyntax { + line_comment_prefixes: vec![LINE_COMMENT_PREFIX.to_string()], + ..LexicalSyntax::default() + }; + match omitted_select_offset(&body.lint_text, &syntax) { + OmittedSelect::At(offset) => Some(offset), + OmittedSelect::Absent => None, + } +} + +fn strings(values: &[String]) -> Value { + Value::from(values.to_vec()) +} + +/// The engine request for one body, with the omitted `SELECT 1` added back. +pub(crate) fn lint_payload( + body: &LintSqlBody, + omitted: Option, + options: &LintSqlOptions, +) -> Value { + let sql: String = match omitted { + Some(offset) => { + let characters: Vec = body.lint_text.chars().collect(); + let head: String = characters[..offset].iter().collect(); + let tail: String = characters[offset..].iter().collect(); + format!("{head}{OMITTED_CEREMONIAL_SELECT_SQL}{tail}") + } + None => body.lint_text.clone(), + }; + let mut payload: Map = Map::new(); + let _ = payload.insert("version".to_string(), Value::from(LINT_API_VERSION)); + let _ = payload.insert("sql".to_string(), Value::from(sql)); + let _ = payload.insert("dialect".to_string(), Value::from(options.dialect.clone())); + if let Some(enabled) = &options.enabled_rules { + let _ = payload.insert("enabled_rules".to_string(), strings(enabled)); + } + let lists: [(&str, &[String]); 4] = [ + ("ignored_rules", &options.ignored_rules), + ("external_identifiers", &body.external_identifiers), + ("dependency_identifiers", &body.dependency_identifiers), + ( + "externally_referenced_ctes", + &body.externally_referenced_ctes, + ), + ]; + for (key, values) in lists { + if !values.is_empty() { + let _ = payload.insert(key.to_string(), strings(values)); + } + } + let flags: [(&str, bool); 3] = [ + ("allows_ceremonial_select", body.allows_ceremonial_select), + ( + "allows_dynamic_output_star", + body.allows_dynamic_output_star, + ), + ("allows_empty_fixture_star", body.allows_empty_fixture_star), + ]; + for (key, enabled) in flags { + if enabled { + let _ = payload.insert(key.to_string(), Value::Bool(true)); + } + } + let _ = payload.insert( + "max_literal_length".to_string(), + Value::from(options.max_literal_length), + ); + let _ = payload.insert( + "max_ranking_order_by".to_string(), + Value::from(options.max_ranking_order_by), + ); + if !body.relation_keys.is_empty() { + let keys: Map = body + .relation_keys + .iter() + .map(|(sentinel, keys)| (sentinel.clone(), Value::from(keys.clone()))) + .collect(); + let _ = payload.insert("relation_keys".to_string(), Value::Object(keys)); + } + Value::Object(payload) +} + +/// Run every payload through the batch engine, one response per payload. +pub(crate) fn run_batch(payloads: &[Value]) -> Result, String> { + if payloads.is_empty() { + return Ok(Vec::new()); + } + let response: String = lint_batch_json(&Value::from(payloads.to_vec()).to_string())?; + match serde_json::from_str::(&response).map_err(|error| error.to_string())? { + Value::Array(responses) if responses.len() == payloads.len() => Ok(responses), + _ => Err(INVALID_BATCH.to_string()), + } +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs new file mode 100644 index 0000000000..36bb34d459 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs @@ -0,0 +1,140 @@ +//! Suppression directive parsing and matching. + +use std::sync::LazyLock; + +use regex::Regex; +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::lint_text::_helpers::lines::LineSpan; +use crate::lint_text::_helpers::sites::compiled; +use crate::lint_text::constants::{ + FORBIDDEN_FIX_REASON, FORBIDDEN_MESSAGE, FORBIDDEN_REMEDIATION, INVALID_MESSAGE, + INVALID_REMEDIATION, NATIVE_RULE_PREFIX, REASON_REQUIRED_FIX_REASON, SUPPRESSION_CODE, + SUPPRESSION_PATTERN, SUPPRESSION_PREFIX, UNUSED_REMEDIATION, +}; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::{ + RetainedViolation, SuppressibleViolation, SuppressionDirective, SuppressionFinding, +}; + +static SUPPRESSION: LazyLock> = + LazyLock::new(|| Regex::new(SUPPRESSION_PATTERN)); + +/// Valid directives, and the one-based lines of invalid ones. +pub(crate) fn parse_directives( + lines: &[LineSpan<'_>], +) -> Result<(Vec, Vec), LintTextError> { + let pattern: &Regex = compiled(&SUPPRESSION)?; + let mut directives: Vec = Vec::new(); + let mut invalid: Vec = Vec::new(); + for (index, line) in lines.iter().enumerate() { + if !line.content.to_lowercase().contains(SUPPRESSION_PREFIX) { + continue; + } + let Some(captures) = pattern.captures(line.content) else { + invalid.push(index + 1); + continue; + }; + let mut target: usize = index + 1; + while target < lines.len() && skippable(lines[target].content) { + target += 1; + } + directives.push(SuppressionDirective { + code: captures["code"].to_string(), + line: index + 1, + target: target + 1, + }); + } + Ok((directives, invalid)) +} + +/// Blank lines and comment lines between a directive and the line it suppresses. +fn skippable(line: &str) -> bool { + let stripped: &str = line.trim_start_matches(python_isspace); + stripped.is_empty() || stripped.starts_with("--") +} + +/// Whether a retained violation is the native finding one directive suppresses. +pub(crate) fn matches_directive( + item: &RetainedViolation, + violations: &[SuppressibleViolation], + file: usize, + directive: &SuppressionDirective, +) -> bool { + let (item_file, line, code, native): (usize, usize, &str, bool) = match item { + RetainedViolation::Existing(index) => { + let violation: &SuppressibleViolation = &violations[*index]; + ( + violation.file, + violation.line, + &violation.code, + violation.native, + ) + } + RetainedViolation::Added(finding) => (finding.file, finding.line, SUPPRESSION_CODE, true), + }; + item_file == file + && line == directive.target + && code == directive.code + && native + && code.starts_with(NATIVE_RULE_PREFIX) +} + +/// Findings for every directive line when the project forbids suppressions. +pub(crate) fn forbidden_findings( + file: usize, + directives: &[SuppressionDirective], + invalid: &[usize], +) -> Vec { + let mut lines: Vec = invalid + .iter() + .copied() + .chain(directives.iter().map(|directive| directive.line)) + .collect(); + lines.sort_unstable(); + lines.dedup(); + lines + .into_iter() + .map(|line| { + RetainedViolation::Added(SuppressionFinding { + file, + line, + message: FORBIDDEN_MESSAGE.to_string(), + remediation: FORBIDDEN_REMEDIATION, + fault: true, + fix: None, + fix_unavailable_reason: Some(FORBIDDEN_FIX_REASON), + }) + }) + .collect() +} + +/// The finding for an invalid directive line. +pub(crate) fn invalid_finding(file: usize, line: usize) -> RetainedViolation { + RetainedViolation::Added(SuppressionFinding { + file, + line, + message: INVALID_MESSAGE.to_string(), + remediation: INVALID_REMEDIATION, + fault: false, + fix: None, + fix_unavailable_reason: Some(REASON_REQUIRED_FIX_REASON), + }) +} + +/// The finding for a directive that suppressed nothing, with an edit removing its line. +pub(crate) fn unused_finding( + file: usize, + directive: &SuppressionDirective, + span: &LineSpan<'_>, +) -> RetainedViolation { + RetainedViolation::Added(SuppressionFinding { + file, + line: directive.line, + message: format!("Unused suppression for {}", directive.code), + remediation: UNUSED_REMEDIATION, + fault: false, + fix: Some((span.start, span.end)), + fix_unavailable_reason: None, + }) +} diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs new file mode 100644 index 0000000000..28b1785577 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -0,0 +1,143 @@ +//! Sentinel spellings, suppression texts and fixture-null markers. + +/// Sentinel standing in for one interpolation site in lint and format SQL. +pub const LINT_SENTINEL_PREFIX: &str = "__sqb_lint_"; +pub(crate) const SENTINEL_SUFFIX: &str = "__"; +pub(crate) const AUDIT_PARAMETER_SENTINEL_PREFIX: &str = "__sqlbuild_audit_parameter_"; +pub(crate) const CONTEXT_SENTINEL_PREFIX: &str = "__sqlbuild_context_parameter_"; +pub(crate) const CONTEXT_INTERPOLATION_TOKEN: &str = "@@CTX:"; + +/// Every SQLBuild intrinsic call protected while formatting. +pub(crate) const FUNCTION_NAMES: [&str; 9] = [ + "__cursor_end", + "__cursor_start", + "__dbt_ref", + "__empty_fixture", + "__ref", + "__seed", + "__source", + "__table_fn", + "__udf", +]; + +pub(crate) const LINT_SITE_PATTERN: &str = + r"^(?:@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\()"; +pub(crate) const FORMAT_SITE_PATTERN: &str = r"(?i)^(?:@@|\$\{|@|(?:__cursor_end|__cursor_start|__dbt_ref|__empty_fixture|__ref|__seed|__source|__table_fn|__udf)\s*\()"; + +/// Diagnostic code of every suppression finding. +pub const SUPPRESSION_CODE: &str = "SQBRSQL000"; +pub(crate) const SUPPRESSION_PREFIX: &str = "sqb: ignore"; +pub(crate) const SUPPRESSION_PATTERN: &str = + r"^\s*--\s*sqb:\s*ignore\s+(?P[A-Za-z0-9_-]+)\s+because\s+(?P\S.*)\s*$"; +pub(crate) const NATIVE_RULE_PREFIX: &str = "SQBRSQL"; +pub(crate) const INVALID_MESSAGE: &str = "Suppression directive is invalid"; +pub(crate) const INVALID_REMEDIATION: &str = "Use '-- sqb: ignore CODE because '."; +pub(crate) const UNUSED_REMEDIATION: &str = "Remove the stale suppression directive."; +pub(crate) const FORBIDDEN_MESSAGE: &str = + "Inline Rule suppressions are not allowed in this project"; +pub(crate) const FORBIDDEN_REMEDIATION: &str = "Remove the directive and fix the finding it suppresses; sqlbuild_project.toml sets [rules] allow_exceptions = false. To allow inline suppressions, set this in sqlbuild_project.toml:\n [rules]\n allow_exceptions = true"; +pub(crate) const FORBIDDEN_FIX_REASON: &str = + "removing the directive exposes the finding it suppresses"; +pub(crate) const REASON_REQUIRED_FIX_REASON: &str = + "a valid suppression reason requires user intent"; + +pub(crate) const FIXTURE_TYPED_NULL_PATTERN: &str = r"(?i)(?:CAST\s*\(\s*NULL\s+AS\b|NULL\s*::)"; +pub(crate) const MODEL_FIXTURE_PATTERN: &str = + r"(?i)\b__ref__(?P[A-Za-z_][A-Za-z0-9_]*)\s+AS\s*\("; +pub(crate) const FIXTURE_CTE_MARKERS: [&str; 3] = ["__ref__", "__source__", "__seed__"]; +pub(crate) const IGNORED_SEARCH_DIRECTORIES: [&str; 4] = + [".git", ".venv", "target", "node_modules"]; +pub(crate) const UNIT_TEST_ROOT: [&str; 2] = ["tests", "unit"]; +pub(crate) const SQL_SUFFIX: &str = ".sql"; + +pub(crate) const FORMAT_API_VERSION: u32 = 1; +pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; +pub(crate) const OMITTED_CEREMONIAL_SELECT_SQL: &str = "\nSELECT 1"; +pub(crate) const OMITTED_SELECT_SUFFIX_PATTERN: &str = r"(?i)\s*\bSELECT\s+1\s*\z"; +pub(crate) const FIXTURE_HEADER_KINDS: [&str; 2] = ["TEST", "SCENARIO"]; +pub(crate) const LINE_COMMENT_PREFIX: &str = "--"; +pub(crate) const DECLINED_WITHOUT_REASON: &str = "native formatter declined without a reason"; +pub(crate) const MOVED_OMITTED_SELECT: &str = + "native formatter moved the omitted ceremonial SELECT 1"; +pub(crate) const DECLARATION_GROUP_DIRECTORY: &str = "_sqlbuild"; +/// `GLOBAL_NAMED_DECLARATION_ROOTS`, in Python's dict order, with whether each holds audits. +pub(crate) const GLOBAL_NAMED_ROOTS: [(&[&str], bool); 5] = [ + (&["audits", "generic"], true), + (&["audits", "singular"], true), + (&["schemas"], false), + (&["hooks", "sql"], false), + (&["hooks", "python"], false), +]; +/// `GROUPED_NAMED_DECLARATION_ROLES`, in Python's dict order, with whether each holds audits. +pub(crate) const GROUPED_NAMED_ROLES: [(&[&str], bool); 9] = [ + (&["audits", "generic"], true), + (&["_audits", "generic"], true), + (&["audits", "singular"], true), + (&["schemas"], false), + (&["_schemas"], false), + (&["hooks", "sql"], false), + (&["_hooks", "sql"], false), + (&["hooks", "python"], false), + (&["_hooks", "python"], false), +]; + +pub(crate) const LINT_API_VERSION: i64 = 1; +pub(crate) const UNUSED_CTE_CODE: &str = "SQBRSQL005"; +pub(crate) const LONG_LITERAL_CODE: &str = "SQBRSQL044"; +pub(crate) const NATIVE_LINT_ERROR_CODE: &str = "L003"; +pub(crate) const PARSE_ERROR_POSITION_PATTERN: &str = + r"^Parse error at line (?P\d+), column (?P\d+):"; +pub(crate) const HARNESS_CTE_PREFIXES: [&str; 13] = [ + "__expected__", + "__assert__", + "__ref__", + "__source__", + "__seed__", + "__dbt_ref__", + "__macro__", + "__macro_actual__", + "__macro_expected__", + "__udf_actual__", + "__udf_expected__", + "__table_fn_actual__", + "__table_fn_expected__", +]; +pub(crate) const PARSE_FAILURE_PREFIX: &str = "Native SQL parser could not analyze this body: "; +pub(crate) const PARSE_FAILURE_REMEDIATION: &str = + "Use SQL syntax supported by the native parser or report the parser gap."; +pub(crate) const INVALID_DIAGNOSTIC: &str = + "native lint diagnostic has invalid code, message, or source span"; +pub(crate) const INVALID_FIX_REASON: &str = + "native lint diagnostic has an invalid fix refusal reason"; +pub(crate) const INVALID_FIX_OBJECT: &str = "native lint diagnostic fix must be an object"; +pub(crate) const INVALID_FIX_EDIT: &str = "native lint diagnostic has an invalid fix edit"; +pub(crate) const NOT_AN_OBJECT: &str = "native lint diagnostic must be an object"; +pub(crate) const NON_OBJECT_RESPONSE: &str = "native lint engine returned a non-object response"; +pub(crate) const UNSUPPORTED_VERSION: &str = + "native lint engine returned an unsupported response version"; +pub(crate) const MISSING_DIAGNOSTICS: &str = "native lint response is missing a diagnostics list"; +pub(crate) const INVALID_BATCH: &str = "native lint engine returned an invalid batch response"; +pub(crate) const NON_CONTIGUOUS_FIX: &str = + "fix overlaps generated SQL or a non-contiguous authored region"; +pub(crate) const GENERATED_FIX: &str = "fix overlaps generated SQL"; +pub(crate) const GENERATED_SQL_SUFFIX: &str = "(in generated SQL)"; +pub(crate) const PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.toml"; +pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +pub(crate) const DEFAULT_LINE_WIDTH: i64 = 100; +pub(crate) const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; +pub(crate) const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; +pub(crate) const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; +pub(crate) const DEFAULT_DIALECT: &str = "generic"; +/// Adapter names whose SQL dialect has another name; other adapters name their dialect. +pub(crate) const ADAPTER_DIALECT_TRANSLATIONS: [(&str, &str); 2] = + [("motherduck", "duckdb"), ("sqlserver", "tsql")]; +pub(crate) const MODELS_DIRECTORY: &str = "models"; +/// Inherited and local declaration folders, whose files declare several headers. +pub(crate) const SCOPED_DECLARATION_DIRECTORIES: [&str; 6] = [ + "macros", + "enums", + "constants", + "_macros", + "_enums", + "_constants", +]; diff --git a/crates/sqlbuild-rules/src/lint_text/errors.rs b/crates/sqlbuild-rules/src/lint_text/errors.rs new file mode 100644 index 0000000000..aebf171128 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/errors.rs @@ -0,0 +1,22 @@ +//! Failures of the authored-SQL text transforms. + +use std::fmt; + +/// A lint text transform could not complete. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum LintTextError { + /// A formatted body no longer holds exactly one of each sentinel. + Restoration(String), + /// A built-in pattern failed to compile. + Pattern(String), +} + +impl fmt::Display for LintTextError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Restoration(message) | Self::Pattern(message) => formatter.write_str(message), + } + } +} + +impl std::error::Error for LintTextError {} diff --git a/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs b/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs new file mode 100644 index 0000000000..590a0f4259 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs @@ -0,0 +1,40 @@ +//! `apply_suppressions`: reason-required next-line suppressions applied to lint violations. + +use crate::lint_text::_helpers::lines::{LineSpan, line_spans}; +use crate::lint_text::_helpers::suppressions::{ + forbidden_findings, invalid_finding, matches_directive, parse_directives, unused_finding, +}; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::{RetainedViolation, SuppressibleViolation}; + +/// Remove matching next-line diagnostics, or reject every directive when suppressions are off. +pub fn apply_suppressions( + files: &[&str], + violations: &[SuppressibleViolation], + allow_suppressions: bool, +) -> Result, LintTextError> { + let mut retained: Vec = (0..violations.len()) + .map(RetainedViolation::Existing) + .collect(); + for (file, contents) in files.iter().enumerate() { + let lines: Vec> = line_spans(contents); + let (directives, invalid) = parse_directives(&lines)?; + if !allow_suppressions { + retained.extend(forbidden_findings(file, &directives, &invalid)); + continue; + } + retained.extend(invalid.into_iter().map(|line| invalid_finding(file, line))); + for directive in directives { + match retained + .iter() + .position(|item| matches_directive(item, violations, file, &directive)) + { + Some(index) => { + let _ = retained.remove(index); + } + None => retained.push(unused_finding(file, &directive, &lines[directive.line - 1])), + } + } + } + Ok(retained) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/first_header_only.rs b/crates/sqlbuild-rules/src/lint_text/main/first_header_only.rs new file mode 100644 index 0000000000..73aa6d3cf2 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/first_header_only.rs @@ -0,0 +1,19 @@ +//! Whether only the first DSL header of a file is lintable. + +use crate::lint_text::constants::{ + DECLARATION_GROUP_DIRECTORY, MODELS_DIRECTORY, SCOPED_DECLARATION_DIRECTORIES, +}; + +/// Model files outside declaration folders hold one header; every other file may hold several. +#[must_use] +pub fn first_header_only(relative_parts: &[String]) -> bool { + let Some((first, rest)) = relative_parts.split_first() else { + return false; + }; + let folders: &[String] = rest.split_last().map_or(&[], |(_, folders)| folders); + first == MODELS_DIRECTORY + && !folders.iter().any(|folder| { + folder == DECLARATION_GROUP_DIRECTORY + || SCOPED_DECLARATION_DIRECTORIES.contains(&folder.as_str()) + }) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs b/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs new file mode 100644 index 0000000000..f823ee3ce7 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/format_sql_bodies.rs @@ -0,0 +1,33 @@ +//! `format_sql_bodies`: SQLBuild-owned safe formatting of every SQL body in files. + +use crate::lint_text::_helpers::format_bodies::{RequestTable, prepare_file, splice_file}; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::{FormatFile, FormattedBodies, PreparedBody}; +use crate::sql_lint::main::format_requests::format_requests; +use crate::sql_lint::models::FormatResponse; + +/// Format SQL bodies, reporting every declined body as a fault and leaving it unchanged. +pub fn format_sql_bodies( + files: &[FormatFile], + dialect: &str, + line_width: usize, +) -> Result { + let mut table: RequestTable = RequestTable::default(); + let mut prepared: Vec> = Vec::with_capacity(files.len()); + for file in files { + let mut bodies: Vec = Vec::new(); + for (mut body, sql, widths) in prepare_file(file, dialect)? { + body.request = table.insert(sql, widths, (dialect, line_width)); + bodies.push(body); + } + prepared.push(bodies); + } + let responses: Vec = format_requests(table.requests); + let mut result: FormattedBodies = FormattedBodies::default(); + for (index, (file, bodies)) in files.iter().zip(prepared).enumerate() { + let (formatted, faults) = splice_file(index, &file.contents, &bodies, &responses); + result.formatted.extend(formatted.map(|text| (index, text))); + result.faults.extend(faults); + } + Ok(result) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/interpolation_text_at.rs b/crates/sqlbuild-rules/src/lint_text/main/interpolation_text_at.rs new file mode 100644 index 0000000000..30a49ebd5b --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/interpolation_text_at.rs @@ -0,0 +1,11 @@ +//! `interpolation_text_at`: the interpolation token beginning at one character offset. + +use crate::lint_text::_helpers::sites::{byte_offset, lint_policy, site_end}; +use crate::sql_lint::main::backtick_identifiers::backtick_identifiers; + +/// Return the interpolation token starting at character position `start`, if there is one. +pub fn interpolation_text_at(body: &str, start: usize, dialect: &str) -> Option { + let byte_start: usize = byte_offset(body, start)?; + let end: usize = site_end(body, byte_start, lint_policy(backtick_identifiers(dialect)))?; + Some(body[byte_start..end].to_string()) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs b/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs new file mode 100644 index 0000000000..70f8360419 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/lint_sql_bodies.rs @@ -0,0 +1,63 @@ +//! `lint_sql_bodies`: native SQL lint of expanded bodies, mapped onto authored source. + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::lint_text::_helpers::sql_diagnostics::{ + AuthoredFile, authored_diagnostic, authored_prefix, parse_failure, response_diagnostics, +}; +use crate::lint_text::_helpers::sql_requests::{lint_payload, omitted_select, run_batch}; +use crate::lint_text::models::{AuthoredDiagnostic, LintSqlBody, LintSqlOptions}; + +/// Lint every body once per distinct request and map each diagnostic onto its authored file. +pub fn lint_sql_bodies( + bodies: &[LintSqlBody], + files: &[&str], + options: &LintSqlOptions, +) -> Vec { + let omitted: Vec> = bodies.iter().map(omitted_select).collect(); + let mut payloads: Vec = Vec::new(); + let mut index_by_request: HashMap = HashMap::new(); + let mut request_of_body: Vec = Vec::with_capacity(bodies.len()); + for (body, offset) in bodies.iter().zip(&omitted) { + let payload: Value = lint_payload(body, *offset, options); + let key: String = payload.to_string(); + let next: usize = payloads.len(); + let index: usize = *index_by_request.entry(key).or_insert(next); + if index == next { + payloads.push(payload); + } + request_of_body.push(index); + } + let responses: Vec, String>> = run_batch(&payloads) + .map(|responses| responses.iter().map(response_diagnostics).collect()) + .unwrap_or_else(|error| vec![Err(error); payloads.len()]); + let authored: Vec> = files + .iter() + .map(|text| AuthoredFile::new(text, &options.dialect)) + .collect(); + let mut diagnostics: Vec = Vec::new(); + for ((body, offset), request) in bodies.iter().zip(omitted).zip(request_of_body) { + let file: &AuthoredFile<'_> = &authored[body.file]; + let raw: Vec = match &responses[request] { + Ok(raw) => raw.clone(), + Err(message) => { + diagnostics.push(parse_failure(message, body, file)); + continue; + } + }; + let raw: Vec = match offset { + Some(offset) => authored_prefix(raw, offset as i64), + None => raw, + }; + for item in &raw { + match authored_diagnostic(item, body, file) { + Ok(Some(diagnostic)) => diagnostics.push(diagnostic), + Ok(None) => {} + Err(message) => diagnostics.push(parse_failure(&message, body, file)), + } + } + } + diagnostics +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs new file mode 100644 index 0000000000..1c2440670a --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -0,0 +1,11 @@ +pub mod apply_suppressions; +pub mod first_header_only; +pub mod format_sql_bodies; +pub mod interpolation_text_at; +pub mod lint_sql_bodies; +pub mod neutralize_audit_parameters; +pub mod neutralize_context_interpolation; +pub mod neutralize_interpolation; +pub mod resolve_format_settings; +pub mod restore_interpolation; +pub mod scan_fixture_null_candidates; diff --git a/crates/sqlbuild-rules/src/lint_text/main/neutralize_audit_parameters.rs b/crates/sqlbuild-rules/src/lint_text/main/neutralize_audit_parameters.rs new file mode 100644 index 0000000000..3d382901c6 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/neutralize_audit_parameters.rs @@ -0,0 +1,66 @@ +//! `neutralize_audit_parameters`: unbound generic-audit `@name` arguments replaced by sentinels. + +use sqlbuild_core::text::main::python_isspace::python_isspace; + +use crate::lint_text::_helpers::sites::{ByteSite, assemble, sentinel}; +use crate::lint_text::constants::AUDIT_PARAMETER_SENTINEL_PREFIX; +use crate::lint_text::models::Neutralized; + +/// Replace `@'name'` and `@name` parameters, but not calls or `@@`, with generated sentinels. +pub fn neutralize_audit_parameters(body: &str) -> Neutralized { + let bytes: &[u8] = body.as_bytes(); + let mut matches: Vec<(usize, usize, bool)> = Vec::new(); + for (start, byte) in bytes.iter().enumerate() { + if *byte != b'@' || start > 0 && bytes[start - 1] == b'@' { + continue; + } + if bytes.get(start + 1) == Some(&b'\'') { + let end: usize = ascii_name_end(bytes, start + 2); + if end > start + 2 && bytes.get(end) == Some(&b'\'') { + matches.push((start, end + 1, true)); + } + continue; + } + let end: usize = ascii_name_end(bytes, start + 1); + if end > start + 1 && !opens_call(body, end) { + matches.push((start, end, false)); + } + } + let byte_sites: Vec = matches + .into_iter() + .enumerate() + .map(|(index, (start, end, quoted))| { + let identifier: String = sentinel(AUDIT_PARAMETER_SENTINEL_PREFIX, index); + let replacement: String = if quoted { + format!("'{identifier}'") + } else { + identifier + }; + (start, end, replacement) + }) + .collect(); + assemble(body, byte_sites) +} + +/// End of an ASCII `[A-Za-z_][A-Za-z0-9_]*` name at `start`, or `start` when none begins there. +fn ascii_name_end(bytes: &[u8], start: usize) -> usize { + match bytes.get(start) { + Some(byte) if byte.is_ascii_alphabetic() || *byte == b'_' => {} + _ => return start, + } + let mut end: usize = start + 1; + while bytes + .get(end) + .is_some_and(|byte| byte.is_ascii_alphanumeric() || *byte == b'_') + { + end += 1; + } + end +} + +fn opens_call(body: &str, end: usize) -> bool { + body[end..] + .chars() + .find(|character| !python_isspace(*character)) + == Some('(') +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/neutralize_context_interpolation.rs b/crates/sqlbuild-rules/src/lint_text/main/neutralize_context_interpolation.rs new file mode 100644 index 0000000000..5a5ad890e9 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/neutralize_context_interpolation.rs @@ -0,0 +1,22 @@ +//! `neutralize_context_interpolation`: runtime `@@CTX:` interpolation replaced by sentinels. + +use crate::lint_text::_helpers::sites::{ByteSite, assemble, name_end, sentinel}; +use crate::lint_text::constants::{CONTEXT_INTERPOLATION_TOKEN, CONTEXT_SENTINEL_PREFIX}; +use crate::lint_text::models::Neutralized; + +/// Replace every `@@CTX:` interpolation with a parseable generated sentinel. +pub fn neutralize_context_interpolation(body: &str) -> Neutralized { + let mut byte_sites: Vec = Vec::new(); + let mut copied_to: usize = 0; + while let Some(found) = body[copied_to..].find(CONTEXT_INTERPOLATION_TOKEN) { + let start: usize = copied_to + found; + let end: usize = name_end(body, start + 2, true); + byte_sites.push(( + start, + end, + sentinel(CONTEXT_SENTINEL_PREFIX, byte_sites.len()), + )); + copied_to = end; + } + assemble(body, byte_sites) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs b/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs new file mode 100644 index 0000000000..2cf0ef8f43 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/neutralize_interpolation.rs @@ -0,0 +1,34 @@ +//! `neutralize_interpolation`: SQLBuild references, macros and templates replaced by sentinels. + +use regex::Regex; + +use crate::lint_text::_helpers::sites::{ + ByteSite, assemble, lint_policy, sentinel, site_end, site_pattern, site_starts, +}; +use crate::lint_text::constants::LINT_SENTINEL_PREFIX; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::Neutralized; +use crate::sql_lint::main::backtick_identifiers::backtick_identifiers; + +/// Neutralize references for lint, or every intrinsic spelling in any case for formatting. +pub fn neutralize_interpolation( + body: &str, + dialect: &str, + formatting: bool, +) -> Result { + let pattern: &Regex = site_pattern(formatting)?; + let policy = lint_policy(backtick_identifiers(dialect)); + let mut byte_sites: Vec = Vec::new(); + let mut copied_to: usize = 0; + for start in site_starts(body, pattern, policy) { + if start < copied_to { + continue; + } + let Some(end) = site_end(body, start, policy) else { + continue; + }; + byte_sites.push((start, end, sentinel(LINT_SENTINEL_PREFIX, byte_sites.len()))); + copied_to = end; + } + Ok(assemble(body, byte_sites)) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs b/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs new file mode 100644 index 0000000000..f017d05a27 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/resolve_format_settings.rs @@ -0,0 +1,54 @@ +//! Resolve format and lint settings from the project's configuration files. + +use std::path::Path; + +use crate::lint_text::_helpers::settings::{adapter_dialect, positive_int, read_table}; +use crate::lint_text::constants::{ + DEFAULT_DIALECT, DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES, DEFAULT_MAX_LITERAL_LENGTH, + DEFAULT_MAX_RANKING_ORDER_BY, LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, +}; +use crate::lint_text::models::FormatSettings; + +/// `[format]` widths, rule thresholds and the adapter dialect, the local adapter winning. +pub fn resolve_format_settings(project_dir: &Path) -> Result { + let project: Option = read_table(&project_dir.join(PROJECT_CONFIG_FILENAME))?; + let format_section: Option<&toml::Table> = project + .as_ref() + .and_then(|table| table.get("format")) + .and_then(toml::Value::as_table); + let thresholds: Option<&toml::Table> = project + .as_ref() + .and_then(|table| table.get("rules")) + .and_then(toml::Value::as_table) + .and_then(|rules| rules.get("thresholds")) + .and_then(toml::Value::as_table); + let mut dialect: String = project + .as_ref() + .and_then(adapter_dialect) + .unwrap_or_else(|| DEFAULT_DIALECT.to_owned()); + if let Some(local) = read_table(&project_dir.join(LOCAL_CONFIG_FILENAME))? + && let Some(local_dialect) = adapter_dialect(&local) + { + dialect = local_dialect; + } + Ok(FormatSettings { + line_width: positive_int(format_section, "line_width", DEFAULT_LINE_WIDTH), + max_description_lines: positive_int( + format_section, + "max_description_lines", + DEFAULT_MAX_DESCRIPTION_LINES, + ), + dialect, + max_literal_length: positive_int( + thresholds, + "max_literal_length", + DEFAULT_MAX_LITERAL_LENGTH, + ), + max_ranking_order_by: positive_int( + thresholds, + "max_ranking_order_by", + DEFAULT_MAX_RANKING_ORDER_BY, + ), + has_project_config: project.is_some(), + }) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/restore_interpolation.rs b/crates/sqlbuild-rules/src/lint_text/main/restore_interpolation.rs new file mode 100644 index 0000000000..ea9e8a1dda --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/restore_interpolation.rs @@ -0,0 +1,30 @@ +//! `restore_interpolation`: sentinels swapped back to the text they stand in for. + +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::InterpolationSite; + +/// Swap each sentinel, matched case-insensitively, back to its original text exactly once. +pub fn restore_interpolation( + fixed: &str, + sites: &[InterpolationSite], +) -> Result { + let mut restored: String = fixed.to_string(); + for site in sites { + let needle: String = site.sentinel.to_ascii_lowercase(); + let folded: String = restored.to_ascii_lowercase(); + let occurrences: Vec = folded + .match_indices(&needle) + .map(|(index, _)| index) + .collect(); + let [start] = occurrences[..] else { + return Err(LintTextError::Restoration(format!( + "formatted SQL contains {} occurrences of sentinel '{}' standing in for '{}'; expected exactly 1", + occurrences.len(), + site.sentinel, + site.original_text + ))); + }; + restored.replace_range(start..start + needle.len(), &site.original_text); + } + Ok(restored) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/scan_fixture_null_candidates.rs b/crates/sqlbuild-rules/src/lint_text/main/scan_fixture_null_candidates.rs new file mode 100644 index 0000000000..bc67300971 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/scan_fixture_null_candidates.rs @@ -0,0 +1,41 @@ +//! `scan_fixture_null_candidates`: SQL-test fixtures that may hold removable typed nulls. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use crate::lint_text::_helpers::fixtures::{fixture_models, in_ignored_directory, sql_files}; +use crate::lint_text::constants::UNIT_TEST_ROOT; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::FixtureNullCandidates; + +/// Resolved `tests/unit` SQL files with typed-null fixtures, optionally only selected ones. +pub fn scan_fixture_null_candidates( + project_dir: &Path, + selected_paths: Option<&BTreeSet>, +) -> Result { + let root: PathBuf = project_dir.join(UNIT_TEST_ROOT[0]).join(UNIT_TEST_ROOT[1]); + let mut paths: BTreeSet = BTreeSet::new(); + let mut model_names: BTreeSet = BTreeSet::new(); + for file in sql_files(&root) { + if in_ignored_directory(&file) { + continue; + } + let Ok(resolved) = file.canonicalize() else { + continue; + }; + if selected_paths.is_some_and(|selected| !selected.contains(&resolved)) { + continue; + } + let Ok(contents) = std::fs::read_to_string(&file) else { + continue; + }; + if let Some(models) = fixture_models(&contents)? { + paths.insert(resolved.to_string_lossy().into_owned()); + model_names.extend(models); + } + } + Ok(FixtureNullCandidates { + paths: paths.into_iter().collect(), + model_names: model_names.into_iter().collect(), + }) +} diff --git a/crates/sqlbuild-rules/src/lint_text/mod.rs b/crates/sqlbuild-rules/src/lint_text/mod.rs new file mode 100644 index 0000000000..e0a0e363a4 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/mod.rs @@ -0,0 +1,10 @@ +//! Authored-SQL text transforms for lint and format: interpolation sentinels and suppressions. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs new file mode 100644 index 0000000000..dafaf23b63 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -0,0 +1,165 @@ +//! Interpolation sites, suppression outcomes and fixture-null candidates. + +/// One interpolation occurrence replaced by a sentinel; offsets are character positions. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InterpolationSite { + pub sentinel: String, + pub neutralized_start: usize, + pub neutralized_end: usize, + pub original_start: usize, + pub original_end: usize, + pub original_text: String, +} + +/// Text with its interpolation replaced by sentinels, and the replaced sites in order. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Neutralized { + pub text: String, + pub sites: Vec, +} + +/// The identity of one existing violation that a suppression can remove. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SuppressibleViolation { + pub file: usize, + pub line: usize, + pub code: String, + pub native: bool, +} + +/// One valid directive: its code, its line and the line it suppresses. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SuppressionDirective { + pub code: String, + pub line: usize, + pub target: usize, +} + +/// A suppression finding on one directive line; offsets are character positions. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SuppressionFinding { + pub file: usize, + pub line: usize, + pub message: String, + pub remediation: &'static str, + pub fault: bool, + pub fix: Option<(usize, usize)>, + pub fix_unavailable_reason: Option<&'static str>, +} + +/// One retained violation: an input violation by index, or a new suppression finding. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RetainedViolation { + Existing(usize), + Added(SuppressionFinding), +} + +/// SQL-test files that may hold removable typed nulls, and the models their fixtures stand in for. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FixtureNullCandidates { + pub paths: Vec, + pub model_names: Vec, +} + +/// One SQL file to format: its project-relative path when inside the project, and its text. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FormatFile { + pub relative_parts: Option>, + pub contents: String, +} + +/// A body the formatter declined, by file index, one-based line and reason. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FormatFault { + pub file: usize, + pub line: usize, + pub reason: String, +} + +/// Formatted file texts by file index, and the bodies left unchanged. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct FormattedBodies { + pub formatted: Vec<(usize, String)>, + pub faults: Vec, +} + +/// One authored body prepared for batched formatting; offsets are character positions. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PreparedBody { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) trailing: String, + pub(crate) request: usize, + pub(crate) sites: Vec, + pub(crate) omits_select: bool, +} + +/// One substitution of an expansion pass: authored source range and generated output range. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ExpansionSpan { + pub source_start: i64, + pub source_end: i64, + pub output_start: i64, + pub output_end: i64, +} + +/// One expanded, neutralized authored body to lint; offsets are character positions. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LintSqlBody { + pub file: usize, + pub body_start: i64, + pub lint_text: String, + pub passes: Vec>, + pub external_identifiers: Vec, + pub dependency_identifiers: Vec, + pub externally_referenced_ctes: Vec, + pub allows_ceremonial_select: bool, + pub allows_dynamic_output_star: bool, + pub allows_empty_fixture_star: bool, + pub relation_keys: Vec<(String, Vec>)>, +} + +/// The rule selection and thresholds of one SQL lint run. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct LintSqlOptions { + pub dialect: String, + pub enabled_rules: Option>, + pub ignored_rules: Vec, + pub max_literal_length: i64, + pub max_ranking_order_by: i64, +} + +/// An authored-source replacement proposed by a rule; offsets are character positions. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthoredEdit { + pub start: i64, + pub end: i64, + pub replacement: String, +} + +/// One SQL lint diagnostic mapped onto authored source. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthoredDiagnostic { + pub file: usize, + pub line: i64, + pub column: i64, + pub code: String, + pub message: String, + pub fault: bool, + pub end: Option<(i64, i64)>, + pub remediation: String, + pub fix: Option, + pub fix_unavailable_reason: Option, +} + +/// Format and lint settings resolved from the project and local configuration files. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FormatSettings { + pub line_width: i64, + pub max_description_lines: i64, + pub dialect: String, + pub max_literal_length: i64, + pub max_ranking_order_by: i64, + /// Whether the project configuration file exists, so its rules configuration applies. + pub has_project_config: bool, +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests.rs b/crates/sqlbuild-rules/src/lint_text/tests.rs new file mode 100644 index 0000000000..a080a424d5 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_format_settings.rs"] +mod format_settings; +#[path = "tests/test_lint_text.rs"] +mod lint_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_format_settings.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_format_settings.rs new file mode 100644 index 0000000000..95603e8bbf --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_format_settings.rs @@ -0,0 +1,104 @@ +use std::fs; + +use tempfile::TempDir; + +use crate::lint_text::main::resolve_format_settings::resolve_format_settings; +use crate::lint_text::models::FormatSettings; +use crate::lint_text::tests::test_types::FormatSettingsTestCase; + +#[test] +fn given_project_config_when_resolving_format_settings_then_values_match_python() { + let test_cases = [ + FormatSettingsTestCase { + description: "format widths, thresholds and a translated local adapter", + project_toml: "\u{feff}name = \"demo\"\nadapter = \"duckdb\"\n[format]\nline_width = 80\nmax_description_lines = 0\n[rules.thresholds]\nmax_literal_length = 40\nmax_ranking_order_by = true\n", + local_toml: "adapter = \"sqlserver\"\n", + expected_settings: FormatSettings { + line_width: 80, + max_description_lines: 10, + dialect: "tsql".to_owned(), + max_literal_length: 40, + max_ranking_order_by: 5, + has_project_config: true, + }, + }, + FormatSettingsTestCase { + description: "an unknown adapter names its own dialect", + project_toml: "name = \"demo\"\nadapter = \"warehouse\"\n", + local_toml: "", + expected_settings: FormatSettings { + line_width: 100, + max_description_lines: 10, + dialect: "warehouse".to_owned(), + max_literal_length: 100, + max_ranking_order_by: 5, + has_project_config: true, + }, + }, + ]; + + for test_case in test_cases { + let project: TempDir = TempDir::new().expect("temporary project"); + fs::write( + project.path().join("sqlbuild_project.toml"), + test_case.project_toml, + ) + .expect("project config"); + fs::write( + project.path().join("sqlbuild_local.toml"), + test_case.local_toml, + ) + .expect("local config"); + assert_eq!( + resolve_format_settings(project.path()), + Ok(test_case.expected_settings), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relative_paths_when_classifying_headers_then_only_plain_models_scan_one_header() { + let test_cases = [ + crate::lint_text::tests::test_types::FirstHeaderTestCase { + description: "a plain model file", + relative_parts: &["models", "staging", "orders.sql"], + expected_first_only: true, + }, + crate::lint_text::tests::test_types::FirstHeaderTestCase { + description: "a scoped declaration file under models", + relative_parts: &["models", "_enums", "status.sql"], + expected_first_only: false, + }, + crate::lint_text::tests::test_types::FirstHeaderTestCase { + description: "a declaration group file under models", + relative_parts: &["models", "_sqlbuild", "orders.sql"], + expected_first_only: false, + }, + crate::lint_text::tests::test_types::FirstHeaderTestCase { + description: "a file named like a folder is still a model", + relative_parts: &["models", "enums.sql"], + expected_first_only: true, + }, + crate::lint_text::tests::test_types::FirstHeaderTestCase { + description: "files outside models scan every header", + relative_parts: &["tests", "unit", "orders.sql"], + expected_first_only: false, + }, + ]; + + for test_case in test_cases { + let parts: Vec = test_case + .relative_parts + .iter() + .map(|part| (*part).to_owned()) + .collect(); + assert_eq!( + crate::lint_text::main::first_header_only::first_header_only(&parts), + test_case.expected_first_only, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs new file mode 100644 index 0000000000..1a887c5bcc --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -0,0 +1,399 @@ +use crate::lint_text::constants::{ + FORBIDDEN_FIX_REASON, FORBIDDEN_MESSAGE, FORBIDDEN_REMEDIATION, INVALID_MESSAGE, + INVALID_REMEDIATION, REASON_REQUIRED_FIX_REASON, UNUSED_REMEDIATION, +}; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::main::apply_suppressions::apply_suppressions; +use crate::lint_text::main::format_sql_bodies::format_sql_bodies; +use crate::lint_text::main::interpolation_text_at::interpolation_text_at; +use crate::lint_text::main::lint_sql_bodies::lint_sql_bodies; +use crate::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; +use crate::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; +use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use crate::lint_text::main::restore_interpolation::restore_interpolation; +use crate::lint_text::models::{ + ExpansionSpan, FormatFile, LintSqlBody, LintSqlOptions, RetainedViolation, + SuppressibleViolation, SuppressionFinding, +}; +use crate::lint_text::tests::test_types::{ + Finding, FormatBodiesTestCase, LintBodiesTestCase, NeutralizeTestCase, ParameterTestCase, + RestoreTestCase, SuppressionTestCase, TokenAtTestCase, +}; + +const SUPPRESSED_FILE: &str = "-- sqb: ignore SQBRSQL005 because kept\n\n-- note\nselect 1\n-- sqb: ignore X because y\n-- sqb: ignore nope\nselect 2\n"; + +#[test] +fn given_authored_sql_when_neutralizing_then_sites_use_character_offsets() +-> Result<(), LintTextError> { + let test_cases = [ + NeutralizeTestCase { + description: "reference, macro call and template", + body: "select @m(a, ')') from __ref('o') where x > ${v}", + formatting: false, + expected_text: "select __sqb_lint_0__ from __sqb_lint_1__ where x > __sqb_lint_2__", + expected_sites: vec![ + ("@m(a, ')')", 7, 17), + ("__ref('o')", 23, 33), + ("${v}", 44, 48), + ], + }, + NeutralizeTestCase { + description: "non-ASCII text before a site shifts character offsets only", + body: "select 'é' as n, @@CTX:run.id", + formatting: false, + expected_text: "select 'é' as n, __sqb_lint_0__", + expected_sites: vec![("@@CTX:run.id", 17, 29)], + }, + NeutralizeTestCase { + description: "comments and strings hide sites", + body: "-- @x\nselect '@y', \"__ref(z)\" /* ${w} */", + formatting: false, + expected_text: "-- @x\nselect '@y', \"__ref(z)\" /* ${w} */", + expected_sites: vec![], + }, + NeutralizeTestCase { + description: "lint leaves spaced and upper-case intrinsics alone", + body: "select __REF('a'), __ref ('b'), __cursor_start()", + formatting: false, + expected_text: "select __REF('a'), __ref ('b'), __cursor_start()", + expected_sites: vec![], + }, + NeutralizeTestCase { + description: "formatting protects every intrinsic spelling", + body: "select __REF('a'), __ref ('b'), __cursor_start()", + formatting: true, + expected_text: "select __sqb_lint_0__, __sqb_lint_1__, __sqb_lint_2__", + expected_sites: vec![ + ("__REF('a')", 7, 17), + ("__ref ('b')", 19, 30), + ("__cursor_start()", 32, 48), + ], + }, + ]; + + for test_case in test_cases { + let neutralized = neutralize_interpolation(test_case.body, "duckdb", test_case.formatting)?; + let sites: Vec<(&str, usize, usize)> = neutralized + .sites + .iter() + .map(|site| { + ( + site.original_text.as_str(), + site.original_start, + site.original_end, + ) + }) + .collect(); + + assert_eq!( + neutralized.text, test_case.expected_text, + "{}", + test_case.description + ); + assert_eq!(sites, test_case.expected_sites, "{}", test_case.description); + } + Ok(()) +} + +#[test] +fn given_generated_parameters_when_neutralizing_then_only_bare_parameters_are_replaced() { + let test_cases = [ + ParameterTestCase { + description: "quoted and raw audit parameters, but not calls or doubled at-signs", + body: "select * from @this where @column < 0 and @'kind' = @f(1) and @@x", + neutralize: neutralize_audit_parameters, + expected_text: "select * from __sqlbuild_audit_parameter_0__ where __sqlbuild_audit_parameter_1__ < 0 and '__sqlbuild_audit_parameter_2__' = @f(1) and @@x", + }, + ParameterTestCase { + description: "context interpolation in hooks", + body: "analyze @@CTX:relation; select @@CTX:a.b", + neutralize: neutralize_context_interpolation, + expected_text: "analyze __sqlbuild_context_parameter_0__; select __sqlbuild_context_parameter_1__", + }, + ]; + + for test_case in test_cases { + let neutralized = (test_case.neutralize)(test_case.body); + + assert_eq!( + neutralized.text, test_case.expected_text, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_formatted_sql_when_restoring_then_each_sentinel_must_appear_once() +-> Result<(), LintTextError> { + let test_cases = [ + RestoreTestCase { + description: "sentinels in any case restore to the original text", + body: "select @m, ${v}", + fixed: "SELECT __SQB_LINT_0__, __sqb_lint_1__", + expected_restored: Ok("SELECT @m, ${v}".to_string()), + }, + RestoreTestCase { + description: "a duplicated sentinel is ambiguous", + body: "select @m", + fixed: "select __sqb_lint_0__, __SQB_LINT_0__", + expected_restored: Err("formatted SQL contains 2 occurrences of sentinel '__sqb_lint_0__' standing in for '@m'; expected exactly 1".to_string()), + }, + ]; + + for test_case in test_cases { + let neutralized = neutralize_interpolation(test_case.body, "duckdb", true)?; + + let restored = restore_interpolation(test_case.fixed, &neutralized.sites) + .map_err(|error| error.to_string()); + + assert_eq!( + restored, test_case.expected_restored, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_character_offset_when_reading_token_then_only_site_starts_have_tokens() { + let test_cases = [ + TokenAtTestCase { + description: "a macro call after a multibyte character", + body: "é @m(x) y", + start: 2, + expected_token: Some("@m(x)"), + }, + TokenAtTestCase { + description: "a plain character", + body: "é @m(x) y", + start: 0, + expected_token: None, + }, + TokenAtTestCase { + description: "a spaced intrinsic call", + body: "x __source ('a', 'b')", + start: 2, + expected_token: Some("__source ('a', 'b')"), + }, + ]; + + for test_case in test_cases { + let token = interpolation_text_at(test_case.body, test_case.start, "duckdb"); + + assert_eq!( + token.as_deref(), + test_case.expected_token, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_suppression_directives_when_applying_then_matching_findings_are_removed() +-> Result<(), LintTextError> { + let test_cases = [ + SuppressionTestCase { + description: "a used, an unused and an invalid directive", + contents: SUPPRESSED_FILE, + violations: vec![(4, "SQBRSQL005"), (7, "SQBRSQL034")], + allow: true, + expected_retained: vec![ + RetainedViolation::Existing(1), + RetainedViolation::Added(SuppressionFinding { + file: 0, + line: 6, + message: INVALID_MESSAGE.to_string(), + remediation: INVALID_REMEDIATION, + fault: false, + fix: None, + fix_unavailable_reason: Some(REASON_REQUIRED_FIX_REASON), + }), + RetainedViolation::Added(SuppressionFinding { + file: 0, + line: 5, + message: "Unused suppression for X".to_string(), + remediation: UNUSED_REMEDIATION, + fault: false, + fix: Some((57, 84)), + fix_unavailable_reason: None, + }), + ], + }, + SuppressionTestCase { + description: "suppressions forbidden by the project", + contents: "select 1\n-- sqb: ignore nope\n", + violations: vec![(1, "SQBRSQL005")], + allow: false, + expected_retained: vec![ + RetainedViolation::Existing(0), + RetainedViolation::Added(SuppressionFinding { + file: 0, + line: 2, + message: FORBIDDEN_MESSAGE.to_string(), + remediation: FORBIDDEN_REMEDIATION, + fault: true, + fix: None, + fix_unavailable_reason: Some(FORBIDDEN_FIX_REASON), + }), + ], + }, + ]; + + for test_case in test_cases { + let violations: Vec = test_case + .violations + .iter() + .map(|(line, code)| SuppressibleViolation { + file: 0, + line: *line, + code: (*code).to_string(), + native: true, + }) + .collect(); + + let retained = apply_suppressions(&[test_case.contents], &violations, test_case.allow)?; + + assert_eq!( + retained, test_case.expected_retained, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_sql_files_when_formatting_bodies_then_sites_and_fixture_ceremony_survive() +-> Result<(), LintTextError> { + let test_cases = [ + FormatBodiesTestCase { + description: "a model body with a reference and a macro", + relative_parts: Some(vec!["models".to_string(), "orders.sql".to_string()]), + contents: "MODEL (description \"d\");\nselect a, @m('x') as m from __ref('o')\n", + expected_formatted: vec![( + 0, + "MODEL (description \"d\");\nSELECT\n a,\n @m('x') AS m\nFROM __ref('o')\n" + .to_string(), + )], + expected_fault_lines: vec![], + }, + FormatBodiesTestCase { + description: "a fixture body keeps its omitted ceremonial select", + relative_parts: Some(vec![ + "tests".to_string(), + "unit".to_string(), + "t.sql".to_string(), + ]), + contents: "TEST();\nwith __ref__o as (select 1 as a)\n", + expected_formatted: vec![( + 0, + "TEST();\nWITH __ref__o AS (\n SELECT 1 AS a\n)\n".to_string(), + )], + expected_fault_lines: vec![], + }, + FormatBodiesTestCase { + description: "an unparseable body is a fault on its first line", + relative_parts: None, + contents: "MODEL ();\n\nSELECT FROM WHERE (((\n", + expected_formatted: vec![], + expected_fault_lines: vec![3], + }, + ]; + + for test_case in test_cases { + let files = [FormatFile { + relative_parts: test_case.relative_parts.clone(), + contents: test_case.contents.to_string(), + }]; + + let result = format_sql_bodies(&files, "duckdb", 100)?; + let fault_lines: Vec = result.faults.iter().map(|fault| fault.line).collect(); + + assert_eq!( + result.formatted, test_case.expected_formatted, + "{}", + test_case.description + ); + assert_eq!( + fault_lines, test_case.expected_fault_lines, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_expanded_bodies_when_linting_then_findings_map_to_authored_lines() { + let test_cases = [ + LintBodiesTestCase { + description: "an unused CTE after a header maps to its authored line", + contents: "MODEL ();\nwith dead as (select 1 as y), live as (select 1 as a)\nselect a from live\n", + body: LintSqlBody { + body_start: 10, + lint_text: + "with dead as (select 1 as y), live as (select 1 as a)\nselect a from live" + .to_string(), + ..LintSqlBody::default() + }, + expected_findings: vec![("SQBRSQL005", 2, 6, Some((2, 10)))], + }, + LintBodiesTestCase { + description: "a parse failure is a fault at the mapped parser position", + contents: "MODEL ();\n\nselect a from from t\n", + body: LintSqlBody { + body_start: 11, + lint_text: "select a from from t".to_string(), + ..LintSqlBody::default() + }, + expected_findings: vec![("L003", 3, 19, None)], + }, + LintBodiesTestCase { + description: "a finding inside generated SQL points at the interpolation token", + contents: "select a from t where @isnull()\n", + body: LintSqlBody { + lint_text: "select a from t where a = null".to_string(), + passes: vec![vec![ExpansionSpan { + source_start: 22, + source_end: 31, + output_start: 22, + output_end: 30, + }]], + ..LintSqlBody::default() + }, + expected_findings: vec![("SQBRSQL001", 1, 23, Some((1, 32)))], + }, + ]; + + for test_case in test_cases { + let options = LintSqlOptions { + dialect: "duckdb".to_string(), + enabled_rules: Some(vec!["SQBRSQL001".to_string(), "SQBRSQL005".to_string()]), + max_literal_length: 100, + max_ranking_order_by: 5, + ..LintSqlOptions::default() + }; + + let result = lint_sql_bodies(&[test_case.body], &[test_case.contents], &options); + let findings: Vec> = result + .iter() + .map(|finding| { + ( + finding.code.as_str(), + finding.line, + finding.column, + finding.end, + ) + }) + .collect(); + + assert_eq!( + findings, test_case.expected_findings, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs new file mode 100644 index 0000000000..372cfcd932 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -0,0 +1,69 @@ +use crate::lint_text::models::{Neutralized, RetainedViolation}; + +pub(crate) struct NeutralizeTestCase { + pub(crate) description: &'static str, + pub(crate) body: &'static str, + pub(crate) formatting: bool, + pub(crate) expected_text: &'static str, + pub(crate) expected_sites: Vec<(&'static str, usize, usize)>, +} + +pub(crate) struct ParameterTestCase { + pub(crate) description: &'static str, + pub(crate) body: &'static str, + pub(crate) neutralize: fn(&str) -> Neutralized, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct RestoreTestCase { + pub(crate) description: &'static str, + pub(crate) body: &'static str, + pub(crate) fixed: &'static str, + pub(crate) expected_restored: Result, +} + +pub(crate) struct TokenAtTestCase { + pub(crate) description: &'static str, + pub(crate) body: &'static str, + pub(crate) start: usize, + pub(crate) expected_token: Option<&'static str>, +} + +pub(crate) struct SuppressionTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) violations: Vec<(usize, &'static str)>, + pub(crate) allow: bool, + pub(crate) expected_retained: Vec, +} + +pub(crate) struct FormatBodiesTestCase { + pub(crate) description: &'static str, + pub(crate) relative_parts: Option>, + pub(crate) contents: &'static str, + pub(crate) expected_formatted: Vec<(usize, String)>, + pub(crate) expected_fault_lines: Vec, +} + +pub(crate) struct LintBodiesTestCase { + pub(crate) description: &'static str, + pub(crate) contents: &'static str, + pub(crate) body: crate::lint_text::models::LintSqlBody, + pub(crate) expected_findings: Vec>, +} + +/// One mapped finding: code, line, column and optional end position. +pub(crate) type Finding<'a> = (&'a str, i64, i64, Option<(i64, i64)>); + +pub(super) struct FormatSettingsTestCase { + pub(super) description: &'static str, + pub(super) project_toml: &'static str, + pub(super) local_toml: &'static str, + pub(super) expected_settings: crate::lint_text::models::FormatSettings, +} + +pub(super) struct FirstHeaderTestCase { + pub(super) description: &'static str, + pub(super) relative_parts: &'static [&'static str], + pub(super) expected_first_only: bool, +} diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs index 8b8183ecd4..8c41ac461f 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/formatter.rs @@ -35,6 +35,23 @@ pub(crate) fn format_batch_json_impl(request_json: &str) -> Result) -> Vec { + requests + .into_par_iter() + .map(|request| { + let original: String = request.sql.clone(); + format_sql(request).unwrap_or_else(|reason| FormatResponse { + version: LINT_API_VERSION, + sql: original, + changed: false, + formatted: false, + reason: Some(reason), + }) + }) + .collect() +} + fn format_sql(request: FormatRequest) -> Result { let original = request.sql.clone(); match try_format_sql(request) { diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs index a80e132f89..f514682f67 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs @@ -5,15 +5,11 @@ use std::sync::LazyLock; use regex::Regex; +use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use crate::lint_text::models::Neutralized; use crate::sql_lint::types::{InterpolationSite, PreparedSql}; -use sqlbuild_sqltext::sql_scan::main::matching_paren::matching_paren as scan_matching_paren; -use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; -use sqlbuild_sqltext::sql_scan::models::QuotePolicy; -static SITE_START: LazyLock> = LazyLock::new(|| { - Regex::new(r"^(?:@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\s*\()") -}); static CTES: LazyLock> = LazyLock::new(|| { Regex::new( r#"(?is)(?:\bWITH(?:\s+RECURSIVE)?|,)\s*["`\[]?(?P[A-Za-z_][A-Za-z0-9_]*)["`\]]?(?:\s*\([^)]*\))?\s+AS\s*\("#, @@ -35,58 +31,30 @@ pub(crate) fn prepare( prior_sites: &[usize], dialect: &str, ) -> Result, String> { - if !expanded.is_ascii() || !before_expansion.is_ascii() { - return Ok(None); - } - let policy = QuotePolicy::SQL_LINT.with_backtick_identifiers(backtick_identifiers(dialect)); - let site_pattern = SITE_START - .as_ref() - .map_err(std::string::ToString::to_string)?; let cte_pattern = CTES.as_ref().map_err(std::string::ToString::to_string)?; let word_pattern = WORDS.as_ref().map_err(std::string::ToString::to_string)?; let opaque_pattern = OPAQUE_CTE .as_ref() .map_err(std::string::ToString::to_string)?; - let mut text = String::with_capacity(expanded.len()); - let mut sites: Vec = Vec::new(); - let mut copied_to = 0; - for start in site_starts(expanded, site_pattern, policy) { - if start < copied_to { - continue; - } - let Some(end) = site_end(expanded, start, policy) else { - continue; - }; - text.push_str(&expanded[copied_to..start]); - let sentinel = format!("__sqb_lint_{}__", sites.len()); - let output_start = text.len(); - text.push_str(&sentinel); - sites.push(( - sentinel, - output_start, - text.len(), - start, - end, - expanded[start..end].to_string(), - )); - copied_to = end; - } - text.push_str(&expanded[copied_to..]); + let neutralized: Neutralized = + neutralize_interpolation(expanded, dialect, false).map_err(|error| error.to_string())?; let names: HashSet = cte_pattern .captures_iter(expanded) .map(|capture| capture["name"].to_ascii_lowercase()) .collect(); let mut referenced: HashSet = HashSet::new(); - for site in &sites { + for site in &neutralized.sites { referenced.extend( word_pattern - .find_iter(&site.5) + .find_iter(&site.original_text) .map(|word| word.as_str().to_ascii_lowercase()) .filter(|word| names.contains(word)), ); } for &start in prior_sites { - let Some(prefix) = before_expansion.get(..start) else { + let Some(prefix) = byte_offset(before_expansion, start) + .and_then(|byte_start| before_expansion.get(..byte_start)) + else { return Ok(None); }; if opaque_pattern.is_match(prefix) { @@ -100,77 +68,27 @@ pub(crate) fn prepare( } let mut referenced: Vec = referenced.into_iter().collect(); referenced.sort(); - Ok(Some((text, sites, referenced))) -} - -/// Offsets of candidate interpolation sites outside comments and quoted text. -fn site_starts(text: &str, site_pattern: &Regex, policy: QuotePolicy) -> Vec { - let bytes = text.as_bytes(); - let mut starts: Vec = Vec::new(); - let mut index = 0; - while index < bytes.len() { - match non_code_end(bytes, index, policy) { - Ok(Some(end)) => { - index = end; - continue; - } - Err(_) => break, - Ok(None) => {} - } - if let Some(found) = site_pattern.find(&text[index..]) { - starts.push(index); - index += found.end(); - } else { - index += 1; - } - } - starts -} - -fn site_end(text: &str, start: usize, policy: QuotePolicy) -> Option { - let bytes = text.as_bytes(); - match bytes[start] { - b'@' if bytes.get(start + 1) == Some(&b'@') => Some(scan_name(bytes, start + 2, true)), - b'@' if bytes.get(start + 1) == Some(&b'\'') => { - text[start + 2..].find('\'').map(|index| start + index + 3) - } - b'@' => { - let first = *bytes.get(start + 1)?; - if !first.is_ascii_alphabetic() && first != b'_' { - return None; - } - let end = scan_name(bytes, start + 1, false); - if bytes.get(end) == Some(&b'(') { - matching_paren(bytes, end, policy) - } else { - Some(end) - } - } - b'$' => text[start + 2..].find('}').map(|index| start + index + 3), - b'_' => { - let end = scan_name(bytes, start, false); - (bytes.get(end) == Some(&b'(')) - .then(|| matching_paren(bytes, end, policy)) - .flatten() - } - _ => None, - } -} - -fn scan_name(bytes: &[u8], mut index: usize, interpolation: bool) -> usize { - while bytes.get(index).is_some_and(|byte| { - byte.is_ascii_alphanumeric() - || *byte == b'_' - || interpolation && matches!(byte, b':' | b'.') - }) { - index += 1; - } - index + let sites: Vec = neutralized + .sites + .into_iter() + .map(|site| { + ( + site.sentinel, + site.neutralized_start, + site.neutralized_end, + site.original_start, + site.original_end, + site.original_text, + ) + }) + .collect(); + Ok(Some((neutralized.text, sites, referenced))) } -fn matching_paren(bytes: &[u8], open: usize, policy: QuotePolicy) -> Option { - match scan_matching_paren(bytes, open, policy) { - Ok(close) => Some(close + 1), - Err(_) => None, - } +/// The byte offset of character position `position`, or `None` past the end. +fn byte_offset(text: &str, position: usize) -> Option { + text.char_indices() + .map(|(index, _)| index) + .chain(std::iter::once(text.len())) + .nth(position) } diff --git a/crates/sqlbuild-rules/src/sql_lint/main/format_requests.rs b/crates/sqlbuild-rules/src/sql_lint/main/format_requests.rs new file mode 100644 index 0000000000..6794204ce3 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/main/format_requests.rs @@ -0,0 +1,8 @@ +//! Typed, in-process batch formatting of neutralized SQL bodies. + +use crate::sql_lint::models::{FormatRequest, FormatResponse}; + +/// Format every request in parallel and return the responses in request order. +pub(crate) fn format_requests(requests: Vec) -> Vec { + crate::sql_lint::_helpers::formatter::format_requests(requests) +} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs index 4a2daec8b0..f14a92656f 100644 --- a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs @@ -3,5 +3,6 @@ pub mod batch_engine; pub mod batch_formatter; pub mod catalogue; pub mod engine; +pub mod format_requests; pub mod formatter; pub mod preparation; diff --git a/python-boundary.toml b/python-boundary.toml index 37c11ad33e..5ed091ad03 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 79435 +remaining_lines = 69877 [[stays]] path = "executor" @@ -286,18 +286,10 @@ note = "A warehouse cursor value that is not an integer is parsed as a timestamp location = "compiler/planner/_helpers/sql_tests/fixture_formatting.py::format_redundant_fixture_nulls" note = "A test file that no longer parses is left unformatted, as before; no Python implementation runs." -[[handled_native_errors]] -location = "lint/_helpers/native_sql.py::_native_responses" -note = "A native lint failure is reported as a lint error result, not recomputed in Python." - [[handled_native_errors]] location = "cli/commands/_helpers/refactor/compile_facts.py::compile_for_refactor" note = "Reports the Python stage errors the native compile driver propagates as refactor diagnostics; a native compiler failure is a RuntimeError and is not caught." -[[handled_native_errors]] -location = "cli/commands/_helpers/cost/refresh.py::refresh_pending_cost_run" -note = "Best-effort cost refresh keeps its existing catch-all; discovery is now entered through the native driver, and no Python implementation runs instead." - [[handled_native_errors]] location = "lint/_helpers/fixes.py::_FixSession._relation_keys" note = "A baseline that does not compile yields no relation keys, as before the compile entry became native; no Python implementation runs instead." diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6e0b59ac04..f457147d0d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -9,7 +9,12 @@ from typing import Any, Literal, TextIO, TypedDict, overload from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile, RelationInfo +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + ExpressionInferenceProfile, + QueryResult, + RelationInfo, +) from sqlbuild.adapter.contract.types import ( AdapterExecute, FrameworkType, @@ -19,16 +24,23 @@ from sqlbuild.adapter.contract.types import ( from sqlbuild.adapter.relations.models import InspectionQueryRecord from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.models import ( + AuditCommandRequest, + BuildCommandRequest, CheckCommandRequest, CloneCommandRequest, + ContractCommandRequest, + CostCommandRequest, DiffCommandRequest, FreshnessCommandRequest, JanitorCommandRequest, LineageCommandRequest, LoadCommandRequest, + PlanSafetyGates, PlaygroundCommandRequest, QueryDiffPreparation, RefactorCommandRequest, + ScenarioCaptureCommandRequest, + ScenarioTestCommandRequest, ScopeCommandRequest, SeedCommandRequest, TestCommandRequest, @@ -89,6 +101,7 @@ from sqlbuild.compiler.compile.models import ( NativeModelConfigInputs, NativeModelConfigSession, PythonSqlReferenceReport, + SqlExpansionContext, SqlReferenceScan, StaticMacroExport, ) @@ -248,7 +261,7 @@ from sqlbuild.integrations.dbt.models import ( ResolvedDbtProfileOutput, ) from sqlbuild.integrations.dbt.types import DbtInteropCommand -from sqlbuild.lint.models import HeaderSpan, LintConfig, LintViolation +from sqlbuild.lint.models import HeaderSpan, LintBody, LintConfig, LintRunResult, LintViolation from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.cli_style import CliStyle from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter @@ -261,6 +274,7 @@ from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import ( DefaultsConfig, + ExecutionLimitsConfig, LocalConfig, ProjectConfig, ScenarioConfig, @@ -270,6 +284,7 @@ from sqlbuild.spec.contracts.models import ( SchemaModelEntry, SchemaSeedEntry, SettingsConfig, + SnapshotsConfig, SourceEntry, SourceFreshnessAgePolicy, SourceLocation, @@ -1754,6 +1769,33 @@ def prepare_lint_sql_batch( requests: list[LintPreparationRequest], ) -> list[tuple[bool, tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None]]: ... def lint_backtick_identifiers(dialect: str) -> bool: ... +def neutralize_lint_interpolation( + body: str, dialect: str, for_formatting: bool = False +) -> tuple[str, list[tuple[str, int, int, int, int, str]]]: ... +def neutralize_lint_context_interpolation( + body: str, +) -> tuple[str, list[tuple[str, int, int, int, int, str]]]: ... +def neutralize_lint_audit_parameters( + body: str, +) -> tuple[str, list[tuple[str, int, int, int, int, str]]]: ... +def restore_lint_interpolation( + fixed: str, sites: list[tuple[str, int, int, int, int, str]] +) -> tuple[bool, str]: ... +def lint_interpolation_text_at(body: str, start: int, dialect: str) -> str | None: ... +def scan_fixture_typed_null_candidates( + project_dir: Path, selected_paths: frozenset[Path] | None = None +) -> tuple[frozenset[Path], frozenset[str]]: ... +def format_lint_sql_bodies( + files: list[tuple[Path, str]], project_dir: Path, config: LintConfig +) -> tuple[dict[Path, str], list[LintViolation]]: ... +def lint_native_sql_bodies( + bodies: list[LintBody], contents_by_path: dict[Path, str], config: LintConfig +) -> list[LintViolation]: ... +def apply_lint_suppressions( + violations: list[LintViolation], + contents_by_path: dict[Path, str], + allow_suppressions: bool, +) -> list[LintViolation]: ... def load_rules_config(*, project_dir: Path) -> RulesConfig: ... def selected_codes_json(request_json: str) -> str: ... def render_owned_skill(content: str, input_fingerprint: str) -> str: ... @@ -2877,6 +2919,60 @@ def write_check_results( def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str: ... def run_test_command(*, request: TestCommandRequest) -> int: ... def run_seed_command(*, request: SeedCommandRequest) -> int: ... +def run_build_command(*, request: BuildCommandRequest) -> int: ... +def enforce_plan_safety_policies( + *, + plan: PlanOutput, + gates: PlanSafetyGates, + snapshots_config: SnapshotsConfig, + target_name: str | None, + execution_limits: ExecutionLimitsConfig, + streams: tuple[TextIO, TextIO], +) -> None: ... +def enforce_model_execution_limit( + *, + model_count: int, + target_name: str | None, + limits: ExecutionLimitsConfig, + refusal_note: str, +) -> None: ... +def run_audit_command(*, request: AuditCommandRequest) -> int: ... +def run_contract_command(*, request: ContractCommandRequest) -> int: ... +def run_init_command(project_dir: Path | None = None) -> int: ... +def run_cost_command(request: CostCommandRequest) -> int: ... +def refresh_pending_cost_run(request: CostCommandRequest) -> None: ... +def parse_cost_since_bound(*, value: str | None) -> datetime | None: ... +def run_query_command( + *, + project_dir: Path | None, + sql: str | None, + query_file: Path | None = None, + selected_target: str | None = None, + output_format: str = "long", + limit: int | None = 20, +) -> int: ... +def render_query_result(*, result: QueryResult, output_format: str, limit: int | None) -> str: ... +def run_dag_command( + *, + project_dir: Path | None, + no_sql_validation: bool = False, + json_output: bool = False, + cli_vars: dict[str, object] | None = None, +) -> int: ... +def run_scenario_test_command(*, request: ScenarioTestCommandRequest) -> int: ... +def run_scenario_capture_command(*, request: ScenarioCaptureCommandRequest) -> int: ... +def run_format_command( + *, + project_dir: Path | None, + select: tuple[str, ...] = (), + exclude: tuple[str, ...] = (), + check: bool = False, + diff: bool = False, + fixtures_only: bool = False, + fix: bool = False, + json_output: bool = False, + no_color: bool = False, +) -> int: ... def format_parameterized_test_label( *, name: str, @@ -3260,6 +3356,22 @@ def sql_body_ranges( *, contents: str, headers: tuple[HeaderSpan, ...] ) -> tuple[tuple[int, int], ...]: ... def measurement_body_ranges(*, contents: str) -> tuple[tuple[int, int], ...]: ... +def run_lint( + *, + project_dir: Path, + config: LintConfig, + value_renderer: TypedSqlValueRenderer | None = None, + selected_paths: frozenset[Path] | None = None, + discovered_inputs: DiscoveredProjectInputs | None = None, + dynamic_output_paths: frozenset[Path] = frozenset(), + compiled_expansions: dict[Path, CompiledSqlExpansion] | None = None, + expansion_context: SqlExpansionContext | None = None, + source_files: dict[Path, str] | None = None, + skip_unexpandable: bool = False, +) -> LintRunResult: ... +def collect_lint_project_files( + *, project_dir: Path, selected_paths: frozenset[Path] | None = None +) -> dict[Path, str]: ... def lint_native_headers( *, contents: str, file_path: Path, headers: tuple[HeaderSpan, ...], config: LintConfig ) -> tuple[LintViolation, ...]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/audit/__init__.py b/src/sqlbuild/cli/commands/_helpers/audit/__init__.py deleted file mode 100644 index 5d6a40ce9d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/audit/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Audit command helpers.""" diff --git a/src/sqlbuild/cli/commands/_helpers/audit/execution.py b/src/sqlbuild/cli/commands/_helpers/audit/execution.py deleted file mode 100644 index fc3409740b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/audit/execution.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Audit command execution phases.""" - -from __future__ import annotations - -from sqlbuild._native import ( - ConnectionProgressReporter, - ExecutionEventWriter, - NativeProgressProjector, - current_native_progress_projector, -) -from sqlbuild.cli.commands.models import ( - AuditCommandRequest, - AuditExecutionPreparation, - AuditInvocation, -) -from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import AuditPlanEntry -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.pipeline.main.run import ( - AuditPipelineCallbacks, - run_audit_pipeline_with_callbacks, -) -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def prepare_audit_execution( - *, - request: AuditCommandRequest, - invocation: AuditInvocation, - pipeline_result: CompilePipelineResult, -) -> AuditExecutionPreparation: - """Prepare nested progress reporting and write the audit section header.""" - - audit_count: int = len(pipeline_result.plan_output.audit_entries) - effective_concurrency: int = ( - request.concurrency - if request.concurrency is not None - else pipeline_result.project.settings.concurrency - ) - worker_count: int = min(effective_concurrency, audit_count) - progress: AuditProgressReporter = AuditProgressReporter( - entries=pipeline_result.plan_output.audit_entries, - worker_limit=worker_count, - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - projector: NativeProgressProjector | None = current_native_progress_projector() - if projector is not None: - for entry in pipeline_result.plan_output.audit_entries: - projector.expect_resource_enrichment(resource_name=entry.name) - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=invocation.adapter_name, - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - return AuditExecutionPreparation( - progress=progress, - execution_connection_progress=execution_connection_progress, - effective_concurrency=effective_concurrency, - worker_count=worker_count, - ) - - -def write_audit_plan_header( - *, invocation: AuditInvocation, pipeline_result: CompilePipelineResult -) -> None: - """Write the selected audit section after effective execution settings.""" - - entries: tuple[AuditPlanEntry, ...] = pipeline_result.plan_output.audit_entries - model_count: int = len( - {entry.attached_target_name for entry in entries if entry.attached_target_name} - ) - header: str = f"Audit ({len(entries)} selected, {model_count} models)" - styled_header: str = CliStyle(use_color=invocation.use_color).success_strong(header) - invocation.progress_stream.write(f"\n{styled_header}\n\n") - invocation.progress_stream.flush() - - -def execute_audit_plan( - *, - invocation: AuditInvocation, - pipeline_result: CompilePipelineResult, - preparation: AuditExecutionPreparation, -) -> tuple[AuditExecutionResult, ...]: - """Execute the audit plan with nested progress reporting.""" - - event_writer: ExecutionEventWriter = ExecutionEventWriter() - preparation.progress.set_result_callback( - lambda result: event_writer.write_build_result( - result=result, - plan=pipeline_result.plan_output, - command="audit", - ) - ) - projector: NativeProgressProjector | None = current_native_progress_projector() - if projector is not None: - projector.configure_resources( - ordinals={ - entry.name: index - for index, entry in enumerate(pipeline_result.plan_output.audit_entries, start=1) - }, - total=len(pipeline_result.plan_output.audit_entries), - ) - try: - with CostContext.scope( - run_id=pipeline_result.project.run_id, - resource_type="run", - resource_name="audit", - ledger_path=( - invocation.effective_project_dir - / "target" - / "executions" - / pipeline_result.project.run_id - / "statements.jsonl" - ), - phase="audit", - ): - return run_audit_pipeline_with_callbacks( - plan=pipeline_result.plan_output, - connection_config=invocation.connection_config, - adapter=invocation.adapter, - max_concurrency=preparation.effective_concurrency, - callbacks=AuditPipelineCallbacks( - on_connection_start=( - preparation.execution_connection_progress.on_connection_start - ), - on_connection_complete=( - lambda connection_count, elapsed_seconds: ( - preparation.execution_connection_progress.on_connection_complete( - connection_count=connection_count, - elapsed_seconds=elapsed_seconds, - ) - ) - ), - on_connection_error=( - lambda connection_count, elapsed_seconds: ( - preparation.execution_connection_progress.on_connection_error( - connection_count=connection_count, - elapsed_seconds=elapsed_seconds, - ) - ) - ), - on_audit_start=preparation.progress.on_item_start, - on_audit_physical_complete=(preparation.progress.on_item_physical_complete), - on_audit_complete=preparation.progress.on_item_complete, - on_audit_error=preparation.progress.on_item_error, - ), - run_id=pipeline_result.project.run_id, - storage_database=pipeline_result.project.effective_target_database, - storage_schema=pipeline_result.project.effective_target_schema, - ) - finally: - preparation.progress.close() - event_writer.close() diff --git a/src/sqlbuild/cli/commands/_helpers/audit/invocation.py b/src/sqlbuild/cli/commands/_helpers/audit/invocation.py deleted file mode 100644 index a5ad81f7d5..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/audit/invocation.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Audit command invocation resolution phase.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( - resolve_reported_command_invocation, -) -from sqlbuild.cli.commands.models import AuditCommandRequest, AuditInvocation - - -def resolve_audit_invocation(*, request: AuditCommandRequest) -> AuditInvocation: - """Resolve discovery, adapter, connection, and reporters for audit.""" - - return resolve_reported_command_invocation(request=request, invocation_type=AuditInvocation) diff --git a/src/sqlbuild/cli/commands/_helpers/audit/outputs.py b/src/sqlbuild/cli/commands/_helpers/audit/outputs.py deleted file mode 100644 index 3c4ebdbc23..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/audit/outputs.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Audit command output writing and exit-code phases.""" - -from __future__ import annotations - -import json - -from sqlbuild._native import ( - format_audit_execution_json, - format_summary_footer, - write_execution_json_output, -) -from sqlbuild.cli.commands.models import AuditCommandRequest, AuditInvocation -from sqlbuild.compiler.auditing.types import AuditOutcome -from sqlbuild.executor.auditing.main.current_audit_result_projection import ( - current_audit_result_projection, -) -from sqlbuild.executor.auditing.models import AuditExecutionResult, AuditResultProjection - - -def write_audit_completion_output( - *, - request: AuditCommandRequest, - invocation: AuditInvocation, - results: tuple[AuditExecutionResult, ...], - configured_concurrency: int, - worker_count: int, -) -> None: - """Write the audit summary footer and optional JSON output.""" - - pass_count: int = sum(1 for r in results if r.outcome == AuditOutcome.PASS) - warn_count: int = sum(1 for r in results if r.outcome == AuditOutcome.WARN) - fail_count: int = sum(1 for r in results if r.outcome == AuditOutcome.ERROR) - insufficient_count: int = sum(1 for r in results if r.outcome == AuditOutcome.INSUFFICIENT) - invocation.progress_stream.write( - "\n" - + format_summary_footer( - counts=( - ("PASS", pass_count), - ("WARN", warn_count), - ("FAIL", fail_count), - ("INSUFFICIENT", insufficient_count), - ("TOTAL", len(results)), - ), - use_color=invocation.use_color, - ) - + "\n" - ) - invocation.progress_stream.flush() - payload: str = format_audit_execution_json( - results=results, - configured_concurrency=configured_concurrency, - worker_count=worker_count, - ) - projection: AuditResultProjection | None = current_audit_result_projection() - if projection is not None: - document: dict[str, object] = json.loads(payload) - document["audit_result_projection"] = { - "attempted_count": projection.attempted_count, - "written_count": projection.written_count, - "failed_count": projection.failed_count, - } - if projection.degraded: - document["projection_degraded"] = True - document["projection_degradation_reasons"] = [ - { - "reason": "audit_result_projection_failure", - "attempted_count": projection.attempted_count, - "written_count": projection.written_count, - "failed_count": projection.failed_count, - } - ] - payload = json.dumps(document, indent=2) + "\n" - write_execution_json_output( - payload=payload, - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_audit_exit_code(results: tuple[AuditExecutionResult, ...]) -> int: - """Resolve the audit exit code from failed audit outcomes.""" - - fail_count: int = sum(1 for r in results if r.outcome == AuditOutcome.ERROR) - return 0 if fail_count == 0 else 1 diff --git a/src/sqlbuild/cli/commands/_helpers/audit/planning.py b/src/sqlbuild/cli/commands/_helpers/audit/planning.py deleted file mode 100644 index 4d62c167a2..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/audit/planning.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Audit command plan compilation phase.""" - -from __future__ import annotations - -from sqlbuild._native import build_static_pipeline_result -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import AuditCommandRequest, AuditInvocation -from sqlbuild.compiler.pipeline.main.static_command import compile_static_command_context -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, - StaticCommandContext, -) -from sqlbuild.compiler.planner.main.commands.audit import build_audit_command_plan - - -def compile_audit_plan( - *, - request: AuditCommandRequest, - invocation: AuditInvocation, -) -> CompilePipelineResult: - """Compile the audit plan for the selected scope.""" - - options: CompilePipelineOptions = CompilePipelineOptions( - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - defer_to=request.defer_to, - select=request.select, - exclude=request.exclude, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ) - context: StaticCommandContext = compile_static_command_context( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=options, - on_progress=invocation.planning_progress.on_progress, - ) - return build_static_pipeline_result( - context=context, - plan_output=build_audit_command_plan( - project=context.project, - adapter=invocation.adapter, - scope=context.scope, - relations=context.relations, - ), - ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/__init__.py b/src/sqlbuild/cli/commands/_helpers/build_execution/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py b/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py deleted file mode 100644 index 2e302edb70..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/checks.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Build command post-build python checks phase.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild._native import build_project_relation_targets, write_check_results -from sqlbuild.cli.commands._helpers.check.core import ( - build_check_relation_targets, - load_results_by_loader_name, - record_python_run_state_results, - relevant_check_functions, -) -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildExecutionPreparation, - BuildInvocation, - BuildRunOutcome, -) -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.python_nodes.main.checks import execute_python_checks -from sqlbuild.executor.python_nodes.models import ( - PythonCheckCallbacks, - PythonCheckExecutionResult, - PythonNodeRunState, - PythonNodeRuntime, -) - - -def run_post_build_python_checks( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - outcome: BuildRunOutcome, - providers: Any, - preparation: BuildExecutionPreparation, -) -> tuple[PythonCheckExecutionResult, ...]: - """Execute relevant python check functions after a successful build.""" - - if outcome.result.status != BuildStatus.SUCCESS: - return () - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=invocation.discovered_inputs - ) - check_functions: tuple[DiscoveredCheckFunction, ...] = relevant_check_functions( - discovered_inputs=invocation.discovered_inputs, - python_graph=python_graph, - exclude=request.exclude, - selected_dependency_names=frozenset(result.node_name for result in outcome.python_results) - | frozenset( - load_results_by_loader_name( - source_map=pipeline_result.plan_output.source_map, - load_results=outcome.result.load_results, - ) - ), - completed_sql_names=frozenset( - ( - *(result.model_name for result in outcome.result.model_results), - *(result.seed_name for result in outcome.result.seed_results), - *(result.source_name for result in outcome.result.load_results), - ) - ), - ) - if not check_functions: - return () - check_connection: object = invocation.adapter.connect(invocation.connection_config) - check_results: tuple[PythonCheckExecutionResult, ...] - try: - check_run_state: PythonNodeRunState = PythonNodeRunState() - record_python_run_state_results( - discovered_inputs=invocation.discovered_inputs, - run_state=check_run_state, - python_results=outcome.python_results, - load_results=outcome.result.load_results, - source_map=pipeline_result.plan_output.source_map, - ) - check_results = execute_python_checks( - check_functions=check_functions, - python_graph=python_graph, - upstream_python_results=outcome.python_results, - upstream_load_results=outcome.result.load_results, - upstream_load_results_by_loader_name=load_results_by_loader_name( - source_map=pipeline_result.plan_output.source_map, - load_results=outcome.result.load_results, - ), - runtime=PythonNodeRuntime( - adapter=invocation.adapter, - connection_config=invocation.connection_config, - connection=check_connection, - run_id=pipeline_result.project.run_id, - target=pipeline_result.project.effective_target_name, - vars=pipeline_result.project.effective_vars, - is_reload=request.reload_sources, - default_database=invocation.adapter.default_database(), - default_schema=invocation.adapter.default_schema(), - relation_targets=build_check_relation_targets( - adapter=invocation.adapter, - pipeline_result=pipeline_result, - python_graph=python_graph, - selected_python_names=frozenset(check.name for check in check_functions), - ), - project_relations=build_project_relation_targets( - adapter=invocation.adapter, - plan_output=pipeline_result.plan_output, - ), - providers=providers, - ), - run_state=check_run_state, - callbacks=PythonCheckCallbacks( - on_check_complete=preparation.callbacks.write_execution_event, - ), - ) - finally: - invocation.adapter.close(check_connection) - write_check_results( - stream=invocation.progress_stream, - results=check_results, - use_color=invocation.use_color, - check_functions=check_functions, - python_graph=python_graph, - ) - return check_results diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/completion.py b/src/sqlbuild/cli/commands/_helpers/build_execution/completion.py deleted file mode 100644 index 3dc504c33d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/completion.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Successful direct-build completion phase.""" - -from __future__ import annotations - -import time -from datetime import UTC, datetime - -from sqlbuild.cli.commands._helpers.build_execution.outputs import ( - resolve_build_exit_code, - write_build_completion_output, - write_build_runtime_targets, -) -from sqlbuild.cli.commands._helpers.build_execution.phase_timings import ( - finish_cost_collection_timing, - write_build_phase_timings, -) -from sqlbuild.cli.commands._helpers.cost.collection import finalize_build_cost, render_build_cost -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildCostFinalization, - BuildExecutionPreparation, - BuildInvocation, - BuildPhaseTimings, - BuildRunOutcome, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.cost.models import CostRunRecord -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult - - -def complete_direct_build( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - preparation: BuildExecutionPreparation, - outcome: BuildRunOutcome, - check_results: tuple[PythonCheckExecutionResult, ...], - build_started_at: datetime, - command_started_at: float, -) -> int: - """Persist and render a completed direct build.""" - - build_completed_at: datetime = datetime.now(UTC) - exit_code: int = resolve_build_exit_code(outcome=outcome, check_results=check_results) - write_build_runtime_targets( - invocation=invocation, - pipeline_result=pipeline_result, - outcome=outcome, - check_results=check_results, - ) - cost_started_at: float = time.monotonic() - try: - cost_record: CostRunRecord | None = finalize_build_cost( - BuildCostFinalization( - project_dir=invocation.effective_project_dir, - adapter_name=invocation.adapter_name, - adapter=invocation.adapter, - connection_config=invocation.connection_config, - target_name=pipeline_result.project.effective_target_name, - target_database=pipeline_result.project.effective_target_database, - run_id=pipeline_result.project.run_id, - build_status="success" if exit_code == 0 else "failed", - started_at=build_started_at, - completed_at=build_completed_at, - config=invocation.discovered_inputs.project_config.cost, - output_stream=invocation.progress_stream, - use_color=invocation.use_color, - render=False, - ) - ) - finally: - cost_collection_seconds: float | None = finish_cost_collection_timing( - started_at=cost_started_at - ) - write_build_completion_output( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - outcome=outcome, - check_results=check_results, - cost_record=cost_record, - ) - _ = render_build_cost( - record=cost_record, - output_stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - if request.verbose or request.debug: - write_build_phase_timings( - stream=invocation.progress_stream, - timings=BuildPhaseTimings( - compile_seconds=pipeline_result.compile_seconds, - planning_seconds=pipeline_result.planning_seconds, - connection_preparation_seconds=( - outcome.result.timings.connection_preparation_seconds - ), - schema_preparation_seconds=outcome.result.timings.schema_preparation_seconds, - execution_seconds=outcome.result.timings.execution_seconds, - cost_collection_seconds=cost_collection_seconds, - total_seconds=time.monotonic() - command_started_at, - ), - use_color=invocation.use_color, - ) - return exit_code diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/exceptional_cost.py b/src/sqlbuild/cli/commands/_helpers/build_execution/exceptional_cost.py deleted file mode 100644 index 34e03eae04..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/exceptional_cost.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Exceptional direct-build cost finalization.""" - -from __future__ import annotations - -from datetime import UTC, datetime - -from sqlbuild.cli.commands._helpers.cost.collection import finalize_build_cost -from sqlbuild.cli.commands.models import BuildCostFinalization, BuildInvocation -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.cost.types import CostStatus - - -def finalize_exceptional_direct_cost( - *, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - build_started_at: datetime, - error: BaseException, -) -> None: - """Persist failed/interrupted cost state without masking the active error.""" - - interrupted: bool = isinstance(error, KeyboardInterrupt) - try: - _ = finalize_build_cost( - BuildCostFinalization( - project_dir=invocation.effective_project_dir, - adapter_name=invocation.adapter_name, - adapter=invocation.adapter, - connection_config=invocation.connection_config, - target_name=pipeline_result.project.effective_target_name, - target_database=pipeline_result.project.effective_target_database, - run_id=pipeline_result.project.run_id, - build_status="interrupted" if interrupted else "failed", - started_at=build_started_at, - completed_at=datetime.now(UTC), - config=invocation.discovered_inputs.project_config.cost, - output_stream=invocation.progress_stream, - use_color=invocation.use_color, - collect=not interrupted, - render=False, - cost_status=CostStatus.PARTIAL, - cost_message="Build was interrupted before cost collection completed.", - ) - ) - except BaseException: - return diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py b/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py deleted file mode 100644 index 69fa66a823..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/execution.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Build command execution preparation and pipeline phases.""" - -from __future__ import annotations - -from typing import Any, TextIO - -from sqlbuild._native import ( - ConnectionProgressReporter, - build_project_relation_targets, - write_execution_header, -) -from sqlbuild.cli.commands._helpers.build_execution.run_context import ( - write_build_run_context, -) -from sqlbuild.cli.commands._helpers.build_python_nodes.python_lifecycle import ( - prepare_direct_python_lifecycle, -) -from sqlbuild.cli.commands._helpers.freshness.source_freshness import ( - append_eligible_direct_source_freshness_records, -) -from sqlbuild.cli.commands._helpers.input.parsing import ( - parse_cursor_integer, - parse_cursor_timestamp, -) -from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildExecutionPreparation, - BuildInvocation, - BuildRunContext, - BuildRunOutcome, - DirectLifecycleCallbacks, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.main.cursor_window.typed_cursor_overrides import ( - typed_cursor_overrides, -) -from sqlbuild.compiler.planner.models import CursorOverrides -from sqlbuild.executor.build.models import ( - BuildCallbacks, - BuildCustomizations, - BuildExecutionResult, - BuildInitialState, - BuildRuntimeParams, -) -from sqlbuild.executor.pipeline.main.run import run_build_pipeline -from sqlbuild.executor.python_nodes.models import CursorWindow - - -def prepare_build_execution( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - providers: Any, -) -> BuildExecutionPreparation: - """Prepare callbacks, concurrency, cursors, and the python lifecycle for execution.""" - - callbacks: BuildProgressCallbacks = BuildProgressCallbacks( - plan=pipeline_result.plan_output, - use_color=invocation.use_color, - verbose=request.verbose, - debug=request.debug or request.json_output, - event_output_path=request.event_output_path, - ) - effective_concurrency: int = ( - request.concurrency - if request.concurrency is not None - else pipeline_result.project.settings.concurrency - ) - if request.verbose or request.debug: - write_build_run_context( - stream=invocation.progress_stream, - context=BuildRunContext( - command="sqb build", - project=pipeline_result.project, - plan=pipeline_result.plan_output, - discovered_inputs=invocation.discovered_inputs, - python_plan_entries=pipeline_result.python_plan_entries, - connection_config=invocation.connection_config, - concurrency=effective_concurrency, - full_refresh=request.full_refresh, - selector_files=request.selector_files, - ), - use_color=invocation.use_color, - ) - else: - write_execution_header( - stream=invocation.progress_stream, - command="sqb build", - target=None, - concurrency=effective_concurrency, - use_color=invocation.use_color, - ) - has_ingress_source_loads: bool = bool(pipeline_result.plan_output.source_load_entries) - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=invocation.adapter_name, - stream=invocation.progress_stream, - blank_line_before_start=has_ingress_source_loads, - blank_line_after_complete=True, - use_color=invocation.use_color, - ) - cursor_overrides: CursorOverrides = ( - pipeline_result.plan_output.cursor_overrides - or typed_cursor_overrides(cursor_overrides=request.cursor_overrides) - or CursorOverrides() - ) - preparation: BuildExecutionPreparation = BuildExecutionPreparation( - callbacks=callbacks, - effective_concurrency=effective_concurrency, - execution_connection_progress=execution_connection_progress, - python_lifecycle=prepare_direct_python_lifecycle( - discovered_inputs=invocation.discovered_inputs, - pipeline_result=pipeline_result, - plan_output=pipeline_result.plan_output, - adapter=invocation.adapter, - connection_config=invocation.connection_config, - include_python=request.include_python, - reload_sources=request.reload_sources, - cursor_window=CursorWindow( - start_cursor_ts=parse_cursor_timestamp(cursor_overrides.start_ts), - end_cursor_ts=parse_cursor_timestamp(cursor_overrides.end_ts), - start_cursor_int=parse_cursor_integer(cursor_overrides.start_int), - end_cursor_int=parse_cursor_integer(cursor_overrides.end_int), - ), - callbacks=DirectLifecycleCallbacks( - use_color=invocation.use_color, - progress_stream=invocation.progress_stream, - on_node_start=lambda name, resource_kind: callbacks.on_node_start( - name=name, resource_kind=resource_kind - ), - on_node_complete=callbacks.on_node_complete, - ), - providers=providers, - ), - start_cursor_ts=parse_cursor_timestamp(cursor_overrides.start_ts), - end_cursor_ts=parse_cursor_timestamp(cursor_overrides.end_ts), - start_cursor_int=parse_cursor_integer(cursor_overrides.start_int), - end_cursor_int=parse_cursor_integer(cursor_overrides.end_int), - ) - return preparation - - -def execute_build_plan( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - preparation: BuildExecutionPreparation, - providers: Any, -) -> BuildRunOutcome: - """Run the build pipeline and finalize python lifecycle results.""" - - result: BuildExecutionResult = run_build_pipeline( - plan=pipeline_result.plan_output, - connection_config=invocation.connection_config, - adapter=invocation.adapter, - settings=pipeline_result.project.settings, - runtime=BuildRuntimeParams( - run_id=pipeline_result.project.run_id, - runtime_dir=invocation.effective_project_dir / "target", - snapshots=invocation.discovered_inputs.project_config.snapshots, - allow_snapshot_schema_change=request.allow_snapshot_schema_change, - run_tests=request.run_tests, - run_audits=request.run_audits, - fail_fast=request.fail_fast, - max_concurrency=preparation.effective_concurrency, - loader_is_reload=request.reload_sources, - start_cursor_ts=preparation.start_cursor_ts, - end_cursor_ts=preparation.end_cursor_ts, - start_cursor_int=preparation.start_cursor_int, - end_cursor_int=preparation.end_cursor_int, - use_color=invocation.use_color, - providers=providers, - project_relations=build_project_relation_targets( - adapter=invocation.adapter, plan_output=pipeline_result.plan_output - ), - ), - callbacks=BuildCallbacks( - on_migration_progress=lambda message: _write_progress_line( - stream=invocation.progress_stream, message=message - ), - on_node_start=lambda name, resource_kind: preparation.callbacks.on_node_start( - name=name, resource_kind=resource_kind - ), - on_node_complete=preparation.python_lifecycle.on_node_complete, - on_sub_progress=preparation.callbacks.on_sub_progress, - on_connection_start=preparation.execution_connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - preparation.execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - preparation.execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_scheduler_state=( - preparation.callbacks.on_scheduler_state - if request.verbose or request.debug - else None - ), - on_statement_complete=( - preparation.callbacks.on_statement_complete - if request.verbose or request.debug - else None - ), - ), - customizations=BuildCustomizations( - custom_materializations=pipeline_result.custom_materializations, - loader_functions=preparation.python_lifecycle.loader_functions, - ), - initial_state=BuildInitialState( - precompleted_keys=preparation.python_lifecycle.precompleted_keys, - initial_load_results=preparation.python_lifecycle.ingress_load_results, - initial_failed_keys=preparation.python_lifecycle.blocked_keys, - ), - ) - append_eligible_direct_source_freshness_records( - plan=pipeline_result.plan_output, - result=result, - adapter=invocation.adapter, - connection_config=invocation.connection_config, - run_id=pipeline_result.project.run_id, - ) - preparation.python_lifecycle.finalize() - return BuildRunOutcome( - result=result, - python_results=preparation.python_lifecycle.python_results, - ) - - -def _write_progress_line(*, stream: TextIO, message: str) -> None: - stream.write(f"{message}\n") - stream.flush() diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/no_work.py b/src/sqlbuild/cli/commands/_helpers/build_execution/no_work.py deleted file mode 100644 index add32112c4..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/no_work.py +++ /dev/null @@ -1,80 +0,0 @@ -"""No-work build completion boundary.""" - -from datetime import UTC, datetime - -from sqlbuild._native import plan_has_executable_work -from sqlbuild.cli.commands._helpers.build_execution.outputs import write_no_work_build_json -from sqlbuild.cli.commands._helpers.build_execution.run_context import ( - write_build_run_context, -) -from sqlbuild.cli.commands._helpers.cost.collection import finalize_build_cost -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildCostFinalization, - BuildInvocation, - BuildRunContext, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.cost.models import CostRunRecord - - -def finalize_no_work_build_if_needed( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, -) -> bool: - """Persist and emit a completed zero-cost record when the plan has no work.""" - - if plan_has_executable_work( - plan=pipeline_result.plan_output, - python_plan_entries=pipeline_result.python_plan_entries, - ): - return False - if request.verbose or request.debug: - effective_concurrency: int = ( - request.concurrency - if request.concurrency is not None - else pipeline_result.project.settings.concurrency - ) - write_build_run_context( - stream=invocation.progress_stream, - context=BuildRunContext( - command="sqb build", - project=pipeline_result.project, - plan=pipeline_result.plan_output, - discovered_inputs=invocation.discovered_inputs, - python_plan_entries=pipeline_result.python_plan_entries, - connection_config=invocation.connection_config, - concurrency=effective_concurrency, - full_refresh=request.full_refresh, - selector_files=request.selector_files, - ), - use_color=invocation.use_color, - ) - completed_at: datetime = datetime.now(UTC) - cost_record: CostRunRecord | None = finalize_build_cost( - BuildCostFinalization( - project_dir=invocation.effective_project_dir, - adapter_name=invocation.adapter_name, - adapter=invocation.adapter, - connection_config=invocation.connection_config, - target_name=pipeline_result.project.effective_target_name, - target_database=pipeline_result.project.effective_target_database, - run_id=pipeline_result.project.run_id, - build_status="success", - started_at=completed_at, - completed_at=completed_at, - config=invocation.discovered_inputs.project_config.cost, - output_stream=invocation.progress_stream, - use_color=invocation.use_color, - collect=False, - had_executable_work=False, - ) - ) - write_no_work_build_json( - request=request, - pipeline_result=pipeline_result, - cost_record=cost_record, - ) - return True diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py b/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py deleted file mode 100644 index 93eb264389..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Build command plan text, runtime target, and completion output phases.""" - -from __future__ import annotations - -import sys -from typing import TextIO - -from sqlbuild._native import ( - cost_output_payload, - format_build_execution_json, - format_degradable_build_execution_json, - format_plan, - write_execution_json_output, -) -from sqlbuild.cli.commands._helpers.build_python_nodes.python_node_output import ( - python_node_results_failed, -) -from sqlbuild.cli.commands._helpers.check.core import check_results_failed -from sqlbuild.cli.commands.classes.build_progress_callbacks import format_build_footer -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildExecutionPreparation, - BuildInvocation, - BuildRunOutcome, -) -from sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target import ( - write_python_check_runtime_target, -) -from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.cost.models import CostRunRecord -from sqlbuild.executor.build.models import BuildExecutionResult -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.python_nodes.models import ( - PythonCheckExecutionResult, - PythonNodeExecutionResult, -) - - -def write_build_plan_text( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, -) -> None: - """Write the formatted plan text to the plan stream.""" - - plan_stream: TextIO = sys.stderr if request.debug or request.json_output else sys.stdout - plan_text: str = format_plan( - plan=pipeline_result.plan_output, - full_refresh=request.full_refresh, - use_color=invocation.use_color, - python_plan_entries=pipeline_result.python_plan_entries, - include_direct_freshness_diagnostics=(False), - ) - plan_stream.write("\n" + plan_text + "\n\n") - plan_stream.flush() - - -def write_build_runtime_targets( - *, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - outcome: BuildRunOutcome, - check_results: tuple[PythonCheckExecutionResult, ...], -) -> None: - """Write runtime and python-check runtime target artifacts.""" - - write_runtime_target( - target_dir=invocation.effective_project_dir / "target", - plan_output=pipeline_result.plan_output, - result=outcome.result, - ) - write_python_check_runtime_target( - target_dir=invocation.effective_project_dir / "target", - results=check_results, - ) - - -def write_build_completion_output( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - preparation: BuildExecutionPreparation, - outcome: BuildRunOutcome, - check_results: tuple[PythonCheckExecutionResult, ...], - cost_record: CostRunRecord | None = None, -) -> None: - """Write the build footer and optional execution JSON output.""" - - footer: str = format_build_footer( - result=outcome.result, - elapsed=preparation.callbacks.elapsed, - use_color=invocation.use_color, - python_node_results=outcome.python_results, - ) - invocation.progress_stream.write("\n" + footer + "\n") - invocation.progress_stream.flush() - execution_succeeded: bool = ( - resolve_build_exit_code(outcome=outcome, check_results=check_results) == 0 - ) - payload: str = execution_json_payload_with_degradation( - result=outcome.result, - plan=pipeline_result.plan_output, - python_node_results=outcome.python_results, - python_check_results=check_results, - command="build", - run_id=(None if cost_record is None else pipeline_result.project.run_id), - cost_record=cost_record, - execution_succeeded=execution_succeeded, - ) - write_execution_json_output( - payload=payload, - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def execution_json_payload_with_degradation( - *, - result: BuildExecutionResult, - plan: PlanOutput, - python_node_results: tuple[PythonNodeExecutionResult, ...], - python_check_results: tuple[PythonCheckExecutionResult, ...], - command: str, - run_id: str | None, - cost_record: CostRunRecord | None, - execution_succeeded: bool, -) -> str: - """Render the execution JSON payload, degrading on projection failures.""" - - return format_degradable_build_execution_json( - result=result, - plan=plan, - python_node_results=python_node_results, - python_check_results=python_check_results, - command=command, - run_id=run_id, - cost=(None if cost_record is None else cost_output_payload(record=cost_record)), - execution_succeeded=execution_succeeded, - ) - - -def write_no_work_build_json( - *, - request: BuildCommandRequest, - pipeline_result: CompilePipelineResult, - cost_record: CostRunRecord | None, -) -> None: - """Write a valid empty execution envelope for a no-work build.""" - - write_execution_json_output( - payload=format_build_execution_json( - result=BuildExecutionResult(status=BuildStatus.SUCCESS), - plan=pipeline_result.plan_output, - run_id=(None if cost_record is None else pipeline_result.project.run_id), - cost=(None if cost_record is None else cost_output_payload(record=cost_record)), - ), - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - - -def resolve_build_exit_code( - *, - outcome: BuildRunOutcome, - check_results: tuple[PythonCheckExecutionResult, ...], -) -> int: - """Resolve the build exit code from execution, python, and check results.""" - - python_failed: bool = python_node_results_failed(outcome.python_results) - checks_failed: bool = check_results_failed(check_results) - return ( - 0 - if outcome.result.status == BuildStatus.SUCCESS and not python_failed and not checks_failed - else 1 - ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py b/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py deleted file mode 100644 index de92b47eac..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Verbose final build phase timing output.""" - -from __future__ import annotations - -import time -from datetime import datetime -from typing import TextIO - -from sqlbuild._native import CliDocument -from sqlbuild.cli.commands._helpers.build_execution.exceptional_cost import ( - finalize_exceptional_direct_cost, -) -from sqlbuild.cli.commands._helpers.build_execution.no_work import ( - finalize_no_work_build_if_needed, -) -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildInvocation, - BuildPhaseTimings, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker -from sqlbuild.diagnostics.models import PartialBuildPhaseTimings -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def write_build_phase_timings( - *, stream: TextIO, timings: BuildPhaseTimings, use_color: bool -) -> None: - """Write available monotonic build phase durations.""" - - document: CliDocument = CliDocument(CliStyle(use_color=use_color)) - document.blank() - document.header(text="Phase timings") - document.fields(rows=_timing_rows(timings), label_width=24) - stream.write(document.render()) - stream.flush() - - -def write_partial_build_phase_timings( - *, stream: TextIO, timings: PartialBuildPhaseTimings, use_color: bool -) -> None: - """Write available exceptional build phase durations.""" - - write_build_phase_timings( - stream=stream, - timings=BuildPhaseTimings( - compile_seconds=timings.compile_seconds, - planning_seconds=timings.planning_seconds, - connection_preparation_seconds=timings.connection_preparation_seconds, - schema_preparation_seconds=timings.schema_preparation_seconds, - execution_seconds=timings.execution_seconds, - cost_collection_seconds=timings.cost_collection_seconds, - total_seconds=timings.total_seconds, - ), - use_color=use_color, - ) - - -def finish_cost_collection_timing(*, started_at: float) -> float | None: - """Retain cost collection elapsed time without affecting the active outcome.""" - - try: - elapsed_seconds: float = time.monotonic() - started_at - timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() - if timing_tracker is not None: - timing_tracker.cost_collection_seconds = elapsed_seconds - return elapsed_seconds - except BaseException: - return None - - -def finalize_no_work_with_timings( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - command_started_at: float, -) -> bool: - """Finalize no-work output and append verbose phase timings when complete.""" - - cost_started_at: float = time.monotonic() - try: - finalized: bool = finalize_no_work_build_if_needed( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - finally: - cost_collection_seconds: float | None = finish_cost_collection_timing( - started_at=cost_started_at - ) - if finalized and (request.verbose or request.debug): - write_build_phase_timings( - stream=invocation.progress_stream, - timings=BuildPhaseTimings( - compile_seconds=pipeline_result.compile_seconds, - planning_seconds=pipeline_result.planning_seconds, - cost_collection_seconds=cost_collection_seconds, - total_seconds=time.monotonic() - command_started_at, - ), - use_color=invocation.use_color, - ) - return finalized - - -def finalize_exceptional_with_timings( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - build_started_at: datetime, - command_started_at: float, - error: BaseException, -) -> None: - """Finalize exceptional cost state and available verbose phase timings.""" - - cost_started_at: float = time.monotonic() - try: - finalize_exceptional_direct_cost( - invocation=invocation, - pipeline_result=pipeline_result, - build_started_at=build_started_at, - error=error, - ) - finally: - _ = finish_cost_collection_timing(started_at=cost_started_at) - - -def _timing_rows(timings: BuildPhaseTimings) -> tuple[tuple[str, str], ...]: - values: tuple[tuple[str, float | None], ...] = ( - ("compile", timings.compile_seconds), - ("planning", timings.planning_seconds), - ("connection preparation", timings.connection_preparation_seconds), - ("schema preparation", timings.schema_preparation_seconds), - ("execution", timings.execution_seconds), - ("cost collection", timings.cost_collection_seconds), - ("total", timings.total_seconds), - ) - return tuple((label, f"{seconds:.2f}s") for label, seconds in values if seconds is not None) diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py b/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py deleted file mode 100644 index 4d5f14ae85..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Resolved verbose build-start context output.""" - -from __future__ import annotations - -from datetime import date, datetime -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import CliDocument, describe_connection_warehouse -from sqlbuild.cli.commands.constants import ( - C0_CONTROL_CODE_LIMIT, - C1_CONTROL_CODE_LIMIT, - C1_CONTROL_CODE_START, -) -from sqlbuild.cli.commands.models import BuildRunContext -from sqlbuild.cli.entry.models import ResolvedWarehouse -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def write_build_run_context(*, stream: TextIO, context: BuildRunContext, use_color: bool) -> None: - """Write resolved non-secret build placement, mode, and selection context.""" - - stream.write(_format_build_run_context(context=context, use_color=use_color)) - stream.flush() - - -def _format_build_run_context(*, context: BuildRunContext, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - document: CliDocument = CliDocument(style) - selected_count: int = ( - len(context.plan.model_entries) - + len(context.plan.seed_entries) - + len(context.plan.function_entries) - + len(context.plan.source_load_entries) - + len(context.python_plan_entries) - ) - total_source_load_count: int = sum( - source.source_entry.loader is not None for source in context.project.sources - ) - total_count: int = ( - len(context.project.models) - + len(context.project.seeds) - + len(context.project.functions) - + total_source_load_count - + len(context.discovered_inputs.task_functions) - + len(context.discovered_inputs.asset_functions) - ) - placement_rows: tuple[tuple[str, str], ...] - placement_rows = (("schema", _display_value(context.project.effective_target_schema)),) - document.header(text="Execution") - document.fields( - rows=( - ("command", _display_value(context.command)), - ("run_id", _display_value(context.project.run_id)), - ("target", _display_value(context.project.effective_target_name)), - ("database", _display_value(context.project.effective_target_database)), - *placement_rows, - ("warehouse", _warehouse_display(context)), - ("concurrency", f"{context.concurrency} configured limit"), - ("full_refresh", str(context.full_refresh).lower()), - ("selected", f"{selected_count:,} of {total_count:,} build resources"), - ("date vars", _date_vars_display(context.project.effective_vars)), - ), - label_width=12, - ) - if context.selector_files: - document.blank() - document.header(text="Selection files") - for selector_file in context.selector_files: - selector_noun: str = "selector" if selector_file.selector_count == 1 else "selectors" - document.fields( - rows=( - ( - "selector_file", - f"{_safe_path(selector_file.path)} " - f"({selector_file.selector_count:,} {selector_noun})", - ), - ), - label_width=12, - ) - document.blank() - return document.render() - - -def _warehouse_display(context: BuildRunContext) -> str: - resolved: ResolvedWarehouse = describe_connection_warehouse( - discovered_inputs=context.discovered_inputs, - selected_target=context.project.effective_target_name, - connection_config=context.connection_config, - ) - if resolved.warehouse is None: - return _display_value(None) - return f"{_display_value(resolved.warehouse)} ({resolved.source})" - - -def _display_value(value: object) -> str: - display_value: str | None = _safe_display_value(value) - return display_value if display_value is not None else "not set" - - -def _safe_display_value(value: object) -> str | None: - if isinstance(value, str | int | float) and not isinstance(value, bool): - return _sanitize(str(value)) - return None - - -def _safe_path(path: Path) -> str: - return _sanitize(str(path)) - - -def _sanitize(value: str) -> str: - return "".join( - f"\\x{ord(character):02x}" - if ord(character) < C0_CONTROL_CODE_LIMIT - or C1_CONTROL_CODE_START <= ord(character) < C1_CONTROL_CODE_LIMIT - else character - for character in value - ) - - -def _validated_iso_value(value: object) -> str | None: - if not isinstance(value, str): - return None - try: - date.fromisoformat(value) - except ValueError: - try: - datetime.fromisoformat(value) - except ValueError: - return None - return _sanitize(value) - - -def _date_vars_display(effective_vars: dict[str, object]) -> str: - start_date: str | None = _validated_iso_value(effective_vars.get("start_date")) - end_date: str | None = _validated_iso_value(effective_vars.get("end_date")) - if start_date is None and end_date is None: - return "not set" - if start_date is None: - return f"end_date={end_date}" - if end_date is None: - return f"start_date={start_date}" - return f"{start_date} to {end_date}" diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py b/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py deleted file mode 100644 index 5209610623..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Build command compile target writing phase.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild._native import write_compile_target -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildInvocation, -) -from sqlbuild.compiler.manifest.main.build import build_manifest -from sqlbuild.compiler.pipeline.models import CompilePipelineResult - - -def write_build_compile_target( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, -) -> None: - """Write the compile target artifacts with the optional manifest payload.""" - - manifest_payload: _native.NativeManifest | None = None - if request.manifest: - manifest_payload = build_manifest( - project=pipeline_result.project, - plan_output=pipeline_result.plan_output, - project_name=invocation.discovered_inputs.project_config.name, - adapter_type=invocation.adapter_name, - upstream_deps=pipeline_result.plan_output.upstream_deps, - downstream_deps=pipeline_result.plan_output.downstream_deps, - ) - write_compile_target( - target_dir=invocation.effective_project_dir / "target", - adapter=invocation.adapter, - plan_output=pipeline_result.plan_output, - manifest=manifest_payload, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py deleted file mode 100644 index 7080ea11cb..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/confirmation.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Typed confirmation shared by the build safety policies.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.commands.exceptions import CliUserError - - -def confirm_typed_action( - *, - action: str, - non_interactive_help: str, - warning: str, - expected: str, - input_stream: TextIO, - output_stream: TextIO, - code: str | None = None, -) -> None: - """Ask the user to type the expected text, or fail when the run is not interactive.""" - - if not input_stream.isatty(): - raise CliUserError( - f"{action} requires confirmation", - code=code, - help=non_interactive_help, - ) - _ = output_stream.write(f"{warning}\n\n") - _ = output_stream.write(f"Type `{expected}` to continue: ") - output_stream.flush() - if input_stream.readline().strip() != expected: - raise CliUserError(f"{action} cancelled", code=code) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py index 148a3b2d0b..b48b7f91a2 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/defer_clone.py @@ -6,11 +6,8 @@ from pathlib import Path from typing import Any -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, enforce_model_execution_limit from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( - enforce_model_execution_limit, -) from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/execution_limits.py b/src/sqlbuild/cli/commands/_helpers/build_planning/execution_limits.py deleted file mode 100644 index f73b3b6d87..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/execution_limits.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Target-scoped build execution-limit enforcement.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.planner.types import PlanAction -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig - - -def executable_model_count(*, plan: PlanOutput) -> int: - """Return the number of model entries that would execute.""" - - return sum(entry.action != PlanAction.SKIP for entry in plan.model_entries) - - -def enforce_model_execution_limit( - *, - model_count: int, - target_name: str | None, - limits: ExecutionLimitsConfig, - refusal_note: str, -) -> None: - """Reject a build whose expanded model count exceeds its target policy.""" - - maximum: int | None = limits.max_models - if maximum is None or model_count <= maximum: - return - target_label: str = target_name or "default" - raise CliUserError( - "Build plan exceeds the configured model limit\n\n" - f"Target: {target_label}\n" - f"Selected models: {model_count}\n" - f"Maximum models: {maximum}\n" - "Configuration: " - f"targets.{target_label}.execution_limits.max_models\n\n" - f"{refusal_note}", - code="C413", - help=limits.remediation, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/full_refresh.py b/src/sqlbuild/cli/commands/_helpers/build_planning/full_refresh.py deleted file mode 100644 index 0dcce87c5a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/full_refresh.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Full-refresh safety enforcement for build commands.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import ( - MaterializationType, - PlanReason, - SnapshotFullRefreshPolicy, -) -from sqlbuild.spec.contracts.models import SnapshotsConfig - -_RULES_STRICTNESS: dict[SnapshotFullRefreshPolicy, int] = { - SnapshotFullRefreshPolicy.ALLOW: 0, - SnapshotFullRefreshPolicy.REQUIRE_CONFIRMATION: 1, - SnapshotFullRefreshPolicy.DENY: 2, -} - - -def enforce_snapshot_full_refresh_policy( - *, - plan: PlanOutput, - snapshots_config: SnapshotsConfig, - allow_snapshot_full_refresh: bool, - non_interactive_help: str, - input_stream: TextIO, - output_stream: TextIO, -) -> None: - """Fail or confirm before executing protected full-refresh entries.""" - - snapshot_entries: tuple[ModelPlanEntry, ...] = tuple( - entry - for entry in plan.model_entries - if entry.materialization_type == MaterializationType.SNAPSHOT - and entry.reason == PlanReason.FULL_REFRESH - ) - if not snapshot_entries: - return - - denied: tuple[ModelPlanEntry, ...] = tuple( - entry - for entry in snapshot_entries - if _effective_policy(entry=entry, snapshots_config=snapshots_config) - == SnapshotFullRefreshPolicy.DENY - ) - if denied: - names: str = _model_names(denied) - raise CliUserError( - f"full refresh is denied for snapshot model {names} by snapshot_full_refresh policy", - code="C238", - help=( - "Set a less strict project/model snapshot_full_refresh policy only if " - "the snapshot history is recoverable." - ), - ) - - confirmation_required: tuple[ModelPlanEntry, ...] = tuple( - entry - for entry in snapshot_entries - if _effective_policy(entry=entry, snapshots_config=snapshots_config) - == SnapshotFullRefreshPolicy.REQUIRE_CONFIRMATION - ) - if not confirmation_required or allow_snapshot_full_refresh: - return - - if not input_stream.isatty(): - raise CliUserError( - "snapshot full refresh requires confirmation", - code="C239", - help=non_interactive_help, - ) - - if not _confirm( - entries=confirmation_required, input_stream=input_stream, output_stream=output_stream - ): - raise CliUserError("snapshot full refresh cancelled", code="C240") - - -def _effective_policy( - *, entry: ModelPlanEntry, snapshots_config: SnapshotsConfig -) -> SnapshotFullRefreshPolicy: - project_policy: SnapshotFullRefreshPolicy = SnapshotFullRefreshPolicy( - snapshots_config.historical_full_refresh - if entry.observed_at_column is not None - else snapshots_config.current_state_full_refresh - ) - if entry.snapshot_full_refresh is None: - return project_policy - model_policy: SnapshotFullRefreshPolicy = SnapshotFullRefreshPolicy(entry.snapshot_full_refresh) - return max((project_policy, model_policy), key=lambda policy: _RULES_STRICTNESS[policy]) - - -def _confirm( - *, entries: tuple[ModelPlanEntry, ...], input_stream: TextIO, output_stream: TextIO -) -> bool: - expected: str = _confirmation_text(entries) - names: str = _model_names(entries) - output_stream.write( - f"Full refresh of snapshot model {names} may permanently discard unrecoverable history.\n\n" - ) - output_stream.write(f"Type `{expected}` to continue: ") - output_stream.flush() - response: str = input_stream.readline().strip() - return response == expected - - -def _confirmation_text(entries: tuple[ModelPlanEntry, ...]) -> str: - if len(entries) == 1: - return f"discard snapshot history for {entries[0].name}" - return f"discard snapshot history for {len(entries)} models" - - -def _model_names(entries: tuple[ModelPlanEntry, ...]) -> str: - if len(entries) == 1: - return f"'{entries[0].name}'" - return ", ".join(f"'{entry.name}'" for entry in entries) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py deleted file mode 100644 index 3a40e4779c..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Build command invocation resolution phase.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import discover_project_inputs, resolve_project_connection_config -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildInvocation, -) -from sqlbuild.cli.progress.main._build_command_progress_reporters import ( - build_command_progress_reporters, -) -from sqlbuild.cli.progress.models import CommandProgressReporters -from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, TargetConfig - - -def resolve_build_invocation(*, request: BuildCommandRequest) -> BuildInvocation: - """Resolve discovery, validations, adapter, connection, and reporters for build.""" - - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir, extract_output_column_locations=False - ) - effective_defer_clone_from: str | None = _resolve_defer_clone_from( - discovered_inputs=discovered_inputs, - selected_target=request.selected_target, - cli_defer_clone_from=request.defer_clone_from, - ) - if request.defer_to is not None and effective_defer_clone_from is not None: - raise CliUserError("--defer-clone-from cannot be used with --defer-to", code="C408") - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - effective_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=request.selected_target, - ) - execution_limits: ExecutionLimitsConfig = _resolve_execution_limits( - discovered_inputs=discovered_inputs, - target_name=effective_target_name, - adapter_name=adapter_name, - adapter=adapter, - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - selected_target=request.selected_target, - cli_vars=request.cli_vars, - ) - machine_output: bool = request.json_output - use_color: bool = not request.no_color and not machine_output and supports_color() - progress_stream: TextIO = sys.stderr if request.debug or machine_output else sys.stdout - reporters: CommandProgressReporters = build_command_progress_reporters( - adapter_name=adapter_name, - stream=progress_stream, - use_color=use_color, - ) - should_load_sources: bool = request.reload_sources or ( - request.load_sources - if request.load_sources is not None - else build_effective_settings_config(discovered_inputs=discovered_inputs).auto_load_sources - ) - return BuildInvocation( - effective_project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - effective_defer_clone_from=effective_defer_clone_from, - adapter_name=adapter_name, - adapter=adapter, - connection_config=connection_config, - use_color=use_color, - progress_stream=progress_stream, - connection_progress=reporters.connection, - planning_progress=reporters.planning, - should_load_sources=should_load_sources, - effective_target_name=effective_target_name, - execution_limits=execution_limits, - ) - - -def _resolve_execution_limits( - *, - discovered_inputs: DiscoveredProjectInputs, - target_name: str | None, - adapter_name: str, - adapter: BaseAdapter, -) -> ExecutionLimitsConfig: - """Resolve the target's execution limits and validate them against the project adapter.""" - - execution_limits: ExecutionLimitsConfig = ( - resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ).execution_limits - if target_name is not None - else ExecutionLimitsConfig() - ) - target_label: str = target_name or "default" - if ( - execution_limits.max_duration is not None - and adapter.execution_duration_limit_seconds is None - ): - raise CliUserError( - f"Target '{target_label}' configures max_duration, but adapter " - f"'{adapter_name}' cannot cancel active statements safely", - code="C415", - help=execution_limits.remediation, - ) - maximum_duration_seconds: int | None = adapter.execution_duration_limit_seconds - if ( - execution_limits.max_duration_seconds is not None - and maximum_duration_seconds is not None - and execution_limits.max_duration_seconds > maximum_duration_seconds - ): - raise CliUserError( - f"Target '{target_label}' max_duration exceeds the largest duration supported by " - f"adapter '{adapter_name}' ({maximum_duration_seconds} seconds)", - code="C416", - help=execution_limits.remediation, - ) - return execution_limits - - -def _resolve_defer_clone_from( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - cli_defer_clone_from: str | None, -) -> str | None: - if cli_defer_clone_from is not None: - return cli_defer_clone_from - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - if target_name is None: - return None - target_config: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - return target_config.defer_clone_from diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py b/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py deleted file mode 100644 index dbbbaac063..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/plan_policies.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Plan safety policies enforced before a direct build executes anything.""" - -from __future__ import annotations - -import sys -from typing import TextIO - -from sqlbuild.cli.commands._helpers.build_planning.confirmation import confirm_typed_action -from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( - enforce_model_execution_limit, - executable_model_count, -) -from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( - enforce_snapshot_full_refresh_policy, -) -from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( - enforce_retention_decrease_policy, -) -from sqlbuild.cli.commands._helpers.build_planning.table_type import ( - enforce_table_type_downgrade_policy, -) -from sqlbuild.cli.commands.constants import ( - EXECUTION_LIMIT_BUILD_NOTE, - EXECUTION_LIMIT_DBT_NOTE, - MISSING_ORIGIN_BUILD_HELP, - MISSING_ORIGIN_DBT_HELP, - RETENTION_DECREASE_BUILD_HELP, - RETENTION_DECREASE_DBT_HELP, - SNAPSHOT_FULL_REFRESH_BUILD_HELP, - SNAPSHOT_FULL_REFRESH_DBT_HELP, - TABLE_TYPE_DOWNGRADE_BUILD_HELP, - TABLE_TYPE_DOWNGRADE_DBT_HELP, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import BuildCommandRequest, BuildInvocation, PlanSafetyGates -from sqlbuild.compiler.migrations.types import ColumnMigrationDecision, MigrationDecision -from sqlbuild.compiler.planner.constants import HIDDEN_ORIGIN_REMEDY -from sqlbuild.compiler.planner.models import ( - ColumnMigrationPlanEntry, - ModelMigrationPlanEntry, - PlanOutput, - PlanWarning, -) -from sqlbuild.compiler.planner.types import WarningSeverity -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig -from sqlbuild.spec.contracts.types import MissingMigrationOriginPolicy - -_OLD_NAME_CODES: frozenset[str] = frozenset({"M114", "P008"}) -_COLUMN_BLOCK_CODES: dict[ColumnMigrationDecision, str] = { - ColumnMigrationDecision.SOURCE_MISSING: "M109", - ColumnMigrationDecision.CONFLICT: "M110", - ColumnMigrationDecision.STILL_PRODUCED: "M111", - ColumnMigrationDecision.UNSUPPORTED: "M112", -} - - -def enforce_build_plan_policies( - *, request: BuildCommandRequest, invocation: BuildInvocation, plan: PlanOutput -) -> None: - """Apply every plan safety gate before a direct build executes anything.""" - - enforce_plan_safety_policies( - plan=plan, - gates=PlanSafetyGates( - allow_missing_migration_origin=request.allow_missing_migration_origin, - allow_snapshot_full_refresh=request.allow_snapshot_full_refresh, - allow_table_type_downgrade=request.allow_table_type_downgrade, - allow_retention_decrease=request.allow_retention_decrease, - missing_origin_help=MISSING_ORIGIN_BUILD_HELP, - snapshot_full_refresh_help=SNAPSHOT_FULL_REFRESH_BUILD_HELP, - table_type_downgrade_help=TABLE_TYPE_DOWNGRADE_BUILD_HELP, - retention_decrease_help=RETENTION_DECREASE_BUILD_HELP, - execution_limit_note=EXECUTION_LIMIT_BUILD_NOTE, - ), - snapshots_config=invocation.discovered_inputs.project_config.snapshots, - target_name=invocation.effective_target_name, - execution_limits=invocation.execution_limits, - streams=(sys.stdin, sys.stdout), - ) - - -def dbt_interop_plan_safety_gates() -> PlanSafetyGates: - """Gates for sqb dbt, which has no allow flags and confirms only on a terminal.""" - - return PlanSafetyGates( - allow_missing_migration_origin=False, - allow_snapshot_full_refresh=False, - allow_table_type_downgrade=False, - allow_retention_decrease=False, - missing_origin_help=MISSING_ORIGIN_DBT_HELP, - snapshot_full_refresh_help=SNAPSHOT_FULL_REFRESH_DBT_HELP, - table_type_downgrade_help=TABLE_TYPE_DOWNGRADE_DBT_HELP, - retention_decrease_help=RETENTION_DECREASE_DBT_HELP, - execution_limit_note=EXECUTION_LIMIT_DBT_NOTE, - ) - - -def enforce_plan_safety_policies( - *, - plan: PlanOutput, - gates: PlanSafetyGates, - snapshots_config: SnapshotsConfig, - target_name: str | None, - execution_limits: ExecutionLimitsConfig, - streams: tuple[TextIO, TextIO], -) -> None: - """Apply migration, execution-limit, and storage safety gates in their required order.""" - - input_stream, output_stream = streams - enforce_migration_plan_policies( - plan=plan, - allow_missing_migration_origin=gates.allow_missing_migration_origin, - non_interactive_help=gates.missing_origin_help, - input_stream=input_stream, - output_stream=output_stream, - ) - _enforce_old_name_policy(plan=plan) - enforce_model_execution_limit( - model_count=executable_model_count(plan=plan), - target_name=target_name, - limits=execution_limits, - refusal_note=gates.execution_limit_note, - ) - enforce_snapshot_full_refresh_policy( - plan=plan, - snapshots_config=snapshots_config, - allow_snapshot_full_refresh=gates.allow_snapshot_full_refresh, - non_interactive_help=gates.snapshot_full_refresh_help, - input_stream=input_stream, - output_stream=output_stream, - ) - enforce_table_type_downgrade_policy( - plan=plan, - allow_table_type_downgrade=gates.allow_table_type_downgrade, - non_interactive_help=gates.table_type_downgrade_help, - input_stream=input_stream, - output_stream=output_stream, - ) - enforce_retention_decrease_policy( - plan=plan, - allow_retention_decrease=gates.allow_retention_decrease, - non_interactive_help=gates.retention_decrease_help, - input_stream=input_stream, - output_stream=output_stream, - ) - - -def enforce_migration_plan_policies( - *, - plan: PlanOutput, - allow_missing_migration_origin: bool, - non_interactive_help: str, - input_stream: TextIO, - output_stream: TextIO, -) -> None: - """Refuse blocked model and column migrations, then confirm building past missing origins.""" - - _enforce_model_migration_policy(plan=plan) - _enforce_column_migration_policy(plan=plan) - _confirm_missing_origins( - plan=plan, - allow_missing_migration_origin=allow_missing_migration_origin, - non_interactive_help=non_interactive_help, - input_stream=input_stream, - output_stream=output_stream, - ) - - -def _enforce_model_migration_policy(*, plan: PlanOutput) -> None: - """Refuse to build while any planned migration conflicts or is incompatible.""" - - blocked: tuple[ModelMigrationPlanEntry, ...] = tuple( - entry for entry in plan.migration_entries if entry.blocks_build - ) - if not blocked: - return - hidden: tuple[ModelMigrationPlanEntry, ...] = tuple( - entry for entry in blocked if entry.origin_hidden - ) - if hidden: - raise CliUserError( - "model migration origin was built in this target but is not visible for " - + _origins(hidden) - + f"; {HIDDEN_ORIGIN_REMEDY}", - code="M102", - help="Run sqb plan to see every model migration decision.", - ) - missing: tuple[ModelMigrationPlanEntry, ...] = tuple( - entry for entry in blocked if entry.decision == MigrationDecision.ORIGIN_MISSING - ) - if missing: - raise CliUserError( - "model migration origin does not exist for " - + _origins(missing) - + " and no recorded migration into it was found, and this target sets " - "missing_migration_origin = deny; if the migration already happened elsewhere or is " - "no longer needed, remove migrate_from from the model header", - code="M102", - help="Run sqb plan to see every model migration decision.", - ) - conflicts: tuple[ModelMigrationPlanEntry, ...] = tuple( - entry for entry in blocked if entry.decision == MigrationDecision.CONFLICT - ) - names: str = ", ".join(f"'{entry.model_name}'" for entry in blocked) - if conflicts: - raise CliUserError( - f"model migration conflict for {names}: the destination already exists with its " - "own build history and no recorded migration", - code="M103", - help=( - "Set migrate_force true on the model to replace the destination; the replaced " - "table remains recoverable through warehouse time travel." - ), - ) - raise CliUserError( - f"model migration source is incompatible with {names}", - code="M104", - help="Run sqb plan to see the blocking schema differences.", - ) - - -def _origins(entries: tuple[ModelMigrationPlanEntry, ...]) -> str: - return ", ".join( - f"'{entry.model_name}' (migrate_from {entry.origin.qualified_name or entry.origin.name})" - for entry in entries - ) - - -def _enforce_column_migration_policy(*, plan: PlanOutput) -> None: - """Refuse to build while any declared column rename cannot be applied safely.""" - - blocked: tuple[ColumnMigrationPlanEntry, ...] = tuple( - entry for entry in plan.column_migration_entries if entry.blocks_build - ) - if not blocked: - return - raise CliUserError( - "column migrations are blocked for " - + ", ".join( - f"'{entry.model_name}' ({entry.destination_column} migrate_from " - f"{entry.origin_column}: {entry.decision.label})" - for entry in blocked - ), - code=_COLUMN_BLOCK_CODES.get(blocked[0].decision, "M109"), - help="Run sqb plan to see why each column rename cannot be applied.", - ) - - -def _confirm_missing_origins( - *, - plan: PlanOutput, - allow_missing_migration_origin: bool, - non_interactive_help: str, - input_stream: TextIO, - output_stream: TextIO, -) -> None: - """Confirm before building past declared migrations whose origin is missing.""" - - names: tuple[str, ...] = tuple( - sorted( - { - entry.model_name - for entry in (*plan.migration_entries, *plan.column_migration_entries) - if entry.origin_missing - and not entry.blocks_build - and entry.missing_origin_policy == MissingMigrationOriginPolicy.REQUIRE_CONFIRMATION - } - ) - ) - if not names or allow_missing_migration_origin: - return - listed: str = ", ".join(f"'{name}'" for name in names) - confirm_typed_action( - action=f"building {listed} without its missing migrate_from origin", - non_interactive_help=non_interactive_help, - warning=f"The migrate_from origin of {listed} does not exist in this target; nothing " - "will be migrated.", - expected=( - f"build {names[0]} without its migration" - if len(names) == 1 - else f"build {len(names)} models without their migrations" - ), - input_stream=input_stream, - output_stream=output_stream, - code="M102", - ) - - -def _enforce_old_name_policy(*, plan: PlanOutput) -> None: - """Refuse to build over a compatibility view or while code reads an old name.""" - - errors: tuple[PlanWarning, ...] = tuple( - warning - for warning in plan.warnings - if warning.severity == WarningSeverity.ERROR and warning.code in _OLD_NAME_CODES - ) - if not errors: - return - raise CliUserError( - "; ".join(warning.message for warning in errors), - code=errors[0].code or "M114", - help="Run sqb plan to see every compatibility view and old-name reference.", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py b/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py deleted file mode 100644 index 4d2469c00a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/planning.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Build command compile pipeline phase.""" - -from __future__ import annotations - -from sqlbuild._native import run_compile_pipeline -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildInvocation, -) -from sqlbuild.cli.progress.main.verbose_inspection_output import verbose_inspection_output -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, -) -from sqlbuild.runtime.contracts.models import ConnectionHooks - - -def compile_build_plan( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, -) -> CompilePipelineResult: - """Run the compile pipeline for the build command.""" - - invocation.progress_stream.write("\n") - invocation.progress_stream.flush() - with verbose_inspection_output(enabled=request.verbose): - return _run_build_compile_pipeline(request=request, invocation=invocation) - - -def _run_build_compile_pipeline( - *, request: BuildCommandRequest, invocation: BuildInvocation -) -> CompilePipelineResult: - return run_compile_pipeline( - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - options=CompilePipelineOptions( - no_sql_validation=request.no_sql_validation, - no_cache=request.no_cache, - selected_target=request.selected_target, - defer_to=request.defer_to, - defer_sources_to=request.defer_sources_to, - cursor_overrides=request.cursor_overrides, - select=request.select, - exclude=request.exclude, - full_refresh=request.full_refresh, - auto_load_sources=invocation.should_load_sources, - reload_sources=request.reload_sources, - connection_config=invocation.connection_config, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - resolve_python_run_selectors=(request.include_python or invocation.should_load_sources), - max_microbatches=request.max_microbatches, - selection_diagnostics=request.selection_diagnostics, - plan_sql_tests=request.run_tests, - accepts_unit_test_selectors=True, - ), - hooks=ConnectionHooks( - on_progress=invocation.planning_progress.on_progress, - on_connection_start=invocation.connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - invocation.connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - invocation.connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/retention_decrease.py b/src/sqlbuild/cli/commands/_helpers/build_planning/retention_decrease.py deleted file mode 100644 index c321c59df4..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/retention_decrease.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Time-travel retention decrease safety enforcement for build commands.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.planner.models import PlanOutput, RetentionPlanEntry -from sqlbuild.spec.contracts.types import RetentionDecreasePolicy - - -def enforce_retention_decrease_policy( - *, - plan: PlanOutput, - allow_retention_decrease: bool, - non_interactive_help: str, - input_stream: TextIO, - output_stream: TextIO, -) -> None: - """Fail or confirm before lowering live time-travel retention.""" - - entries: tuple[RetentionPlanEntry, ...] = tuple( - entry for entry in plan.retention_entries if entry.decreases - ) - denied: tuple[RetentionPlanEntry, ...] = tuple( - entry for entry in entries if entry.decrease_policy == RetentionDecreasePolicy.DENY - ) - if denied: - raise CliUserError( - f"time travel retention decrease is denied for {_names(denied)}", - help=( - "Keep the current retention, or set the target's " - "time_travel_retention_decrease to 'require_confirmation' or 'allow'." - ), - ) - confirmation: tuple[RetentionPlanEntry, ...] = tuple( - entry - for entry in entries - if entry.decrease_policy == RetentionDecreasePolicy.REQUIRE_CONFIRMATION - ) - if not confirmation or allow_retention_decrease: - return - if not input_stream.isatty(): - raise CliUserError( - "time travel retention decrease requires confirmation", - help=non_interactive_help, - ) - expected: str = _confirmation_text(confirmation) - output_stream.write( - f"Decreasing time travel retention for {_names(confirmation)} permanently discards " - "history older than the new retention.\n\n" - ) - output_stream.write(f"Type `{expected}` to continue: ") - output_stream.flush() - if input_stream.readline().strip() != expected: - raise CliUserError("time travel retention decrease cancelled") - - -def _model_names(entries: tuple[RetentionPlanEntry, ...]) -> tuple[str, ...]: - names: dict[str, None] = {} - for entry in entries: - names.update(dict.fromkeys(entry.model_names)) - return tuple(names) - - -def _confirmation_text(entries: tuple[RetentionPlanEntry, ...]) -> str: - names: tuple[str, ...] = _model_names(entries) - if len(names) == 1: - return f"decrease retention for {names[0]}" - return f"decrease retention for {len(names)} models" - - -def _names(entries: tuple[RetentionPlanEntry, ...]) -> str: - return ", ".join(f"'{name}'" for name in _model_names(entries)) diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/table_type.py b/src/sqlbuild/cli/commands/_helpers/build_planning/table_type.py deleted file mode 100644 index 146af850fd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/table_type.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Table-type downgrade safety enforcement for build commands.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.planner.models import PlanOutput, TableTypePlanEntry -from sqlbuild.spec.contracts.types import TableTypeDowngradePolicy - - -def enforce_table_type_downgrade_policy( - *, - plan: PlanOutput, - allow_table_type_downgrade: bool, - non_interactive_help: str, - input_stream: TextIO, - output_stream: TextIO, -) -> None: - """Fail or confirm before executing permanent-to-transient conversions.""" - - entries: tuple[TableTypePlanEntry, ...] = tuple( - entry for entry in plan.table_type_entries if entry.downgrade - ) - denied: tuple[TableTypePlanEntry, ...] = tuple( - entry for entry in entries if entry.downgrade_policy == TableTypeDowngradePolicy.DENY - ) - if denied: - raise CliUserError( - f"table-type downgrade is denied for {_names(denied)} by table_type_downgrade policy" - ) - confirmation: tuple[TableTypePlanEntry, ...] = tuple( - entry - for entry in entries - if entry.downgrade_policy == TableTypeDowngradePolicy.REQUIRE_CONFIRMATION - ) - if not confirmation or allow_table_type_downgrade: - return - if not input_stream.isatty(): - raise CliUserError( - "table-type downgrade requires confirmation", - help=non_interactive_help, - ) - expected: str = _confirmation_text(confirmation) - output_stream.write( - f"Downgrading {_names(confirmation)} to transient may discard up to 90 days of " - "time-travel history.\n\n" - ) - output_stream.write(f"Type `{expected}` to continue: ") - output_stream.flush() - if input_stream.readline().strip() != expected: - raise CliUserError("table-type downgrade cancelled") - - -def _confirmation_text(entries: tuple[TableTypePlanEntry, ...]) -> str: - if len(entries) == 1: - return f"downgrade table type for {entries[0].model_name}" - return f"downgrade table type for {len(entries)} models" - - -def _names(entries: tuple[TableTypePlanEntry, ...]) -> str: - return ", ".join(f"'{entry.model_name}'" for entry in entries) diff --git a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/__init__.py b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py deleted file mode 100644 index 35e3b8147a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle.py +++ /dev/null @@ -1,224 +0,0 @@ -"""Direct-mode Python lifecycle orchestration for build commands.""" - -from __future__ import annotations - -from collections.abc import Callable -from datetime import datetime -from typing import Any, TextIO - -from sqlbuild._native import ( - build_project_relation_targets, - build_python_relation_targets, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.build_python_nodes.python_lifecycle_selection import ( - python_node_result_names, - scheduled_sql_resource_names, - task_asset_python_node_names, -) -from sqlbuild.cli.commands._helpers.build_python_nodes.python_node_output import ( - write_python_node_results, -) -from sqlbuild.cli.commands.classes.direct_python_lifecycle_state import ( - DirectPythonLifecycleState, -) -from sqlbuild.cli.commands.models import DirectLifecycleCallbacks -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.main.selection.loader_dag import build_intermediate_source_map -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.main.run_lifecycle import build_python_sql_run_lifecycle -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.python_nodes.main.ingress import run_ingress_python_loader_nodes -from sqlbuild.executor.python_nodes.main.read_side import create_read_side_python_execution_tracker -from sqlbuild.executor.python_nodes.models import ( - CursorWindow, - IngressCallbacks, - PythonIngressLoaderExecutorResult, - PythonNodeExecutionResult, - PythonNodeRuntime, -) -from sqlbuild.provider.main.runtime import ProviderContainer -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.runtime.contracts.types import NodeStartCallback -from sqlbuild.spec.contracts.models import SourceEntry - - -def prepare_direct_python_lifecycle( - *, - discovered_inputs: DiscoveredProjectInputs, - pipeline_result: CompilePipelineResult, - plan_output: PlanOutput, - adapter: BaseAdapter, - connection_config: dict[str, object], - include_python: bool, - reload_sources: bool, - cursor_window: CursorWindow, - callbacks: DirectLifecycleCallbacks, - providers: ProviderContainer | None = None, -) -> DirectPythonLifecycleState: - """Execute ingress Python and prepare read-side dispatch for direct run/build.""" - - start_cursor_ts: datetime | None = cursor_window.start_cursor_ts - end_cursor_ts: datetime | None = cursor_window.end_cursor_ts - start_cursor_int: int | None = cursor_window.start_cursor_int - end_cursor_int: int | None = cursor_window.end_cursor_int - use_color: bool = callbacks.use_color - progress_stream: TextIO = callbacks.progress_stream - on_node_start: NodeStartCallback | None = callbacks.on_node_start - on_node_complete: Callable[[object], None] = callbacks.on_node_complete - selected_task_asset_names: frozenset[str] = ( - task_asset_python_node_names( - selected_names=pipeline_result.python_node_names, - discovered_inputs=discovered_inputs, - ) - if include_python - else frozenset() - ) - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=discovered_inputs - ) - lifecycle_plan: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle( - selection=PythonSqlRunSelection( - sql_keys=frozenset(plan_output.upstream_deps), - python_node_names=pipeline_result.python_node_names, - ), - python_graph=python_graph, - ) - relation_targets: dict[SqlResourceRef, str] = build_python_relation_targets( - adapter=adapter, - project=pipeline_result.project, - plan_output=plan_output, - required_refs=python_graph.selected_sql_refs( - selected_names=( - lifecycle_plan.ingress_python_node_names - | lifecycle_plan.read_side_python_node_names - ), - ), - ) - project_relations: dict[SqlResourceRef, str] | None = build_project_relation_targets( - adapter=adapter, plan_output=plan_output - ) - default_database: str | None = pipeline_result.project.effective_target_database - if default_database is None: - default_database = adapter.default_database() - default_schema: str | None = pipeline_result.project.effective_target_schema - if default_schema is None: - default_schema = adapter.default_schema() - ingress_source_map: dict[str, SourceEntry] = dict(plan_output.source_map) - ingress_source_map.update( - build_intermediate_source_map( - project=pipeline_result.project, - selected_keys=frozenset( - CompiledObjectKey(name=loader_name, resource_type=CompiledResourceType.SOURCE) - for loader_name in lifecycle_plan.ingress_loader_names - ), - ) - ) - ingress_python_results: tuple[PythonNodeExecutionResult, ...] = () - ingress_load_results: tuple[LoadExecutionResult, ...] = () - if lifecycle_plan.ingress_python_node_names: - ingress_connection: object = adapter.connect(connection_config) - try: - ingress_result: PythonIngressLoaderExecutorResult = run_ingress_python_loader_nodes( - python_graph=python_graph, - selected_python_names=lifecycle_plan.ingress_python_node_names, - loader_functions=discovered_inputs.loader_functions, - source_map=ingress_source_map, - runtime=PythonNodeRuntime( - adapter=adapter, - connection_config=connection_config, - connection=ingress_connection, - run_id=pipeline_result.project.run_id, - target=pipeline_result.project.effective_target_name, - vars=pipeline_result.project.effective_vars, - is_reload=reload_sources, - default_database=default_database, - default_schema=default_schema, - start_cursor_ts=start_cursor_ts, - end_cursor_ts=end_cursor_ts, - start_cursor_int=start_cursor_int, - end_cursor_int=end_cursor_int, - relation_targets=relation_targets, - project_relations=project_relations, - providers=providers, - ), - callbacks=IngressCallbacks( - use_color=use_color, - on_node_start=on_node_start, - on_node_complete=on_node_complete, - ), - ) - finally: - adapter.close(ingress_connection) - ingress_python_results = ingress_result.python_results - ingress_load_results = ingress_result.load_results - write_python_node_results( - stream=progress_stream, - results=ingress_python_results, - use_color=use_color, - ) - read_side_names: frozenset[str] = ( - selected_task_asset_names - - lifecycle_plan.ingress_python_node_names - - python_node_result_names(ingress_python_results) - ) - read_side_connection: object | None = None - read_side_tracker: Any | None = None - if read_side_names: - read_side_connection = adapter.connect(connection_config) - read_side_tracker = create_read_side_python_execution_tracker( - python_graph=python_graph, - selected_python_names=read_side_names, - scheduled_sql_names=scheduled_sql_resource_names(plan_output) - | frozenset(result.source_name for result in ingress_load_results), - runtime=PythonNodeRuntime( - adapter=adapter, - connection_config=connection_config, - connection=read_side_connection, - run_id=pipeline_result.project.run_id, - target=pipeline_result.project.effective_target_name, - vars=pipeline_result.project.effective_vars, - is_reload=reload_sources, - default_database=default_database, - default_schema=default_schema, - relation_targets=relation_targets, - project_relations=project_relations, - start_cursor_ts=start_cursor_ts, - end_cursor_ts=end_cursor_ts, - start_cursor_int=start_cursor_int, - end_cursor_int=end_cursor_int, - providers=providers, - ), - ) - for ingress_load_result in ingress_load_results: - read_side_tracker.record_sql_result(ingress_load_result) - read_side_tracker.dispatch_ready_python_nodes() - initial_read_side_results: tuple[PythonNodeExecutionResult, ...] = read_side_tracker.results - if initial_read_side_results: - write_python_node_results( - stream=progress_stream, - results=initial_read_side_results, - use_color=use_color, - ) - return DirectPythonLifecycleState( - plan_output=plan_output, - discovered_inputs=discovered_inputs, - adapter=adapter, - progress_stream=progress_stream, - use_color=use_color, - base_on_node_complete=on_node_complete, - read_side_connection=read_side_connection, - read_side_tracker=read_side_tracker, - ingress_python_results=ingress_python_results, - ingress_load_results=ingress_load_results, - ingress_loader_names=lifecycle_plan.ingress_loader_names, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle_selection.py b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle_selection.py deleted file mode 100644 index 4d0571dbb0..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_lifecycle_selection.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Python-node selection and SQL loader lifecycle handoff.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredProjectInputs -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult - - -def task_asset_python_node_names( - *, selected_names: frozenset[str], discovered_inputs: DiscoveredProjectInputs -) -> frozenset[str]: - """Return selected task/asset names, excluding loaders and checks.""" - - task_asset_names: set[str] = { - *(node.name for node in discovered_inputs.task_functions), - *(node.name for node in discovered_inputs.asset_functions), - } - return frozenset(name for name in selected_names if name in task_asset_names) - - -def scheduled_sql_resource_names(plan: PlanOutput) -> frozenset[str]: - """Return the models, seeds, and source loads this run executes and reports results for.""" - - scheduled_keys: frozenset[CompiledObjectKey] = frozenset(plan.execution_order) - return frozenset( - entry.name - for entry in (*plan.model_entries, *plan.seed_entries, *plan.source_load_entries) - if entry.key in scheduled_keys - ) - - -def sql_loader_functions_for_lifecycle_handoff( - *, - discovered_inputs: DiscoveredProjectInputs, - ingress_loader_names: frozenset[str], -) -> tuple[DiscoveredLoaderFunction, ...]: - """Return SQL-stage loaders after Python ingress handled mixed dependencies.""" - - loader_functions: frozenset[object] = frozenset( - loader.function for loader in discovered_inputs.loader_functions - ) - handoff_loaders: list[DiscoveredLoaderFunction] = [] - for loader in discovered_inputs.loader_functions: - dependencies: list[object] = [] - for dependency in loader.depends_on: - if dependency in loader_functions or loader.name not in ingress_loader_names: - dependencies.append(dependency) - handoff_loaders.append(replace(loader, depends_on=tuple(dependencies))) - return tuple(handoff_loaders) - - -def python_node_result_names(results: tuple[PythonNodeExecutionResult, ...]) -> frozenset[str]: - """Return Python-node result names.""" - - return frozenset(result.node_name for result in results) - - -def load_result_key_or_none( - *, plan: PlanOutput, result: LoadExecutionResult -) -> CompiledObjectKey | None: - """Return the plan key for a source-load result when it is part of the SQL plan.""" - - for entry in plan.source_load_entries: - if entry.name == result.source_name: - return entry.key - return None diff --git a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_node_output.py b/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_node_output.py deleted file mode 100644 index 4d43a960d6..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_python_nodes/python_node_output.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Build Python-node result output.""" - -from __future__ import annotations - -from typing import TextIO - -from sqlbuild.compiler.python_nodes.types import PythonNodeStatus -from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def write_python_node_results( - *, stream: TextIO, results: tuple[PythonNodeExecutionResult, ...], use_color: bool -) -> None: - """Write human-readable task and asset execution rows.""" - - style: CliStyle = CliStyle(use_color=use_color) - result: PythonNodeExecutionResult - for result in results: - status_text: str = python_node_status_text(result.status) - stream.write( - f" {'python':<10}{result.kind.value:<10}{result.node_name:<50} " - f"{style.status(status=status_text)}" - ) - if result.error_message: - stream.write(f" {result.error_message}") - elif result.skip_reason: - stream.write(f" {result.skip_reason}") - stream.write("\n") - stream.flush() - - -def python_node_status_text(status: PythonNodeStatus) -> str: - """Return the short human status label for a Python-node status.""" - - if status == PythonNodeStatus.SUCCESS: - return "OK" - if status == PythonNodeStatus.SKIPPED: - return "SKIP" - return "FAIL" - - -def python_node_results_failed(results: tuple[PythonNodeExecutionResult, ...]) -> bool: - """Return whether any Python-node result failed.""" - - return any(result.status == PythonNodeStatus.FAILED for result in results) diff --git a/src/sqlbuild/cli/commands/_helpers/contract/__init__.py b/src/sqlbuild/cli/commands/_helpers/contract/__init__.py deleted file mode 100644 index 20120f1e95..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/contract/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Contract command helpers.""" diff --git a/src/sqlbuild/cli/commands/_helpers/contract/progress.py b/src/sqlbuild/cli/commands/_helpers/contract/progress.py deleted file mode 100644 index 2a2c008629..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/contract/progress.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Lifecycle progress for contract inspection and repository updates.""" - -from __future__ import annotations - -import time -from pathlib import Path -from typing import Any, TextIO - -from sqlbuild._native import ( - ConnectionProgressReporter, - compare_contracts, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.models import AdapterConnectionContext, ContractCommandRequest -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledSourceView -from sqlbuild.compiler.contract_adoption.models import ContractAdoptionResult, ContractEvidence -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter - - -def inspect_contracts_with_progress( - *, - context: AdapterConnectionContext, - request: ContractCommandRequest, - selected_models: tuple[CompiledModelView, ...], - selected_sources: tuple[CompiledSourceView, ...], - progress_stream: TextIO, - use_progress_color: bool, -) -> tuple[ContractEvidence, ...]: - """Connect once and report contract inspection lifecycle progress.""" - - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=context.adapter_name, - stream=progress_stream, - use_color=use_progress_color, - ) - connection_started: float = time.monotonic() - connection_progress.on_connection_start(1) - try: - connection: Any = context.adapter.connect(context.connection_config) - except Exception: - connection_progress.on_connection_error( - connection_count=1, - elapsed_seconds=time.monotonic() - connection_started, - ) - raise - try: - connection_progress.on_connection_complete( - connection_count=1, - elapsed_seconds=time.monotonic() - connection_started, - ) - - resource_count: int = len(selected_models) + len(selected_sources) - resource_label: str = "contract" if resource_count == 1 else "contracts" - inspection_progress: TransientStatusReporter = TransientStatusReporter( - stream=progress_stream, - use_color=use_progress_color, - ) - inspection_started: float = time.monotonic() - inspection_progress.start( - f"Inspecting {resource_count} {resource_label} from target '{request.from_target}'..." - ) - try: - evidence: tuple[ContractEvidence, ...] = compare_contracts( - adapter=context.adapter, - connection=connection, - models=selected_models, - sources=selected_sources, - ) - except Exception: - inspection_progress.error( - f"Contract inspection failed after {time.monotonic() - inspection_started:.2f}s" - ) - raise - inspection_progress.complete( - message=( - f"Inspected {resource_count} {resource_label} from target " - f"'{request.from_target}' ({time.monotonic() - inspection_started:.2f}s)" - ) - ) - finally: - context.adapter.close(connection) - return evidence - - -def write_contract_updates_with_progress( - *, - project_dir: Path, - graph: ProjectGraph, - result: ContractAdoptionResult, - request: ContractCommandRequest, - adapter: BaseAdapter, - resolved_connection: dict[str, object], - progress_stream: TextIO, - use_progress_color: bool, -) -> ContractAdoptionResult: - """Write contract declarations and report repository-update lifecycle progress.""" - - from sqlbuild.compiler.contract_adoption.main.write import write_contracts - - write_progress: TransientStatusReporter = TransientStatusReporter( - stream=progress_stream, - use_color=use_progress_color, - ) - write_started: float = time.monotonic() - write_progress.start("Updating repository contracts...") - try: - updated_result: ContractAdoptionResult = write_contracts( - project_dir=project_dir, - graph=graph, - result=result, - overwrite=request.overwrite, - adapter=adapter, - resolved_connection=resolved_connection, - cli_vars=request.cli_vars, - ) - except Exception: - write_progress.error( - f"Repository contract update failed after {time.monotonic() - write_started:.2f}s" - ) - raise - written_count: int = len(updated_result.written_paths) - written_label: str = "file" if written_count == 1 else "files" - write_progress.complete( - message=( - f"Updated repository contracts ({written_count} {written_label}, " - f"{time.monotonic() - write_started:.2f}s)" - ) - ) - return updated_result diff --git a/src/sqlbuild/cli/commands/_helpers/cost/command.py b/src/sqlbuild/cli/commands/_helpers/cost/command.py deleted file mode 100644 index b8835e7738..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/cost/command.py +++ /dev/null @@ -1,303 +0,0 @@ -"""CLI run-cost history and drill-down.""" - -from __future__ import annotations - -import re -import sys -from dataclasses import replace -from datetime import UTC, date, datetime, time, timedelta -from pathlib import Path -from typing import Any - -from sqlbuild._native import ( - format_cost_breakdown, - format_cost_history, - format_cost_history_json, - format_cost_json, - list_cost_runs, - resolve_cost_run, -) -from sqlbuild.cli.commands.constants import ( - COST_DEFAULT_DETAIL_SORT, - COST_DEFAULT_HISTORY_LIMIT, - COST_DEFAULT_HISTORY_SORT, - COST_DESCENDING_ORDER, - COST_HISTORY_SELECTOR, - COST_LATEST_SELECTOR, - ISO_DATE_LENGTH, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import CostCommandRequest -from sqlbuild.cost.models import CostRunRecord, ResourceCost -from sqlbuild.cost.types import CostStatus -from sqlbuild.cursor_algebra.constants import MINUTE_TO_DAY_DURATION_UNITS -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.presentation.main.supports_color import supports_color - -_HISTORY_SORT_FIELDS: frozenset[str] = frozenset( - {"completed", "cost", "credits", "duration", "models", "status"} -) -_BREAKDOWN_SORT_FIELDS: frozenset[str] = frozenset( - {"cost", "credits", "busy", "scanned", "model", "warehouse"} -) -_MISSING_METRIC_STATUSES: frozenset[CostStatus] = frozenset( - { - CostStatus.PENDING, - CostStatus.PARTIAL, - CostStatus.UNAVAILABLE, - CostStatus.COLLECTION_FAILED, - } -) -_HISTORY_METRIC_SORT_FIELDS: frozenset[str] = frozenset({"cost", "credits"}) -_ZERO_DURATION_PATTERN: re.Pattern[str] = re.compile(r"^0+[dhm]$") - - -def run_cost_command(request: CostCommandRequest) -> int: - """Render persisted cost history or one run breakdown.""" - - project_dir: Path = request.project_dir if request.project_dir is not None else Path.cwd() - _validate_limit(request=request) - if request.selector == COST_HISTORY_SELECTOR: - return _run_history(request=request, project_dir=project_dir) - return _run_detail(request=request, project_dir=project_dir) - - -def _run_history(*, request: CostCommandRequest, project_dir: Path) -> int: - sort_field: str = request.sort or COST_DEFAULT_HISTORY_SORT - order: str = request.order or COST_DESCENDING_ORDER - if sort_field not in _HISTORY_SORT_FIELDS: - raise CliUserError( - f"cost history --sort must be one of: {', '.join(sorted(_HISTORY_SORT_FIELDS))}", - code="C701", - ) - since: datetime | None = _parse_since_bound(value=request.since) - until: datetime | None = _parse_bound(value=request.until, label="--until", end_of_day=True) - if since is not None and until is not None and since > until: - raise CliUserError("cost --since must not be after --until", code="C711") - until_is_date: bool = request.until is not None and len(request.until) == ISO_DATE_LENGTH - matching_records: tuple[CostRunRecord, ...] = tuple( - record - for record in list_cost_runs(project_dir=project_dir) - if (since is None or record.completed_at >= since) - and ( - until is None - or (record.completed_at < until if until_is_date else record.completed_at <= until) - ) - ) - records: tuple[CostRunRecord, ...] = _sort_history_records( - records=matching_records, - field=sort_field, - descending=order == COST_DESCENDING_ORDER, - ) - limit: int | None = ( - None - if request.no_limit - else COST_DEFAULT_HISTORY_LIMIT - if request.limit is None - else request.limit - ) - if limit is not None: - records = records[:limit] - payload: str = format_cost_history_json(records=records, matching_count=len(matching_records)) - _write_json_output(payload=payload, path=request.json_output_path) - if request.json_output: - sys.stdout.write(payload + "\n") - else: - sys.stdout.write( - format_cost_history( - records=records, - matching_count=len(matching_records), - use_color=(not request.no_color) and supports_color(), - ) - + "\n" - ) - return 0 - - -def _run_detail(*, request: CostCommandRequest, project_dir: Path) -> int: - if request.since is not None or request.until is not None: - raise CliUserError("--since and --until are only valid for cost history", code="C702") - sort_field: str = request.sort or COST_DEFAULT_DETAIL_SORT - order: str = request.order or COST_DESCENDING_ORDER - if sort_field not in _BREAKDOWN_SORT_FIELDS: - raise CliUserError( - f"cost --sort must be one of: {', '.join(sorted(_BREAKDOWN_SORT_FIELDS))}", - code="C703", - ) - record: CostRunRecord = _resolve_run(project_dir=project_dir, selector=request.selector) - resources: tuple[ResourceCost, ...] = tuple( - sorted( - record.cost.resources, - key=lambda resource: ( - _resource_sort_value(resource=resource, field=sort_field), - resource.resource_name, - resource.warehouse_name, - ), - reverse=order == COST_DESCENDING_ORDER, - ) - ) - record = replace(record, cost=replace(record.cost, resources=resources)) - limit: int | None = None if request.no_limit else request.limit - json_record: CostRunRecord = ( - record - if limit is None - else replace(record, cost=replace(record.cost, resources=resources[:limit])) - ) - payload: str = format_cost_json(json_record) - _write_json_output(payload=payload, path=request.json_output_path) - if request.json_output: - sys.stdout.write(payload + "\n") - else: - sys.stdout.write( - format_cost_breakdown( - record=record, - use_color=(not request.no_color) and supports_color(), - limit=limit, - ) - + "\n" - ) - return 0 - - -def _resolve_run(*, project_dir: Path, selector: str) -> CostRunRecord: - records: tuple[CostRunRecord, ...] = list_cost_runs(project_dir=project_dir) - if selector == COST_LATEST_SELECTOR: - latest: CostRunRecord | None = resolve_cost_run( - project_dir=project_dir, - selector=selector, - ) - if latest is not None: - return latest - raise CliUserError("No persisted SQLBuild cost runs were found", code="C704") - exact: tuple[CostRunRecord, ...] = tuple( - record for record in records if record.run_id == selector - ) - if exact: - return exact[0] - matches: tuple[CostRunRecord, ...] = tuple( - record for record in records if record.run_id.startswith(selector) - ) - if not matches: - raise CliUserError(f"No cost run matches '{selector}'", code="C705") - if len(matches) > 1: - matching_ids: str = ", ".join(record.run_id for record in matches) - raise CliUserError( - f"Cost run prefix '{selector}' is ambiguous; matches: {matching_ids}", - code="C706", - ) - return matches[0] - - -def _parse_bound(*, value: str | None, label: str, end_of_day: bool) -> datetime | None: - if value is None: - return None - try: - if len(value) == ISO_DATE_LENGTH: - parsed_date: date = date.fromisoformat(value) - start: datetime = datetime.combine(parsed_date, time.min, tzinfo=UTC) - return start + timedelta(days=1) if end_of_day else start - parsed: datetime = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError as error: - raise CliUserError(f"cost {label} must be an ISO date or datetime", code="C707") from error - if parsed.tzinfo is None: - raise CliUserError(f"cost {label} datetime must include a timezone", code="C708") - return parsed.astimezone(UTC) - - -def _parse_since_bound(*, value: str | None) -> datetime | None: - if value is None: - return None - if _ZERO_DURATION_PATTERN.fullmatch(value) is not None: - return datetime.now(UTC) - duration: Duration | None = Duration.parse(value) - if duration is None: - return _parse_bound(value=value, label="--since", end_of_day=False) - if not duration.is_single_unit_in(MINUTE_TO_DAY_DURATION_UNITS): - raise CliUserError("cost --since relative duration must use one of: d, h, m", code="C707") - return datetime.now(UTC) - timedelta(seconds=duration.fixed_seconds) - - -def _sort_history_records( - *, records: tuple[CostRunRecord, ...], field: str, descending: bool -) -> tuple[CostRunRecord, ...]: - if field not in _HISTORY_METRIC_SORT_FIELDS: - return _sort_history_partition( - records=records, - field=field, - descending=descending, - ) - available: tuple[CostRunRecord, ...] = tuple( - record for record in records if record.cost.status not in _MISSING_METRIC_STATUSES - ) - missing: tuple[CostRunRecord, ...] = tuple( - record for record in records if record.cost.status in _MISSING_METRIC_STATUSES - ) - sorted_available: tuple[CostRunRecord, ...] = _sort_history_partition( - records=available, - field=field, - descending=descending, - ) - sorted_missing: tuple[CostRunRecord, ...] = tuple( - sorted( - sorted(missing, key=lambda record: record.run_id), - key=lambda record: record.completed_at, - reverse=True, - ) - ) - return (*sorted_available, *sorted_missing) - - -def _sort_history_partition( - *, records: tuple[CostRunRecord, ...], field: str, descending: bool -) -> tuple[CostRunRecord, ...]: - by_run_id: list[CostRunRecord] = sorted(records, key=lambda record: record.run_id) - by_completion: list[CostRunRecord] = sorted( - by_run_id, - key=lambda record: record.completed_at, - reverse=True, - ) - return tuple( - sorted( - by_completion, - key=lambda record: _history_sort_value(record=record, field=field), - reverse=descending, - ) - ) - - -def _history_sort_value(*, record: CostRunRecord, field: str) -> Any: - values: dict[str, Any] = { - "completed": record.completed_at, - "cost": record.cost.estimated_usd, - "credits": record.cost.estimated_compute_credits, - "duration": record.completed_at - record.started_at, - "models": len(record.cost.resources), - "status": record.cost.status.value, - } - return values[field] - - -def _resource_sort_value(*, resource: ResourceCost, field: str) -> Any: - values: dict[str, Any] = { - "cost": resource.estimated_usd, - "credits": resource.estimated_compute_credits, - "busy": resource.attributed_seconds, - "scanned": resource.bytes_scanned, - "model": resource.resource_name, - "warehouse": resource.warehouse_name, - } - return values[field] - - -def _validate_limit(*, request: CostCommandRequest) -> None: - if request.limit is not None and request.limit < 0: - raise CliUserError("cost --limit must be non-negative", code="C709") - if request.no_limit and request.limit is not None: - raise CliUserError("cost --limit and --no-limit cannot be combined", code="C710") - - -def _write_json_output(*, payload: str, path: Path | None) -> None: - if path is None: - return - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(payload + "\n", encoding="utf-8") diff --git a/src/sqlbuild/cli/commands/_helpers/cost/refresh.py b/src/sqlbuild/cli/commands/_helpers/cost/refresh.py deleted file mode 100644 index edd452b0ac..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/cost/refresh.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Best-effort delayed Snowflake cost refresh.""" - -from dataclasses import replace -from pathlib import Path - -from sqlbuild._native import ( - discover_project_inputs, - read_cost_run, - resolve_cost_run, - resolve_project_connection_config, - write_cost_run, -) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.constants import COST_HISTORY_SELECTOR, COST_LATEST_SELECTOR -from sqlbuild.cli.commands.models import CostCommandRequest -from sqlbuild.compiler.compile.main.effective_target import build_effective_target_config -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.cost.constants import LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX -from sqlbuild.cost.models import CostRunRecord, RunCostSummary -from sqlbuild.cost.types import CostAwareAdapter, CostCapability, CostStatus -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import TargetConfig - -_REFRESH_STATUS_RANK: dict[CostStatus, int] = { - CostStatus.COLLECTION_FAILED: 0, - CostStatus.UNAVAILABLE: 0, - CostStatus.PENDING: 1, - CostStatus.PARTIAL: 2, - CostStatus.COMPLETE: 3, -} - - -def refresh_pending_cost_run(request: CostCommandRequest) -> None: - """Refresh one delayed record without making lookup depend on Snowflake availability.""" - - if request.selector == COST_HISTORY_SELECTOR: - return - project_dir: Path = request.project_dir if request.project_dir is not None else Path.cwd() - try: - record: CostRunRecord | None = _resolve_refresh_record( - project_dir=project_dir, - selector=request.selector, - ) - if record is None or record.cost.status not in { - CostStatus.PENDING, - CostStatus.PARTIAL, - }: - return - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - target_config: TargetConfig | None = build_effective_target_config( - discovered_inputs=discovered, - selected_target=record.target_name, - ) - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered.project_config, - local_config=discovered.local_config, - ) - adapter: object = resolve_adapter(adapter_name=adapter_name, project_dir=project_dir) - if not isinstance(adapter, CostAwareAdapter): - return - if adapter.cost_capability() == CostCapability.NONE: - return - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered, - project_dir=project_dir, - selected_target=record.target_name, - ) - connection_database: object | None = connection_config.get("database") - target_database: str | None = ( - target_config.database - if target_config is not None and target_config.database is not None - else (None if connection_database is None else str(connection_database)) - ) - summary: RunCostSummary = adapter.collect_run_cost( - connection_config=connection_config, - target_database=target_database, - run_id=record.run_id, - started_at=record.started_at, - completed_at=record.completed_at, - statement_ledger_path=( - project_dir / "target" / "executions" / record.run_id / "statements.jsonl" - ), - usd_per_credit=record.cost.usd_per_credit, - rate_source=record.cost.rate_source, - ) - if _is_non_destructive_refresh(current=record.cost, candidate=summary): - write_cost_run(project_dir=project_dir, record=replace(record, cost=summary)) - except BaseException: - return - - -def _resolve_refresh_record(*, project_dir: Path, selector: str) -> CostRunRecord | None: - if selector == COST_LATEST_SELECTOR: - return resolve_cost_run(project_dir=project_dir, selector=selector) - exact: CostRunRecord | None = read_cost_run(project_dir=project_dir, run_id=selector) - if exact is not None: - return exact - return resolve_cost_run(project_dir=project_dir, selector=selector) - - -def _is_non_destructive_refresh(*, current: RunCostSummary, candidate: RunCostSummary) -> bool: - """Accept refreshed telemetry only when persisted coverage cannot regress.""" - - if _REFRESH_STATUS_RANK[candidate.status] < _REFRESH_STATUS_RANK[current.status]: - return False - if _has_ledger_persistence_failure(summary=current) and not _has_coverage_gain( - current=current, - candidate=candidate, - ): - return False - return ( - candidate.expected_statement_count >= current.expected_statement_count - and candidate.observed_statement_count >= current.observed_statement_count - and candidate.query_count >= current.query_count - ) - - -def _has_ledger_persistence_failure(*, summary: RunCostSummary) -> bool: - return any( - limitation.startswith(LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX) - for limitation in summary.limitations - ) - - -def _has_coverage_gain(*, current: RunCostSummary, candidate: RunCostSummary) -> bool: - return ( - candidate.expected_statement_count > current.expected_statement_count - or candidate.observed_statement_count > current.observed_statement_count - or candidate.query_count > current.query_count - ) diff --git a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py index 5e360c7595..edc2843934 100644 --- a/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py +++ b/src/sqlbuild/cli/commands/_helpers/dbt/sqlbuild_work.py @@ -6,17 +6,24 @@ from pathlib import Path from typing import TextIO -from sqlbuild._native import ConnectionProgressReporter, format_execution_header -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.build_planning.plan_policies import ( - dbt_interop_plan_safety_gates, +from sqlbuild._native import ( + ConnectionProgressReporter, enforce_plan_safety_policies, + format_execution_header, ) +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands.classes.build_progress_callbacks import ( BuildProgressCallbacks, format_build_footer, ) -from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext +from sqlbuild.cli.commands.constants import ( + EXECUTION_LIMIT_DBT_NOTE, + MISSING_ORIGIN_DBT_HELP, + RETENTION_DECREASE_DBT_HELP, + SNAPSHOT_FULL_REFRESH_DBT_HELP, + TABLE_TYPE_DOWNGRADE_DBT_HELP, +) +from sqlbuild.cli.commands.models import DbtSqlbuildWorkContext, PlanSafetyGates from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import ( @@ -35,6 +42,22 @@ from sqlbuild.presentation.classes.cli_style import CliStyle +def dbt_interop_plan_safety_gates() -> PlanSafetyGates: + """Gates for sqb dbt, which has no allow flags and confirms only on a terminal.""" + + return PlanSafetyGates( + allow_missing_migration_origin=False, + allow_snapshot_full_refresh=False, + allow_table_type_downgrade=False, + allow_retention_decrease=False, + missing_origin_help=MISSING_ORIGIN_DBT_HELP, + snapshot_full_refresh_help=SNAPSHOT_FULL_REFRESH_DBT_HELP, + table_type_downgrade_help=TABLE_TYPE_DOWNGRADE_DBT_HELP, + retention_decrease_help=RETENTION_DECREASE_DBT_HELP, + execution_limit_note=EXECUTION_LIMIT_DBT_NOTE, + ) + + def execute_sqlbuild_build_work( *, context: DbtSqlbuildWorkContext, diff --git a/src/sqlbuild/cli/commands/_helpers/init/__init__.py b/src/sqlbuild/cli/commands/_helpers/init/__init__.py deleted file mode 100644 index 94bca4d6cd..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/init/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers for sqb init.""" diff --git a/src/sqlbuild/cli/commands/_helpers/init/scaffold.py b/src/sqlbuild/cli/commands/_helpers/init/scaffold.py deleted file mode 100644 index f90180a96d..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/init/scaffold.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Project scaffolding helpers for sqb init.""" - -from __future__ import annotations - -from pathlib import Path - - -def scaffold_blank_project(*, base_dir: Path, project_name: str) -> None: - """Create the default blank SQLBuild project structure.""" - - directories: tuple[str, ...] = ( - "models/staging", - "models/marts", - "schemas", - "sources", - "seeds", - "python", - "hooks/sql", - "hooks/python", - "tests/unit", - "tests/scenarios", - "functions/sql", - "functions/python", - "macros", - "audits/generic", - "audits/singular", - ) - for directory in directories: - (base_dir / directory).mkdir(parents=True, exist_ok=True) - - (base_dir / "sqlbuild_project.toml").write_text(_build_project_toml(project_name=project_name)) - - for directory in directories: - gitkeep: Path = base_dir / directory / ".gitkeep" - if not any((base_dir / directory).iterdir()): - gitkeep.touch() - - -def _build_project_toml(*, project_name: str) -> str: - return f'''name = "{project_name}" -adapter = "duckdb" -default_target = "dev" - -[connections.developer] -database = "{project_name}.duckdb" - -[settings] -default_audit_severity = "warn" - -[defaults] -materialized = "table" - -[targets.prod] -connection = "developer" -schema = "prod" - -[targets.dev] -connection = "developer" -schema = "dev" -''' diff --git a/src/sqlbuild/cli/commands/_helpers/lint/__init__.py b/src/sqlbuild/cli/commands/_helpers/lint/__init__.py deleted file mode 100644 index 77752ebee0..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lint/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers for the lint and format CLI commands.""" diff --git a/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py b/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py deleted file mode 100644 index 51d1f3fe00..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lint/diagnostics.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Render actionable authored-source lint diagnostics.""" - -from __future__ import annotations - -import textwrap -from pathlib import Path - -from rich.cells import cell_len - -from sqlbuild.lint.constants import TAB_CHARACTER, VIOLATION_SEVERITY_FAULT -from sqlbuild.lint.models import LintRunResult, LintViolation -from sqlbuild.presentation.classes.cli_style import CliStyle - -_REPORT_LINE_WIDTH: int = 100 -_HELP_PREFIX: str = " = help: " -_HELP_CONTINUATION: str = " " - - -def format_lint_diagnostics( - *, result: LintRunResult, root: Path, use_color: bool -) -> tuple[str, ...]: - """Format every lint violation as a source-pointing diagnostic block.""" - - style: CliStyle = CliStyle(use_color=use_color) - return tuple( - _format_violation(violation=violation, result=result, root=root, style=style) - for violation in result.violations - ) - - -def _format_violation( - *, - violation: LintViolation, - result: LintRunResult, - root: Path, - style: CliStyle, -) -> str: - severity: str = "error" if violation.severity == VIOLATION_SEVERITY_FAULT else "warning" - label: str = f"{severity}[{violation.code}]" - styled_label: str = ( - style.error_strong(label) - if violation.severity == VIOLATION_SEVERITY_FAULT - else style.warning_strong(label) - ) - location: str = ( - f"{_display_path(path=violation.file_path, root=root)}:{violation.line}:{violation.column}" - ) - lines: list[str] = [ - f"{styled_label}: {violation.message}", - style.muted(f" --> {location}"), - ] - source_text: str | None = result.source_texts.get(violation.file_path) - if source_text is not None: - source_lines: list[str] = source_text.splitlines() - if 1 <= violation.line <= len(source_lines): - lines.extend( - _source_excerpt( - violation=violation, - source_line=source_lines[violation.line - 1], - style=style, - ) - ) - if violation.remediation is not None: - lines.extend(_help_lines(remediation=violation.remediation, style=style)) - return "\n".join(lines) - - -def _source_excerpt( - *, violation: LintViolation, source_line: str, style: CliStyle -) -> tuple[str, ...]: - line_number: str = str(violation.line) - gutter: str = " " * len(line_number) - start_index: int = min(max(violation.column - 1, 0), len(source_line)) - end_index: int = _excerpt_end_index( - violation=violation, - source_line=source_line, - start_index=start_index, - ) - padding: str = _display_padding(source_line[:start_index]) - caret_count: int = max(1, cell_len(source_line[start_index:end_index])) - caret: str = "^" * caret_count - styled_caret: str = ( - style.error_strong(caret) - if violation.severity == VIOLATION_SEVERITY_FAULT - else style.warning_strong(caret) - ) - return ( - style.muted(f"{gutter} |"), - f"{style.muted(f'{line_number} |')} {source_line}", - f"{style.muted(f'{gutter} |')} {padding}{styled_caret}", - style.muted(f"{gutter} |"), - ) - - -def _excerpt_end_index(*, violation: LintViolation, source_line: str, start_index: int) -> int: - if violation.end_line == violation.line and violation.end_column is not None: - return min(max(violation.end_column - 1, start_index + 1), len(source_line)) - if violation.end_line is not None and violation.end_line != violation.line: - return len(source_line) - return min(start_index + 1, len(source_line)) - - -def _display_padding(prefix: str) -> str: - return "".join( - TAB_CHARACTER if character == TAB_CHARACTER else " " * cell_len(character) - for character in prefix - ) - - -def _help_lines(*, remediation: str, style: CliStyle) -> tuple[str, ...]: - wrapped: list[str] = textwrap.wrap( - remediation, - width=_REPORT_LINE_WIDTH, - initial_indent=_HELP_PREFIX, - subsequent_indent=_HELP_CONTINUATION, - ) - if not wrapped: - return () - first: str = wrapped[0].removeprefix(_HELP_PREFIX) - return (f" {style.muted('= help:')} {first}", *wrapped[1:]) - - -def _display_path(*, path: Path, root: Path) -> str: - try: - return path.relative_to(root).as_posix() - except ValueError: - return path.as_posix() diff --git a/src/sqlbuild/cli/commands/_helpers/lint/positional_paths.py b/src/sqlbuild/cli/commands/_helpers/lint/positional_paths.py deleted file mode 100644 index fbe1e9168f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lint/positional_paths.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Translate `sqb format` positional paths into formatter path selectors.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.lint.constants import LINT_DIRECTORY_NAMES - -_FORMAT_PATH_ERROR_CODE: str = "C113" - - -def format_path_selectors( - *, paths: tuple[str, ...], project_dir: Path | None, cwd: Path -) -> tuple[str, ...]: - """Return one `path:` selector per positional path, relative to the project root.""" - - project_root: Path = (project_dir if project_dir is not None else cwd).resolve() - selectors: list[str] = [] - for raw_path in paths: - candidate: Path = Path(raw_path) - resolved: Path = (candidate if candidate.is_absolute() else cwd / candidate).resolve() - if not resolved.exists(): - raise CliUserError( - f"format path '{raw_path}' does not exist", code=_FORMAT_PATH_ERROR_CODE - ) - if not resolved.is_relative_to(project_root): - raise CliUserError( - f"format path '{raw_path}' is outside the project '{project_root}'", - code=_FORMAT_PATH_ERROR_CODE, - ) - relative: Path = resolved.relative_to(project_root) - if not relative.parts or relative.parts[0] not in LINT_DIRECTORY_NAMES: - raise CliUserError( - f"format path '{raw_path}' is not under a formatted folder", - code=_FORMAT_PATH_ERROR_CODE, - help="use a path under " + ", ".join(f"{name}/" for name in LINT_DIRECTORY_NAMES), - ) - selectors.append(f"path:{relative.as_posix()}") - return tuple(selectors) diff --git a/src/sqlbuild/cli/commands/_helpers/lint/runs.py b/src/sqlbuild/cli/commands/_helpers/lint/runs.py deleted file mode 100644 index 1a90134410..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lint/runs.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Shared helpers for the lint and format CLI commands.""" - -from __future__ import annotations - -import json -import tomllib -from pathlib import Path -from typing import cast - -from sqlbuild.cli.commands._helpers.lint.diagnostics import format_lint_diagnostics -from sqlbuild.compiler.discovery.constants import LOCAL_CONFIG_FILENAME -from sqlbuild.lint.constants import ( - ADAPTER_CONFIG_KEY, - ADAPTER_DIALECT_TRANSLATIONS, - DEFAULT_LINE_WIDTH, - DEFAULT_MAX_DESCRIPTION_LINES, - DEFAULT_MAX_LITERAL_LENGTH, - DEFAULT_MAX_RANKING_ORDER_BY, - FORMAT_SECTION_KEY, - LINE_WIDTH_KEY, - MAX_DESCRIPTION_LINES_KEY, - MAX_LITERAL_LENGTH_THRESHOLD, - MAX_RANKING_ORDER_BY_THRESHOLD, - PROJECT_CONFIG_FILENAME_KEY, -) -from sqlbuild.lint.models import ( - LintConfig, - LintRunResult, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.rule_engine.main.load_config import load_rules_config - - -def resolve_lint_config(*, project_dir: Path) -> LintConfig: - """Resolve native lint and format configuration from project settings.""" - - max_description_lines: int = DEFAULT_MAX_DESCRIPTION_LINES - line_width: int = DEFAULT_LINE_WIDTH - dialect: str = "generic" - thresholds: dict[str, object] = {} - allow_suppressions: bool = True - config_file: Path = project_dir / PROJECT_CONFIG_FILENAME_KEY - if config_file.is_file(): - payload: dict[str, object] = tomllib.loads(config_file.read_text(encoding="utf-8-sig")) - thresholds = _rule_thresholds(payload) - allow_suppressions = load_rules_config(project_dir=project_dir).allow_exceptions - format_section: object = payload.get(FORMAT_SECTION_KEY) - raw_adapter: object = payload.get(ADAPTER_CONFIG_KEY) - if isinstance(raw_adapter, str): - dialect = ADAPTER_DIALECT_TRANSLATIONS.get(raw_adapter, raw_adapter) - if isinstance(format_section, dict): - line_width = _resolve_int( - section=format_section, - key=LINE_WIDTH_KEY, - current=line_width, - ) - max_description_lines = _resolve_int( - section=format_section, - key=MAX_DESCRIPTION_LINES_KEY, - current=max_description_lines, - ) - local_config_file: Path = project_dir / LOCAL_CONFIG_FILENAME - if local_config_file.is_file(): - local_payload: dict[str, object] = tomllib.loads( - local_config_file.read_text(encoding="utf-8-sig") - ) - local_adapter: object = local_payload.get(ADAPTER_CONFIG_KEY) - if isinstance(local_adapter, str): - dialect = ADAPTER_DIALECT_TRANSLATIONS.get(local_adapter, local_adapter) - return LintConfig( - line_width=line_width, - max_description_lines=max_description_lines, - dialect=dialect, - max_literal_length=_threshold( - thresholds=thresholds, - key=MAX_LITERAL_LENGTH_THRESHOLD, - default=DEFAULT_MAX_LITERAL_LENGTH, - ), - max_ranking_order_by=_threshold( - thresholds=thresholds, - key=MAX_RANKING_ORDER_BY_THRESHOLD, - default=DEFAULT_MAX_RANKING_ORDER_BY, - ), - allow_suppressions=allow_suppressions, - ) - - -def _rule_thresholds(payload: dict[str, object]) -> dict[str, object]: - rules: object = payload.get("rules") - if not isinstance(rules, dict): - return {} - thresholds: object = cast(dict[str, object], rules).get("thresholds") - return cast(dict[str, object], thresholds) if isinstance(thresholds, dict) else {} - - -def _threshold(*, thresholds: dict[str, object], key: str, default: int) -> int: - value: object = thresholds.get(key) - return ( - value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else default - ) - - -def prepare_lint_run(*, project_dir: Path) -> tuple[LintConfig, str | None]: - """Resolve native lint configuration.""" - - return resolve_lint_config(project_dir=project_dir), None - - -def render_lint_result( - *, - result: LintRunResult, - root: Path, - use_color: bool, - show_formatted: bool, - formatted_heading: str = "Formatted files:", -) -> None: - """Print a lint or format run result to stdout.""" - - if show_formatted and result.formatted_files: - print(formatted_heading) - formatted_path: Path - for formatted_path in result.formatted_files: - print(f" {formatted_path}") - print() - if result.violations: - print("\n\n".join(format_lint_diagnostics(result=result, root=root, use_color=use_color))) - print() - for fix in result.rule_fixes: - print(f"{fix.status.upper()} {fix.code} {fix.file_path}:{fix.line}: {fix.reason}") - summary: str = ( - f"Completed. FAULT={len(result.faults)} WARN={len(result.warnings)} " - f"FILES={result.files_checked}" - ) - print(_styled_summary(summary=summary, result=result, use_color=use_color)) - - -def render_lint_result_json(*, result: LintRunResult) -> None: - """Print a stable machine-readable lint or format result.""" - - print( - json.dumps( - { - "faults": len(result.faults), - "files_checked": result.files_checked, - "formatted_files": [str(path) for path in result.formatted_files], - "rule_fixes": [ - { - "file": str(fix.file_path), - "code": fix.code, - "line": fix.line, - "status": fix.status, - "reason": fix.reason, - } - for fix in result.rule_fixes - ], - "violations": [ - { - "code": violation.code, - "column": violation.column, - "end_column": violation.end_column, - "end_line": violation.end_line, - "engine": violation.engine, - "file": str(violation.file_path), - "line": violation.line, - "message": violation.message, - "remediation": violation.remediation, - "severity": violation.severity, - } - for violation in result.violations - ], - "warnings": len(result.warnings), - }, - sort_keys=True, - separators=(",", ":"), - ) - ) - - -def _styled_summary(*, summary: str, result: LintRunResult, use_color: bool) -> str: - style: CliStyle = CliStyle(use_color=use_color) - if result.faults: - return style.error_strong(summary) - if result.warnings: - return style.warning_strong(summary) - return style.success_strong(summary) - - -def _resolve_int(*, section: dict[str, object], key: str, current: int) -> int: - raw_value: object = section.get(key) - if isinstance(raw_value, int) and raw_value > 0: - return raw_value - return current diff --git a/src/sqlbuild/cli/commands/_helpers/lint/selection.py b/src/sqlbuild/cli/commands/_helpers/lint/selection.py deleted file mode 100644 index 63fe30b415..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/lint/selection.py +++ /dev/null @@ -1,356 +0,0 @@ -"""Canonical model selector resolution for lint and format CLI commands.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import build_project_graph, discover_project_inputs -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.classes.selected_contract_input_discoverer import ( - SelectedContractInputDiscoverer, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile -from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.lint.constants import LINT_DIRECTORY_NAMES, PARENT_PATH_SEGMENT, SQL_FILE_SUFFIX -from sqlbuild.lint.main.scan_fixture_typed_null_candidates import ( - scan_fixture_typed_null_candidates, -) -from sqlbuild.lint.models import FixtureNullCandidateScan -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) - -_PATH_SEPARATOR: str = "/" -_GRAPH_SELECTOR_MARKERS: tuple[str, ...] = ("+", ":", "~", "/", "\\", ",") - - -def resolve_lint_inputs( - *, project_dir: Path, select: tuple[str, ...], exclude: tuple[str, ...] -) -> tuple[BaseAdapter, frozenset[Path] | None, DiscoveredProjectInputs]: - """Resolve the adapter and optional model-file scope through canonical selectors.""" - - lint_paths: frozenset[Path] | None = _resolve_lint_paths( - project_dir=project_dir, - select=select, - exclude=exclude, - ) - if lint_paths is not None: - _validate_selected_paths(paths=lint_paths) - candidates: FixtureNullCandidateScan = scan_fixture_typed_null_candidates( - project_dir=project_dir, - selected_paths=lint_paths, - ) - discovered: DiscoveredProjectInputs = SelectedContractInputDiscoverer.discover( - project_dir=project_dir, - selected_test_paths=candidates.paths, - referenced_model_names=candidates.model_names, - ) - adapter: BaseAdapter = _resolve_discovered_adapter( - project_dir=project_dir, - discovered=discovered, - ) - return adapter, lint_paths, discovered - discovered = discover_project_inputs( - project_dir=project_dir, - sql_analysis_enabled_override=False, - extract_output_column_locations=False, - ) - reject_unit_test_selectors(select=select, exclude=exclude, discovered_inputs=discovered) - adapter = _resolve_discovered_adapter( - project_dir=project_dir, - discovered=discovered, - ) - if not select and not exclude: - return adapter, None, discovered - combined_paths: frozenset[Path] | None = _resolve_combined_lint_paths( - project_dir=project_dir, - discovered=discovered, - adapter=adapter, - select=select, - exclude=exclude, - ) - if combined_paths is not None: - _validate_selected_paths(paths=combined_paths) - return adapter, combined_paths, discovered - paths: frozenset[Path] = _resolve_model_paths( - project_dir=project_dir, - discovered=discovered, - adapter=adapter, - select=select, - exclude=exclude, - ) - _validate_selected_paths(paths=paths) - return adapter, paths, discovered - - -def _resolve_model_paths( - *, - project_dir: Path, - discovered: DiscoveredProjectInputs, - adapter: BaseAdapter, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> frozenset[Path]: - """Resolve model selectors to authored model paths.""" - - exact_paths: frozenset[Path] | None = _resolve_exact_model_paths( - project_dir=project_dir, - discovered=discovered, - select=select, - exclude=exclude, - ) - if exact_paths is not None: - return exact_paths - graph: ProjectGraph = build_project_graph(discovered_inputs=discovered, adapter=adapter) - selected_keys: frozenset[CompiledObjectKey] = select_project_graph( - graph=graph, select=select, exclude=exclude - ) - selected_names: frozenset[str] = frozenset( - key.name for key in selected_keys if key.resource_type == CompiledResourceType.MODEL - ) - return frozenset( - (project_dir / model.relative_path).resolve() - for model in graph.project.models - if model.name in selected_names - ) - - -def _resolve_combined_lint_paths( - *, - project_dir: Path, - discovered: DiscoveredProjectInputs, - adapter: BaseAdapter, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> frozenset[Path] | None: - """Resolve default or path-based format scopes before graph selector expansion.""" - - if any(not selector.strip() for selector in (*select, *exclude)): - return None - selected_path_tokens, selected_model_tokens = _partition_lint_selectors(select) - excluded_path_tokens, excluded_model_tokens = _partition_lint_selectors(exclude) - has_path_selector: bool = bool(selected_path_tokens or excluded_path_tokens) - if not has_path_selector and select: - return None - all_paths: frozenset[Path] = frozenset( - _paths_for_prefixes(project_dir=project_dir, prefixes=()) - ) - if select: - selected: set[Path] = set( - _paths_for_prefixes(project_dir=project_dir, prefixes=selected_path_tokens) - if selected_path_tokens - else () - ) - if selected_model_tokens: - selected.update( - _resolve_model_paths( - project_dir=project_dir, - discovered=discovered, - adapter=adapter, - select=selected_model_tokens, - exclude=(), - ) - ) - else: - selected = set(all_paths) - excluded: set[Path] = set( - _paths_for_prefixes(project_dir=project_dir, prefixes=excluded_path_tokens) - if excluded_path_tokens - else () - ) - if excluded_model_tokens: - excluded.update( - _resolve_model_paths( - project_dir=project_dir, - discovered=discovered, - adapter=adapter, - select=excluded_model_tokens, - exclude=(), - ) - ) - return frozenset(selected - excluded) - - -def _partition_lint_selectors( - raw_selectors: tuple[str, ...], -) -> tuple[tuple[str, ...], tuple[str, ...]]: - """Separate valid formatter-root paths from model selectors.""" - - path_tokens: list[str] = [] - model_tokens: list[str] = [] - for raw_selector in raw_selectors: - for token in raw_selector.split(): - _reject_unknown_lint_root(token) - prefixes: tuple[str, ...] | None = _lint_path_prefixes(raw_selectors=(token,)) - if prefixes is not None: - path_tokens.extend(prefixes) - else: - model_tokens.append(token) - return tuple(path_tokens), tuple(model_tokens) - - -def _resolve_discovered_adapter( - *, project_dir: Path, discovered: DiscoveredProjectInputs -) -> BaseAdapter: - return resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered.project_config, - local_config=discovered.local_config, - ), - project_dir=project_dir, - ) - - -def _resolve_lint_paths( - *, - project_dir: Path, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> frozenset[Path] | None: - """Resolve simple path selectors across every formatter-owned SQL root.""" - - selected_prefixes: tuple[str, ...] | None = _lint_path_prefixes(raw_selectors=select) - excluded_prefixes: tuple[str, ...] | None = _lint_path_prefixes(raw_selectors=exclude) - if selected_prefixes is None or excluded_prefixes is None: - return None - selected: tuple[Path, ...] = _paths_for_prefixes( - project_dir=project_dir, - prefixes=selected_prefixes, - ) - return frozenset( - path - for path in selected - if not _matches_path_prefix( - relative_path=path.relative_to(project_dir.resolve()).as_posix(), - prefixes=excluded_prefixes, - ) - ) - - -def _paths_for_prefixes(*, project_dir: Path, prefixes: tuple[str, ...]) -> tuple[Path, ...]: - roots: tuple[Path, ...] = ( - tuple(project_dir / prefix for prefix in prefixes) - if prefixes - else tuple(project_dir / directory_name for directory_name in LINT_DIRECTORY_NAMES) - ) - paths: set[Path] = set() - for root in roots: - if root.is_file() and root.suffix.casefold() == SQL_FILE_SUFFIX: - paths.add(root.resolve()) - elif root.is_dir(): - paths.update(file_path.resolve() for file_path in root.rglob("*.sql")) - return tuple(sorted(paths)) - - -def _lint_path_prefixes(*, raw_selectors: tuple[str, ...]) -> tuple[str, ...] | None: - prefixes: list[str] = [] - for raw_selector in raw_selectors: - if not raw_selector.strip(): - return None - for token in raw_selector.split(): - normalized: str = token.replace("\\", "/") - if not normalized.startswith("path:") and not _is_bare_lint_path(normalized): - return None - prefix: str = normalized.removeprefix("path:").strip("/") - if any(marker in prefix for marker in ("+", "~", ",")): - return None - parts: tuple[str, ...] = tuple(part for part in prefix.split("/") if part) - if not parts or parts[0] not in LINT_DIRECTORY_NAMES or PARENT_PATH_SEGMENT in parts: - return None - prefixes.append("/".join(parts)) - return tuple(prefixes) - - -def _is_bare_lint_path(token: str) -> bool: - """Treat `models/orders.sql`-style tokens as formatter paths without a `path:` prefix.""" - - first_segment: str = token.split(_PATH_SEPARATOR, 1)[0] - return _PATH_SEPARATOR in token and first_segment in LINT_DIRECTORY_NAMES - - -def _reject_unknown_lint_root(token: str) -> None: - normalized: str = token.replace("\\", "/") - if not normalized.startswith("path:"): - return - root: str = normalized.removeprefix("path:").strip(_PATH_SEPARATOR).split(_PATH_SEPARATOR, 1)[0] - if root in LINT_DIRECTORY_NAMES: - return - raise PlannerInputError( - "format path selectors must start with one of: " - + ", ".join(f"'{name}/'" for name in LINT_DIRECTORY_NAMES), - code="S012", - ) - - -def _matches_path_prefix(*, relative_path: str, prefixes: tuple[str, ...]) -> bool: - return any( - relative_path == prefix or relative_path.startswith(f"{prefix}/") for prefix in prefixes - ) - - -def _resolve_exact_model_paths( - *, - project_dir: Path, - discovered: DiscoveredProjectInputs, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> frozenset[Path] | None: - """Resolve unexpanded model-name selectors without compiling the project graph.""" - - model_names_and_paths: list[tuple[str, Path]] = [] - model_file: DiscoveredSqlModelFile - for model_file in discovered.model_files: - configured_name: object = model_file.header_values.get("name") - model_name: str = ( - configured_name - if isinstance(configured_name, str) and configured_name - else model_file.relative_path.stem - ) - model_names_and_paths.append( - (model_name, (project_dir / model_file.relative_path).resolve()) - ) - selected_names: frozenset[str] | None = _exact_names(raw_selectors=select) - excluded_names: frozenset[str] | None = _exact_names(raw_selectors=exclude) - if selected_names is None or excluded_names is None: - return None - available_names: frozenset[str] = frozenset(name for name, _path in model_names_and_paths) - unknown_names: frozenset[str] = (selected_names | excluded_names) - available_names - if unknown_names: - unknown_name: str = sorted(unknown_names)[0] - raise PlannerInputError(f"unknown selector name '{unknown_name}'", code="S007") - names: frozenset[str] = selected_names or available_names - return frozenset( - path for name, path in model_names_and_paths if name in names and name not in excluded_names - ) - - -def _exact_names(*, raw_selectors: tuple[str, ...]) -> frozenset[str] | None: - """Return plain model names, or None when canonical graph resolution is required.""" - - names: set[str] = set() - raw_selector: str - for raw_selector in raw_selectors: - if not raw_selector.strip(): - return None - token: str - for token in raw_selector.split(): - if any(marker in token for marker in _GRAPH_SELECTOR_MARKERS): - return None - names.add(token) - return frozenset(names) - - -def _validate_selected_paths(*, paths: frozenset[Path]) -> None: - """Reject lint selections that contain no SQL files.""" - - if not paths: - raise PlannerInputError( - "lint selection matched no SQL files", - code="S007", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/query/__init__.py b/src/sqlbuild/cli/commands/_helpers/query/__init__.py deleted file mode 100644 index 30dcf76056..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/query/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Query command helpers.""" diff --git a/src/sqlbuild/cli/commands/_helpers/query/output.py b/src/sqlbuild/cli/commands/_helpers/query/output.py deleted file mode 100644 index 140ab533eb..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/query/output.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Output formatters for ad hoc query results.""" - -from __future__ import annotations - -import csv -import io -import json -from decimal import Decimal -from typing import Any - -from sqlbuild.adapter.contract.models import QueryResult -from sqlbuild.cli.commands.exceptions import CliUserError - -_LONG_OUTPUT_FORMAT: str = "long" -_TABLE_OUTPUT_FORMAT: str = "table" -_JSON_OUTPUT_FORMAT: str = "json" -_CSV_OUTPUT_FORMAT: str = "csv" - - -def render_query_result(*, result: QueryResult, output_format: str, limit: int | None) -> str: - """Render a query result using the requested output format.""" - - if not result.columns: - return "OK\n" - if output_format == _LONG_OUTPUT_FORMAT: - return render_long_query_result(result=result, limit=limit) - if output_format == _TABLE_OUTPUT_FORMAT: - return render_table_query_result(result=result, limit=limit) - if output_format == _JSON_OUTPUT_FORMAT: - return render_json_query_result(result) - if output_format == _CSV_OUTPUT_FORMAT: - return render_csv_query_result(result) - raise CliUserError(f"unsupported query output format '{output_format}'") - - -def render_long_query_result(*, result: QueryResult, limit: int | None) -> str: - lines: list[str] = [] - column_width: int = max(len(column) for column in result.columns) - row_index: int - row: tuple[object, ...] - for row_index, row in enumerate(result.rows, start=1): - if lines: - lines.append("") - header: str = f"-[ RECORD {row_index} ]" - lines.append(header + "-" * max(0, 40 - len(header)) + "+") - column: str - value: object - for column, value in zip(result.columns, row, strict=True): - lines.append(f"{column:<{column_width}} | {_format_display_value(value)}") - lines.append("") - lines.append(_format_row_count(len(result.rows))) - if result.truncated: - lines.append(_format_truncated_message(limit)) - return "\n".join(lines) + "\n" - - -def render_table_query_result(*, result: QueryResult, limit: int | None) -> str: - display_rows: list[tuple[str, ...]] = [] - source_row: tuple[object, ...] - for source_row in result.rows: - display_values: list[str] = [] - source_value: object - for source_value in source_row: - display_values.append(_format_display_value(source_value)) - display_rows.append(tuple(display_values)) - rows: tuple[tuple[str, ...], ...] = tuple(display_rows) - widths: list[int] = [len(column) for column in result.columns] - row: tuple[str, ...] - for row in rows: - index: int - value: str - for index, value in enumerate(row): - widths[index] = max(widths[index], len(value)) - separator: str = " | " - lines: list[str] = [ - separator.join(column.ljust(widths[index]) for index, column in enumerate(result.columns)), - separator.join("-" * width for width in widths), - ] - for row in rows: - lines.append( - separator.join(value.ljust(widths[index]) for index, value in enumerate(row)).rstrip() - ) - lines.append("") - lines.append(_format_row_count(len(result.rows))) - if result.truncated: - lines.append(_format_truncated_message(limit)) - return "\n".join(lines) + "\n" - - -def render_json_query_result(result: QueryResult) -> str: - rows: list[dict[str, object]] = [ - dict(zip(result.columns, row, strict=True)) for row in result.rows - ] - return json.dumps(rows, default=_json_default) + "\n" - - -def render_csv_query_result(result: QueryResult) -> str: - stream: io.StringIO = io.StringIO() - writer: Any = csv.writer(stream, lineterminator="\n") - writer.writerow(result.columns) - writer.writerows(result.rows) - return stream.getvalue() - - -def _format_display_value(value: object) -> str: - if value is None: - return "NULL" - if isinstance(value, Decimal): - return format(value, "f") - return str(value) - - -def _format_row_count(count: int) -> str: - noun: str = "row" if count == 1 else "rows" - return f"{count} {noun}" - - -def _format_truncated_message(limit: int | None) -> str: - if limit is None: - return "" - return f"Showing {limit} rows. Use --limit to show more or --no-limit to disable the limit." - - -def _json_default(value: object) -> str: - if isinstance(value, Decimal): - return format(value, "f") - return str(value) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/__init__.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py deleted file mode 100644 index 1e608b5522..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Scenario snapshot capture command runner.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ( - PlanningProgressReporter, - discover_project_inputs, - resolve_project_connection_config, - run_compile_only_pipeline, - write_execution_header, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.scenario_capture.capture_run import ( - build_scenario_capture_settings, - run_scenario_capture_run, -) -from sqlbuild.cli.commands._helpers.scenario_capture.dialect import require_scenario_capture_dialect -from sqlbuild.cli.commands._helpers.scenario_capture.snapshot_limits import ( - build_scenario_snapshot_capture_limits, - scenario_snapshot_capture_warning, -) -from sqlbuild.cli.commands._helpers.scenario_execution.namespace import resolve_scenario_namespace -from sqlbuild.cli.commands._helpers.scenario_execution.selection import select_scenarios -from sqlbuild.cli.commands.constants import ( - SCENARIO_CLI_SQL_VALIDATION_REQUIRED, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - ScenarioCaptureCommandRequest, - ScenarioRunOutputContext, - ScenarioSnapshotLimitInputs, -) -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, -) -from sqlbuild.executor.scenario.models import ScenarioSnapshotCaptureLimits -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( - resolve_effective_scenario_config, -) - - -def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: - """Execute the scenario capture command.""" - - project_dir: Path | None = request.project_dir - no_sql_validation: bool = request.no_sql_validation - no_color: bool = request.no_color - selectors: tuple[str, ...] = request.selectors - exclude: tuple[str, ...] = request.exclude - retain: bool = request.retain - limit_inputs: ScenarioSnapshotLimitInputs = request.limit_inputs - force: bool = limit_inputs.force - if no_sql_validation: - raise CliUserError( - "scenario capture requires SQL analysis, but `--no-sql-analysis` turns it off", - code=SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - help="run the capture without `--no-sql-analysis`", - ) - - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - reject_unit_test_selectors( - select=selectors, exclude=exclude, discovered_inputs=discovered_inputs - ) - discovered_inputs, namespace = resolve_scenario_namespace( - inputs=discovered_inputs, cli_value=request.scenario_namespace - ) - _validate_capture_sql_analysis_enabled(discovered_inputs=discovered_inputs) - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - capture_dialect: str = require_scenario_capture_dialect( - adapter=adapter, adapter_name=adapter_name - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - ) - use_color: bool = not no_color and supports_color() - progress_stream: TextIO = sys.stdout - progress_stream.write( - f"Scenario namespace: {namespace.value or '(unset)'} (source: {namespace.source})\n" - ) - target_label: str | None = " ".join(selectors) if selectors else None - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=progress_stream, - use_color=use_color, - ) - progress_stream.write("\n") - write_execution_header( - stream=progress_stream, - command="sqb scenario capture", - target=target_label, - concurrency=1, - use_color=use_color, - ) - progress_stream.write(f"{scenario_snapshot_capture_warning(force=force)}\n\n") - progress_stream.flush() - - with OperationLifecycle(operation_kind="project", operation_name="project_compile"): - pipeline_result: CompilePipelineResult = run_compile_only_pipeline( - discovered_inputs=discovered_inputs, - adapter=adapter, - options=CompilePipelineOptions( - no_sql_validation=no_sql_validation, - connection_config=connection_config, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - ), - ), - on_progress=planning_progress.on_progress, - ) - scenarios: tuple[CompiledSqlScenario, ...] = select_scenarios( - project=pipeline_result.project, - selectors=selectors, - exclude=exclude, - project_dir=effective_project_dir, - ) - capture_limits: ScenarioSnapshotCaptureLimits = build_scenario_snapshot_capture_limits( - scenario_config=resolve_effective_scenario_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - limit_inputs=limit_inputs, - ) - exit_code, _ = run_scenario_capture_run( - project_dir=effective_project_dir, - pipeline_result=pipeline_result, - scenarios=scenarios, - connection_config=connection_config, - adapter=adapter, - adapter_name=adapter_name, - project_name=discovered_inputs.project_config.name, - settings=build_scenario_capture_settings( - capture_adapter=adapter_name, - capture_dialect=capture_dialect, - retain=retain, - limits=capture_limits, - ), - output_context=ScenarioRunOutputContext( - namespace=namespace, - progress_stream=progress_stream, - use_color=use_color, - ), - ) - return exit_code - - -def _validate_capture_sql_analysis_enabled(*, discovered_inputs: DiscoveredProjectInputs) -> None: - guidance: tuple[str, str] | None = sql_analysis_off_guidance( - discovered_inputs=discovered_inputs, - purpose="to capture snapshots for local scenario replay", - ) - if guidance is not None: - note, help_text = guidance - raise CliUserError( - f"scenario capture requires SQL analysis; {note}", - code=SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - help=help_text, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py deleted file mode 100644 index 2609f40093..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py +++ /dev/null @@ -1,286 +0,0 @@ -"""Shared scenario snapshot capture execution.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ConnectionProgressReporter, format_summary_footer -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.scenario_output.namespace import ( - scenario_activity_message, - write_namespace_completion, -) -from sqlbuild.cli.commands._helpers.scenario_output.result_output import render_result_error -from sqlbuild.cli.commands.constants import SUCCESS_STATUS -from sqlbuild.cli.commands.models import ScenarioRunOutputContext -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.models import ScenarioExecutionPlan -from sqlbuild.executor.build.models import SeedExecutionResult -from sqlbuild.executor.pipeline.main.run import run_scenario_capture_pipeline -from sqlbuild.executor.scenario.models import ( - ScenarioCaptureSettings, - ScenarioFixtureExecutionResult, - ScenarioSnapshotCaptureLimits, - ScenarioSnapshotCaptureRelationResult, - ScenarioSnapshotCaptureResult, - ScenarioSnapshotCaptureRunResult, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.runtime.contracts.models import ConnectionHooks - -_SCENARIO_NAME_WIDTH: int = 64 -_CAPTURE_RELATION_KIND_WIDTH: int = 8 -_CAPTURE_RELATION_NAME_WIDTH: int = _SCENARIO_NAME_WIDTH - 4 - _CAPTURE_RELATION_KIND_WIDTH - 1 - - -def run_scenario_capture_run( - *, - project_dir: Path, - pipeline_result: CompilePipelineResult, - scenarios: tuple[CompiledSqlScenario, ...], - connection_config: dict[str, object], - adapter: BaseAdapter, - adapter_name: str, - project_name: str, - settings: ScenarioCaptureSettings, - output_context: ScenarioRunOutputContext, - capture_results_out: list[ScenarioSnapshotCaptureRunResult] | None = None, -) -> tuple[int, list[ScenarioSnapshotCaptureRunResult] | None]: - """Capture selected scenarios to durable snapshots and render results.""" - - progress_stream: TextIO = output_context.progress_stream - use_color: bool = output_context.use_color - style: CliStyle = CliStyle(use_color=use_color) - header: str = f"Scenario Capture ({len(scenarios)} selected)" - progress_stream.write(f"\n{style.success_strong(header)}\n\n") - progress_stream.flush() - scenario_status: TransientStatusReporter = TransientStatusReporter( - stream=progress_stream, - use_color=use_color, - ) - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=adapter_name, - blank_line_after_complete=True, - stream=progress_stream, - use_color=use_color, - ) - status_is_tty: bool = hasattr(progress_stream, "isatty") and progress_stream.isatty() - activity: str = scenario_activity_message( - activity="Capturing scenarios...", context=output_context - ) - if not status_is_tty: - progress_stream.write(f"{activity}\n\n") - progress_stream.flush() - results: tuple[ScenarioSnapshotCaptureRunResult, ...] = run_scenario_capture_pipeline( - project_dir=project_dir, - pipeline_result=pipeline_result, - scenarios=scenarios, - connection_config=connection_config, - adapter=adapter, - project_name=project_name, - settings=settings, - connection_hooks=ConnectionHooks( - on_connection_start=execution_connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - on_scenario_start=lambda _scenario: ( - scenario_status.start(activity) if status_is_tty else None - ), - on_scenario_complete=lambda _scenario, scenario_plan, result: _complete_capture_run( - scenario_status=scenario_status, - status_is_tty=status_is_tty, - project_dir=project_dir, - scenario_plan=scenario_plan, - result=result, - progress_stream=progress_stream, - use_color=use_color, - ), - ) - scenario_status.close() - if capture_results_out is not None: - capture_results_out.extend(results) - pass_count: int = sum(1 for result in results if result.status == SUCCESS_STATUS) - fail_count: int = len(results) - pass_count - progress_stream.write( - "\n" - + format_summary_footer( - counts=( - ("PASS", pass_count), - ("FAIL", fail_count), - ("TOTAL", len(results)), - ), - use_color=use_color, - ) - + "\n" - ) - progress_stream.flush() - write_namespace_completion(context=output_context, succeeded=fail_count == 0) - return (0 if fail_count == 0 else 1), capture_results_out - - -def _complete_capture_run( - *, - scenario_status: TransientStatusReporter, - status_is_tty: bool, - project_dir: Path, - scenario_plan: ScenarioExecutionPlan | None, - result: ScenarioSnapshotCaptureRunResult, - progress_stream: TextIO, - use_color: bool, -) -> None: - if status_is_tty: - scenario_status.close() - _write_capture_result( - result=result, - scenario_plan=scenario_plan, - project_dir=project_dir, - stream=progress_stream, - use_color=use_color, - ) - - -def _write_capture_result( - *, - result: ScenarioSnapshotCaptureRunResult, - scenario_plan: ScenarioExecutionPlan | None, - project_dir: Path, - stream: TextIO, - use_color: bool, -) -> None: - status_text: str = "PASS" if result.status == SUCCESS_STATUS else "FAIL" - style: CliStyle = CliStyle(use_color=use_color) - status: str = style.status(status=status_text) - detail: str = _capture_detail(result) - stream.write(f"{result.scenario_name:<{_SCENARIO_NAME_WIDTH}} {status}{detail}\n") - if result.error_message: - rendered_error_message: str = render_result_error( - error_code=result.error_code, - error_message=result.error_message, - error_help=result.error_help, - use_color=use_color, - ) - error_line: str - for error_line in rendered_error_message.splitlines(): - stream.write(f" {error_line}\n") - if not result.retained: - stream.write(" Rerun with --retain to inspect scenario-owned artifacts.\n") - if result.capture_result is not None and result.capture_result.manifest_path is not None: - _write_capture_relation_rows(result=result, stream=stream, use_color=use_color) - snapshot_path: str = _display_snapshot_path( - manifest_path=result.capture_result.manifest_path, - project_dir=project_dir, - ) - stream.write(f" {'snapshot':<{_CAPTURE_RELATION_KIND_WIDTH}} {snapshot_path}\n") - if result.retained and scenario_plan is not None: - stream.write(" Retained relations:\n") - fixture_result: ScenarioFixtureExecutionResult - for fixture_result in result.fixture_results: - stream.write( - f" {fixture_result.kind.value:<6} " - f"{fixture_result.logical_name} -> {fixture_result.target_relation}\n" - ) - seed_result: SeedExecutionResult - for seed_result in result.seed_results: - stream.write(f" seed {seed_result.seed_name}\n") - stream.flush() - - -def _write_capture_relation_rows( - *, result: ScenarioSnapshotCaptureRunResult, stream: TextIO, use_color: bool -) -> None: - capture_result: ScenarioSnapshotCaptureResult | None = result.capture_result - if capture_result is None: - return - relation_result: ScenarioSnapshotCaptureRelationResult - for relation_result in capture_result.relation_results: - status_text: str = "PASS" if relation_result.status == SUCCESS_STATUS else "FAIL" - style: CliStyle = CliStyle(use_color=use_color) - status: str = style.status(status=status_text) - row_label: str = "row" if relation_result.row_count == 1 else "rows" - detail: str = ( - f" {relation_result.row_count} {row_label}, " - f"{_format_snapshot_size(relation_result.byte_count)}" - ) - stream.write( - f" {relation_result.kind.value:<{_CAPTURE_RELATION_KIND_WIDTH}} " - f"{relation_result.logical_name:<{_CAPTURE_RELATION_NAME_WIDTH}} " - f"{status}{detail}\n" - ) - - -def _format_snapshot_size(byte_count: int) -> str: - binary_unit_size: int = 1024 - if byte_count < binary_unit_size: - return f"{byte_count} B" - kibibytes: float = byte_count / 1024 - if kibibytes < binary_unit_size: - return f"{kibibytes:.1f} KB" - mebibytes: float = kibibytes / 1024 - return f"{mebibytes:.1f} MB" - - -def _display_snapshot_path(*, manifest_path: Path, project_dir: Path) -> str: - try: - return manifest_path.relative_to(project_dir).as_posix() - except ValueError: - return manifest_path.as_posix() - - -def _capture_detail(result: ScenarioSnapshotCaptureRunResult) -> str: - capture_result: ScenarioSnapshotCaptureResult | None = result.capture_result - if capture_result is None: - return "" - relation_results: tuple[ScenarioSnapshotCaptureRelationResult, ...] = ( - capture_result.relation_results - ) - relation_count: int = sum( - 1 for relation in relation_results if relation.status == SUCCESS_STATUS - ) - row_count: int = sum(relation.row_count for relation in relation_results) - relation_label: str = "relation" if relation_count == 1 else "relations" - row_label: str = "row" if row_count == 1 else "rows" - return f" {relation_count} {relation_label}, {row_count} {row_label}" - - -def build_scenario_capture_settings( - *, - capture_adapter: str, - capture_dialect: str, - retain: bool, - limits: ScenarioSnapshotCaptureLimits, -) -> ScenarioCaptureSettings: - """Build capture settings with current provenance for a scenario capture run.""" - - return ScenarioCaptureSettings( - captured_at=_captured_at(), - capture_adapter=capture_adapter, - capture_dialect=capture_dialect, - sqlbuild_version=_sqlbuild_version(), - retain=retain, - limits=limits, - ) - - -def _captured_at() -> str: - return datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _sqlbuild_version() -> str: - try: - return version("sqlbuild") - except PackageNotFoundError: - return "unknown" diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/dialect.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/dialect.py deleted file mode 100644 index 93d6143b99..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/dialect.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Scenario dialect helpers.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.constants import ( - SCENARIO_CLI_CAPTURE_DIALECT_REQUIRED, -) -from sqlbuild.cli.commands.exceptions import CliUserError - - -def require_scenario_capture_dialect(*, adapter: BaseAdapter, adapter_name: str) -> str: - """Return the SQL analysis dialect required for scenario snapshot compatibility.""" - - dialect: str | None = adapter.sql_analysis_dialect() - if dialect is None: - raise CliUserError( - f"Adapter '{adapter_name}' must define a SQL analysis dialect for scenario snapshots", - code=SCENARIO_CLI_CAPTURE_DIALECT_REQUIRED, - help=( - "Set sql_analysis_dialect_name on the adapter so captured snapshots can be " - "validated and replayed locally." - ), - ) - return dialect diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/snapshot_limits.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/snapshot_limits.py deleted file mode 100644 index 0e8c7fdb4a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/snapshot_limits.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Scenario snapshot capture limit helpers.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.constants import ( - DEFAULT_MAX_SNAPSHOT_BYTES_PER_RELATION, - DEFAULT_MAX_SNAPSHOT_ROWS_PER_RELATION, - DEFAULT_MAX_SNAPSHOT_TOTAL_BYTES, - DEFAULT_MAX_SNAPSHOT_TOTAL_ROWS, -) -from sqlbuild.cli.commands.models import ScenarioSnapshotLimitInputs -from sqlbuild.executor.scenario.models import ScenarioSnapshotCaptureLimits -from sqlbuild.spec.contracts.models import ScenarioConfig - - -def build_scenario_snapshot_capture_limits( - *, - scenario_config: ScenarioConfig, - limit_inputs: ScenarioSnapshotLimitInputs, -) -> ScenarioSnapshotCaptureLimits: - """Build scenario snapshot capture limits from CLI options.""" - - return ScenarioSnapshotCaptureLimits( - max_rows_per_relation=_resolve_limit( - cli_value=limit_inputs.max_snapshot_rows, - config_value=scenario_config.snapshot_limits.max_rows_per_relation, - default_value=DEFAULT_MAX_SNAPSHOT_ROWS_PER_RELATION, - ), - max_total_rows=_resolve_limit( - cli_value=limit_inputs.max_snapshot_total_rows, - config_value=scenario_config.snapshot_limits.max_total_rows, - default_value=DEFAULT_MAX_SNAPSHOT_TOTAL_ROWS, - ), - max_bytes_per_relation=_resolve_limit( - cli_value=limit_inputs.max_snapshot_bytes, - config_value=scenario_config.snapshot_limits.max_bytes_per_relation, - default_value=DEFAULT_MAX_SNAPSHOT_BYTES_PER_RELATION, - ), - max_total_bytes=_resolve_limit( - cli_value=limit_inputs.max_snapshot_total_bytes, - config_value=scenario_config.snapshot_limits.max_total_bytes, - default_value=DEFAULT_MAX_SNAPSHOT_TOTAL_BYTES, - ), - force=limit_inputs.force, - ) - - -def _resolve_limit(*, cli_value: int | None, config_value: int | None, default_value: int) -> int: - if cli_value is not None: - return cli_value - if config_value is not None: - return config_value - return default_value - - -def scenario_snapshot_capture_warning(*, force: bool) -> str: - """Return a warning shown before committable snapshot files are written.""" - - suffix: str = " Size limits are bypassed." if force else "" - return ( - "Review captured scenario snapshots before committing; they may contain sensitive " - f"warehouse data.{suffix}" - ) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/__init__.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/local_run.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/local_run.py deleted file mode 100644 index bc581ca271..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/local_run.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Shared local DuckDB scenario replay execution.""" - -from __future__ import annotations - -from collections import Counter -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.scenario_output.result_output import complete_scenario_run -from sqlbuild.cli.commands._helpers.scenario_output.run_presentation import ( - begin_scenario_run, - finish_scenario_run, -) -from sqlbuild.cli.commands.models import ScenarioRunOutputContext, ScenarioRunPresentation -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.pipeline.main.run import run_scenario_local_test_pipeline -from sqlbuild.executor.scenario.models import ScenarioLocalReplaySource, ScenarioRunResult -from sqlbuild.executor.scenario.types import ScenarioLocalRunStatus - - -def run_local_scenarios( - *, - project_dir: Path, - pipeline_result: CompilePipelineResult, - scenarios: tuple[CompiledSqlScenario, ...], - adapter: BaseAdapter, - project_name: str, - strict: bool, - replay_source: ScenarioLocalReplaySource, - target_dir: Path, - output_context: ScenarioRunOutputContext, -) -> int: - """Replay selected scenarios on run-scoped DuckDB and render results.""" - - presentation: ScenarioRunPresentation = begin_scenario_run( - context=output_context, scenario_count=len(scenarios) - ) - results: tuple[ScenarioRunResult, ...] = run_scenario_local_test_pipeline( - project_dir=project_dir, - pipeline_result=pipeline_result, - scenarios=scenarios, - adapter=adapter, - project_name=project_name, - strict=strict, - replay_source=replay_source, - on_scenario_start=lambda _scenario: ( - presentation.scenario_status.start(presentation.activity) - if presentation.status_is_tty - else None - ), - on_scenario_complete=lambda _scenario, scenario_plan, result: complete_scenario_run( - scenario_status=presentation.scenario_status, - status_is_tty=presentation.status_is_tty, - target_dir=target_dir, - adapter=adapter, - scenario_plan=scenario_plan, - result=result, - progress_stream=output_context.progress_stream, - use_color=output_context.use_color, - ), - ) - counts: Counter[ScenarioLocalRunStatus | None] = Counter( - result.local_status for result in results - ) - succeeded: bool = ( - counts[ScenarioLocalRunStatus.FAIL] == 0 and counts[ScenarioLocalRunStatus.ERROR] == 0 - ) - finish_scenario_run( - context=output_context, - presentation=presentation, - results=results, - counts=( - ("PASS", counts[ScenarioLocalRunStatus.PASS]), - ("FAIL", counts[ScenarioLocalRunStatus.FAIL]), - ("ERROR", counts[ScenarioLocalRunStatus.ERROR]), - ("SKIP", counts[ScenarioLocalRunStatus.SKIP]), - ("TOTAL", len(results)), - ), - succeeded=succeeded, - local=True, - ) - return 0 if succeeded else 1 diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/namespace.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/namespace.py deleted file mode 100644 index a200adfe9c..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/namespace.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Resolve scenario namespace overrides before planning or warehouse work.""" - -from __future__ import annotations - -import os -from dataclasses import replace - -from sqlbuild.cli.commands.constants import SCENARIO_CLI_INVALID_NAMESPACE -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ScenarioRunNamespace -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.parse_scenario_run_namespace import ( - parse_scenario_run_namespace, -) - - -def resolve_scenario_namespace( - *, inputs: DiscoveredProjectInputs, cli_value: str | None -) -> tuple[DiscoveredProjectInputs, ScenarioRunNamespace]: - """Apply CLI > environment > local config > project config precedence.""" - - candidates: tuple[tuple[str | None, str], ...] = ( - (cli_value, "cli"), - (os.environ.get("SQLBUILD_SCENARIO_NAMESPACE"), "env"), - (inputs.local_config.scenario.run_namespace, "local config"), - (inputs.project_config.scenario.run_namespace, "project config"), - ) - value: str | None - source: str - for value, source in candidates: - if value is None: - continue - try: - value = parse_scenario_run_namespace(value) - except SpecConfigError as exc: - raise CliUserError(str(exc), code=SCENARIO_CLI_INVALID_NAMESPACE) from exc - return replace( - inputs, - local_config=replace( - inputs.local_config, - scenario=replace(inputs.local_config.scenario, run_namespace=value), - ), - ), ScenarioRunNamespace(value=value, source=source) - return inputs, ScenarioRunNamespace() diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py deleted file mode 100644 index cfece81eda..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py +++ /dev/null @@ -1,560 +0,0 @@ -"""Scenario command runner.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ( - ConnectionProgressReporter, - PlanningProgressReporter, - discover_project_inputs, - format_scenario_snapshot_execution_json, - format_summary_footer, - resolve_project_connection_config, - run_compile_only_pipeline, - write_execution_header, - write_execution_json_output, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.types import BuiltinAdapter -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.scenario_capture.capture_run import ( - build_scenario_capture_settings, -) -from sqlbuild.cli.commands._helpers.scenario_capture.dialect import ( - require_scenario_capture_dialect, -) -from sqlbuild.cli.commands._helpers.scenario_capture.snapshot_limits import ( - build_scenario_snapshot_capture_limits, - scenario_snapshot_capture_warning, -) -from sqlbuild.cli.commands._helpers.scenario_execution.local_run import run_local_scenarios -from sqlbuild.cli.commands._helpers.scenario_execution.namespace import resolve_scenario_namespace -from sqlbuild.cli.commands._helpers.scenario_execution.selection import select_scenarios -from sqlbuild.cli.commands._helpers.scenario_execution.warehouse_run import ( - run_warehouse_scenarios, -) -from sqlbuild.cli.commands._helpers.scenario_output.result_output import render_result_error -from sqlbuild.cli.commands.constants import ( - SCENARIO_CLI_LOCAL_RETAIN_UNSUPPORTED, - SCENARIO_CLI_LOCAL_SNAPSHOT_FLAG_REQUIRED, - SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - SUCCESS_STATUS, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import ( - LocalSnapshotSyncInputs, - ScenarioRunOutputContext, - ScenarioSnapshotLimitInputs, - ScenarioTestCommandRequest, -) -from sqlbuild.compiler.compile.main.effective_settings import build_effective_settings_config -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.discovery.main.sql_analysis_off_guidance import sql_analysis_off_guidance -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ( - CompilePipelineOptions, - CompilePipelineResult, -) -from sqlbuild.executor.pipeline.main.run import ( - run_scenario_capture_pipeline, - select_scenario_snapshot_capture_candidates, -) -from sqlbuild.executor.scenario.models import ( - ScenarioLocalReplaySource, - ScenarioSnapshotCaptureLimits, - ScenarioSnapshotCaptureRelationResult, - ScenarioSnapshotCaptureRunResult, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( - resolve_effective_scenario_config, -) - -_SCENARIO_NAME_WIDTH: int = 64 -_CAPTURE_RELATION_KIND_WIDTH: int = 8 -_CAPTURE_RELATION_NAME_WIDTH: int = _SCENARIO_NAME_WIDTH - 4 - _CAPTURE_RELATION_KIND_WIDTH - 1 - - -def run_scenario(request: ScenarioTestCommandRequest) -> int: - """Execute the scenario test command.""" - - project_dir: Path | None = request.project_dir - no_sql_validation: bool = request.no_sql_validation - no_color: bool = request.no_color - selectors: tuple[str, ...] = request.selectors - exclude: tuple[str, ...] = request.exclude - retain: bool = request.retain - local: bool = request.local - strict: bool = request.strict - sync_snapshots: bool = request.sync_snapshots - refresh: bool = request.refresh - limit_inputs: ScenarioSnapshotLimitInputs = request.limit_inputs - json_output: bool = request.json_output - json_output_path: Path | None = request.json_output_path - if local and retain: - raise CliUserError( - "scenario test --local does not support --retain", - code=SCENARIO_CLI_LOCAL_RETAIN_UNSUPPORTED, - help=("Local scenario DuckDB files are always kept under target/run/scenarios/."), - ) - if not local and (sync_snapshots or refresh): - raise CliUserError( - "scenario snapshot sync flags require --local", - code=SCENARIO_CLI_LOCAL_SNAPSHOT_FLAG_REQUIRED, - help=( - "Use sqb scenario test --local --sync-snapshots or " - "sqb scenario test --local --refresh." - ), - ) - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - reject_unit_test_selectors( - select=selectors, exclude=exclude, discovered_inputs=discovered_inputs - ) - discovered_inputs, namespace = resolve_scenario_namespace( - inputs=discovered_inputs, cli_value=request.scenario_namespace - ) - if local: - _validate_local_scenario_sql_analysis_enabled( - discovered_inputs=discovered_inputs, - no_sql_validation=no_sql_validation, - ) - project_adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter_name: str = BuiltinAdapter.DUCKDB.value if local else project_adapter_name - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - project_adapter: BaseAdapter = resolve_adapter( - adapter_name=project_adapter_name, - project_dir=effective_project_dir, - ) - local_capture_dialect: str = "" - if local: - local_capture_dialect = require_scenario_capture_dialect( - adapter=project_adapter, adapter_name=project_adapter_name - ) - connection_config: dict[str, object] = ( - {"database": ":memory:"} - if local - else resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - ) - ) - use_color: bool = not no_color and supports_color() - concurrency: int = max( - 1, - request.concurrency - if request.concurrency is not None - else build_effective_settings_config(discovered_inputs=discovered_inputs).concurrency, - ) - progress_stream: TextIO = sys.stderr if json_output else sys.stdout - progress_stream.write( - f"Scenario namespace: {namespace.value or '(unset)'} (source: {namespace.source})\n" - ) - target_label: str | None = " ".join(selectors) if selectors else None - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=progress_stream, - use_color=use_color, - ) - progress_stream.write("\n") - write_execution_header( - stream=progress_stream, - command="sqb scenario test --local" if local else "sqb scenario test", - target=target_label, - concurrency=1 if local else concurrency, - use_color=use_color, - ) - - try: - with OperationLifecycle(operation_kind="project", operation_name="project_compile"): - pipeline_result: CompilePipelineResult = run_compile_only_pipeline( - discovered_inputs=discovered_inputs, - adapter=adapter, - options=CompilePipelineOptions( - no_sql_validation=True if local else no_sql_validation, - source_deferral_enabled=False, - connection_config=connection_config, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - ), - ), - on_progress=planning_progress.on_progress, - ) - finally: - planning_progress.finish() - scenarios: tuple[CompiledSqlScenario, ...] = select_scenarios( - project=pipeline_result.project, - selectors=selectors, - exclude=exclude, - project_dir=effective_project_dir, - ) - if local and (sync_snapshots or refresh): - capture_results: list[ScenarioSnapshotCaptureRunResult] = [] - capture_exit_code, capture_results = _sync_local_snapshots( - inputs=LocalSnapshotSyncInputs( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - local_pipeline_result=pipeline_result, - local_scenarios=scenarios, - local_adapter=adapter, - project_adapter=project_adapter, - project_adapter_name=project_adapter_name, - capture_dialect=local_capture_dialect, - project_connection_config=resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - ), - project_name=discovered_inputs.project_config.name, - no_sql_validation=no_sql_validation, - refresh=refresh, - ), - limit_inputs=limit_inputs, - output_context=ScenarioRunOutputContext( - namespace=namespace, - progress_stream=progress_stream, - use_color=use_color, - json_output=json_output, - json_output_path=json_output_path, - ), - capture_results_out=capture_results, - ) - if capture_exit_code != 0: - write_execution_json_output( - payload=format_scenario_snapshot_execution_json( - run_namespace=namespace.value, - namespace_source=namespace.source, - results=tuple(capture_results), - refresh=refresh, - ), - json_output=json_output, - json_output_path=json_output_path, - ) - return capture_exit_code - if not local: - return run_warehouse_scenarios( - pipeline_result=pipeline_result, - scenarios=scenarios, - connection_config=connection_config, - adapter=adapter, - adapter_name=adapter_name, - project_name=discovered_inputs.project_config.name, - target_dir=effective_project_dir / "target", - retain=retain, - concurrency=concurrency, - output_context=ScenarioRunOutputContext( - namespace=namespace, - progress_stream=progress_stream, - use_color=use_color, - json_output=json_output, - json_output_path=json_output_path, - ), - ) - return run_local_scenarios( - project_dir=effective_project_dir, - pipeline_result=pipeline_result, - scenarios=scenarios, - adapter=adapter, - project_name=discovered_inputs.project_config.name, - strict=strict, - replay_source=ScenarioLocalReplaySource( - lexical_syntax=project_adapter.sql_lexical_syntax, - capture_adapter=project_adapter_name, - capture_dialect=local_capture_dialect, - ), - target_dir=effective_project_dir / "target", - output_context=ScenarioRunOutputContext( - namespace=namespace, - progress_stream=progress_stream, - use_color=use_color, - json_output=json_output, - json_output_path=json_output_path, - ), - ) - - -def _validate_local_scenario_sql_analysis_enabled( - *, discovered_inputs: DiscoveredProjectInputs, no_sql_validation: bool -) -> None: - if no_sql_validation: - raise CliUserError( - "scenario test --local requires SQL analysis, but `--no-sql-analysis` turns it off", - code=SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - help="run the command without `--no-sql-analysis`", - ) - guidance: tuple[str, str] | None = sql_analysis_off_guidance( - discovered_inputs=discovered_inputs, - purpose="to run local scenario replay, snapshot sync, or snapshot refresh", - ) - if guidance is not None: - note, help_text = guidance - raise CliUserError( - f"scenario test --local requires SQL analysis; {note}", - code=SCENARIO_CLI_SQL_VALIDATION_REQUIRED, - help=help_text, - ) - - -def _sync_local_snapshots( - *, - inputs: LocalSnapshotSyncInputs, - limit_inputs: ScenarioSnapshotLimitInputs, - output_context: ScenarioRunOutputContext, - capture_results_out: list[ScenarioSnapshotCaptureRunResult], -) -> tuple[int, list[ScenarioSnapshotCaptureRunResult]]: - project_dir: Path = inputs.project_dir - discovered_inputs: DiscoveredProjectInputs = inputs.discovered_inputs - project_adapter: BaseAdapter = inputs.project_adapter - project_adapter_name: str = inputs.project_adapter_name - capture_dialect: str = inputs.capture_dialect - project_connection_config: dict[str, object] = inputs.project_connection_config - project_name: str = inputs.project_name - no_sql_validation: bool = inputs.no_sql_validation - refresh: bool = inputs.refresh - force: bool = limit_inputs.force - progress_stream: TextIO = output_context.progress_stream - use_color: bool = output_context.use_color - capture_names: tuple[str, ...] = select_scenario_snapshot_capture_candidates( - project_dir=project_dir, - pipeline_result=inputs.local_pipeline_result, - scenarios=inputs.local_scenarios, - adapter=inputs.local_adapter, - project_name=project_name, - capture_adapter=project_adapter_name, - capture_dialect=capture_dialect, - refresh=refresh, - source_lexical_syntax=project_adapter.sql_lexical_syntax, - ) - if not capture_names: - progress_stream.write("\nSnapshots are fresh.\n") - progress_stream.flush() - return 0, capture_results_out - - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=progress_stream, - use_color=use_color, - ) - try: - with OperationLifecycle(operation_kind="project", operation_name="project_compile"): - project_pipeline_result: CompilePipelineResult = run_compile_only_pipeline( - discovered_inputs=discovered_inputs, - adapter=project_adapter, - options=CompilePipelineOptions( - no_sql_validation=no_sql_validation, - source_deferral_enabled=False, - connection_config=project_connection_config, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - ), - ), - on_progress=planning_progress.on_progress, - ) - finally: - planning_progress.finish() - capture_scenarios: tuple[CompiledSqlScenario, ...] = select_scenarios( - project=project_pipeline_result.project, - selectors=capture_names, - project_dir=project_dir, - ) - phase_name: str = "Refresh" if refresh else "Sync" - header: str = f"Snapshot {phase_name} ({len(capture_scenarios)} selected)" - style: CliStyle = CliStyle(use_color=use_color) - styled_header: str = style.success_strong(header) - progress_stream.write(f"\n{styled_header}\n\n") - progress_stream.write(f"{scenario_snapshot_capture_warning(force=force)}\n\n") - progress_stream.flush() - scenario_status: TransientStatusReporter = TransientStatusReporter( - stream=progress_stream, - use_color=use_color, - ) - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=project_adapter_name, - blank_line_after_complete=True, - stream=progress_stream, - use_color=use_color, - ) - status_is_tty: bool = hasattr(progress_stream, "isatty") and progress_stream.isatty() - if not status_is_tty: - progress_stream.write("Capturing snapshots...\n\n") - progress_stream.flush() - capture_limits: ScenarioSnapshotCaptureLimits = build_scenario_snapshot_capture_limits( - scenario_config=resolve_effective_scenario_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - limit_inputs=limit_inputs, - ) - results: tuple[ScenarioSnapshotCaptureRunResult, ...] = run_scenario_capture_pipeline( - project_dir=project_dir, - pipeline_result=project_pipeline_result, - scenarios=capture_scenarios, - connection_config=project_connection_config, - adapter=project_adapter, - project_name=project_name, - settings=build_scenario_capture_settings( - capture_adapter=project_adapter_name, - capture_dialect=capture_dialect, - retain=False, - limits=capture_limits, - ), - connection_hooks=ConnectionHooks( - on_connection_start=execution_connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - on_scenario_start=lambda _scenario: ( - scenario_status.start("Capturing snapshots...") if status_is_tty else None - ), - on_scenario_complete=lambda _scenario, _scenario_plan, result: _complete_snapshot_sync( - scenario_status=scenario_status, - status_is_tty=status_is_tty, - project_dir=project_dir, - refresh=refresh, - result=result, - progress_stream=progress_stream, - use_color=use_color, - ), - ) - if capture_results_out is not None: - capture_results_out.extend(results) - scenario_status.close() - pass_count: int = sum(1 for result in results if result.status == SUCCESS_STATUS) - fail_count: int = len(results) - pass_count - summary_prefix: str = "REFRESH" if refresh else "SYNC" - progress_stream.write( - "\n" - + format_summary_footer( - counts=( - (f"{summary_prefix}_PASS", pass_count), - (f"{summary_prefix}_FAIL", fail_count), - ), - use_color=use_color, - ) - + "\n" - ) - progress_stream.flush() - return (0 if fail_count == 0 else 1), capture_results_out - - -def _complete_snapshot_sync( - *, - scenario_status: TransientStatusReporter, - status_is_tty: bool, - project_dir: Path, - refresh: bool, - result: ScenarioSnapshotCaptureRunResult, - progress_stream: TextIO, - use_color: bool, -) -> None: - if status_is_tty: - scenario_status.close() - success_status_text: str = "REFRESHED" if refresh else "CAPTURED" - status_text: str = success_status_text if result.status == SUCCESS_STATUS else "FAIL" - style: CliStyle = CliStyle(use_color=use_color) - status: str = style.status(status=status_text) - detail: str = _snapshot_capture_detail(result) - progress_stream.write(f"{result.scenario_name:<{_SCENARIO_NAME_WIDTH}} {status}{detail}\n") - if result.error_message: - rendered_error_message: str = render_result_error( - error_code=result.error_code, - error_message=result.error_message, - error_help=result.error_help, - use_color=use_color, - ) - error_line: str - for error_line in rendered_error_message.splitlines(): - progress_stream.write(f" {error_line}\n") - _write_snapshot_capture_relation_rows( - result=result, stream=progress_stream, use_color=use_color - ) - if result.capture_result is not None and result.capture_result.manifest_path is not None: - snapshot_path: str = _display_snapshot_path( - manifest_path=result.capture_result.manifest_path, - project_dir=project_dir, - ) - progress_stream.write(f" {'snapshot':<{_CAPTURE_RELATION_KIND_WIDTH}} {snapshot_path}\n") - progress_stream.flush() - - -def _write_snapshot_capture_relation_rows( - *, result: ScenarioSnapshotCaptureRunResult, stream: TextIO, use_color: bool -) -> None: - if result.capture_result is None: - return - relation_result: ScenarioSnapshotCaptureRelationResult - for relation_result in result.capture_result.relation_results: - status_text: str = "PASS" if relation_result.status == SUCCESS_STATUS else "FAIL" - style: CliStyle = CliStyle(use_color=use_color) - status: str = style.status(status=status_text) - row_label: str = "row" if relation_result.row_count == 1 else "rows" - detail: str = ( - f" {relation_result.row_count} {row_label}, " - f"{_format_snapshot_size(relation_result.byte_count)}" - ) - stream.write( - f" {relation_result.kind.value:<{_CAPTURE_RELATION_KIND_WIDTH}} " - f"{relation_result.logical_name:<{_CAPTURE_RELATION_NAME_WIDTH}} " - f"{status}{detail}\n" - ) - - -def _snapshot_capture_detail(result: ScenarioSnapshotCaptureRunResult) -> str: - if result.capture_result is None: - return "" - relation_results: tuple[ScenarioSnapshotCaptureRelationResult, ...] = ( - result.capture_result.relation_results - ) - relation_count: int = sum( - 1 for relation in relation_results if relation.status == SUCCESS_STATUS - ) - row_count: int = sum(relation.row_count for relation in relation_results) - relation_label: str = "relation" if relation_count == 1 else "relations" - row_label: str = "row" if row_count == 1 else "rows" - return f" {relation_count} {relation_label}, {row_count} {row_label}" - - -def _format_snapshot_size(byte_count: int) -> str: - binary_unit_size: int = 1024 - if byte_count < binary_unit_size: - return f"{byte_count} B" - kibibytes: float = byte_count / 1024 - if kibibytes < binary_unit_size: - return f"{kibibytes:.1f} KB" - mebibytes: float = kibibytes / 1024 - return f"{mebibytes:.1f} MB" - - -def _display_snapshot_path(*, manifest_path: Path, project_dir: Path) -> str: - try: - return manifest_path.relative_to(project_dir).as_posix() - except ValueError: - return manifest_path.as_posix() diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/selection.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/selection.py deleted file mode 100644 index 2874d350c8..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/selection.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Scenario selector helpers shared by scenario CLI commands.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.commands.constants import ( - GRAPH_SELECTOR_EXPANSION_MARKER, - GRAPH_SELECTOR_PATH_MARKER, - SCENARIO_CLI_NONE_DISCOVERED, - SCENARIO_CLI_UNKNOWN_SELECTOR, - SCENARIO_CLI_UNSUPPORTED_GRAPH_SELECTOR, - SQL_FILE_SUFFIX, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.compile.models import ( - CompiledSqlScenario, -) -from sqlbuild.compiler.compile.types import CompiledProjectView - - -def select_scenarios( - *, - project: CompiledProjectView, - selectors: tuple[str, ...], - exclude: tuple[str, ...] = (), - project_dir: Path, -) -> tuple[CompiledSqlScenario, ...]: - """Resolve scenario names, files, and folder selectors into scenarios.""" - - if not selectors: - if not project.sql_scenarios: - raise CliUserError( - "No SQL scenarios were discovered under tests/scenarios", - code=SCENARIO_CLI_NONE_DISCOVERED, - ) - selected_scenarios: tuple[CompiledSqlScenario, ...] = project.sql_scenarios - else: - selected: list[CompiledSqlScenario] = [] - selected_names: set[str] = set() - selector: str - for selector in selectors: - matches: tuple[CompiledSqlScenario, ...] = _select_scenarios_for_selector( - project=project, - selector=selector, - project_dir=project_dir, - ) - scenario: CompiledSqlScenario - for scenario in matches: - if scenario.name in selected_names: - continue - selected.append(scenario) - selected_names.add(scenario.name) - selected_scenarios = tuple(selected) - - if not exclude: - return selected_scenarios - - excluded_names: set[str] = set() - exclude_selector: str - for exclude_selector in exclude: - excluded_names.update( - scenario.name - for scenario in _select_scenarios_for_selector( - project=project, - selector=exclude_selector, - project_dir=project_dir, - ) - ) - return tuple(scenario for scenario in selected_scenarios if scenario.name not in excluded_names) - - -def _select_scenarios_for_selector( - *, project: CompiledProjectView, selector: str, project_dir: Path -) -> tuple[CompiledSqlScenario, ...]: - _validate_scenario_selector(selector) - selector_path: Path = Path(selector) - matches: list[CompiledSqlScenario] = [] - scenario: CompiledSqlScenario - for scenario in project.sql_scenarios: - if scenario.name == selector: - matches.append(scenario) - continue - scenario_path: Path = scenario.scenario_file.file_path - scenario_relative_path: Path = scenario.scenario_file.relative_path - scenario_root_relative_path: Path = _scenario_root_relative_path(scenario_relative_path) - if selector_path.suffix == SQL_FILE_SUFFIX and _scenario_file_matches_selector( - selector_path=selector_path, - project_dir=project_dir, - scenario_path=scenario_path, - scenario_relative_path=scenario_relative_path, - scenario_root_relative_path=scenario_root_relative_path, - ): - matches.append(scenario) - continue - if selector_path.suffix != SQL_FILE_SUFFIX and _scenario_path_is_under_selector( - selector_path=selector_path, - project_dir=project_dir, - scenario_path=scenario_path, - scenario_relative_path=scenario_relative_path, - scenario_root_relative_path=scenario_root_relative_path, - ): - matches.append(scenario) - if matches: - return tuple(matches) - raise CliUserError( - f"Unknown scenario selector '{selector}'", - code=SCENARIO_CLI_UNKNOWN_SELECTOR, - ) - - -def _validate_scenario_selector(selector: str) -> None: - if ( - selector.startswith(GRAPH_SELECTOR_EXPANSION_MARKER) - or selector.endswith(GRAPH_SELECTOR_EXPANSION_MARKER) - or GRAPH_SELECTOR_PATH_MARKER in selector - ): - raise CliUserError( - f"Scenario selector '{selector}' uses graph operators, which are not supported", - code=SCENARIO_CLI_UNSUPPORTED_GRAPH_SELECTOR, - help=( - "Use scenario names, scenario file paths, folders, --select-file, and --exclude. " - "Graph expansion operators such as '+model', 'model+', and 'a~b' are only " - "supported by graph resource commands." - ), - ) - - -def _scenario_file_matches_selector( - *, - selector_path: Path, - project_dir: Path, - scenario_path: Path, - scenario_relative_path: Path, - scenario_root_relative_path: Path, -) -> bool: - return selector_path in ( - scenario_path, - project_dir / selector_path, - scenario_relative_path, - scenario_root_relative_path, - ) - - -def _scenario_path_is_under_selector( - *, - selector_path: Path, - project_dir: Path, - scenario_path: Path, - scenario_relative_path: Path, - scenario_root_relative_path: Path, -) -> bool: - selector_candidates: tuple[Path, ...] = (selector_path, project_dir / selector_path) - candidate: Path - for candidate in selector_candidates: - if _path_is_under(path=scenario_path, prefix=candidate): - return True - return _path_is_under(path=scenario_relative_path, prefix=selector_path) or _path_is_under( - path=scenario_root_relative_path, - prefix=selector_path, - ) - - -def _path_is_under(*, path: Path, prefix: Path) -> bool: - return path == prefix or prefix in path.parents - - -def _scenario_root_relative_path(relative_path: Path) -> Path: - scenario_root: Path = Path("tests") / "scenarios" - try: - return relative_path.relative_to(scenario_root) - except ValueError: - return relative_path diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py deleted file mode 100644 index b7eb01a152..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Shared warehouse-direct scenario execution.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import ConnectionProgressReporter -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.scenario_output.result_output import complete_scenario_run -from sqlbuild.cli.commands._helpers.scenario_output.run_presentation import ( - begin_scenario_run, - finish_scenario_run, -) -from sqlbuild.cli.commands.constants import SUCCESS_STATUS -from sqlbuild.cli.commands.models import ScenarioRunOutputContext, ScenarioRunPresentation -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.pipeline.main.run import run_scenario_test_pipeline -from sqlbuild.executor.scenario.models import ScenarioRunResult -from sqlbuild.runtime.contracts.models import ConnectionHooks - - -def run_warehouse_scenarios( - *, - pipeline_result: CompilePipelineResult, - scenarios: tuple[CompiledSqlScenario, ...], - connection_config: dict[str, object], - adapter: BaseAdapter, - adapter_name: str, - project_name: str, - target_dir: Path, - retain: bool, - concurrency: int, - output_context: ScenarioRunOutputContext, -) -> int: - """Run selected scenarios warehouse-direct and render results.""" - - presentation: ScenarioRunPresentation = begin_scenario_run( - context=output_context, scenario_count=len(scenarios) - ) - execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=adapter_name, - blank_line_after_complete=True, - stream=output_context.progress_stream, - use_color=output_context.use_color, - ) - results: tuple[ScenarioRunResult, ...] = run_scenario_test_pipeline( - pipeline_result=pipeline_result, - scenarios=scenarios, - connection_config=connection_config, - adapter=adapter, - project_name=project_name, - retain=retain, - concurrency=concurrency, - connection_hooks=ConnectionHooks( - on_connection_start=execution_connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - execution_connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - on_scenario_start=lambda _scenario: ( - presentation.scenario_status.start(presentation.activity) - if presentation.status_is_tty - else None - ), - on_scenario_complete=lambda _scenario, scenario_plan, result: complete_scenario_run( - scenario_status=presentation.scenario_status, - status_is_tty=presentation.status_is_tty, - target_dir=target_dir, - adapter=adapter, - scenario_plan=scenario_plan, - result=result, - progress_stream=output_context.progress_stream, - use_color=output_context.use_color, - ), - ) - pass_count: int = sum(1 for result in results if result.status == SUCCESS_STATUS) - fail_count: int = len(results) - pass_count - finish_scenario_run( - context=output_context, - presentation=presentation, - results=results, - counts=(("PASS", pass_count), ("FAIL", fail_count), ("TOTAL", len(results))), - succeeded=fail_count == 0, - local=False, - ) - return 0 if fail_count == 0 else 1 diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_output/__init__.py b/src/sqlbuild/cli/commands/_helpers/scenario_output/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_output/namespace.py b/src/sqlbuild/cli/commands/_helpers/scenario_output/namespace.py deleted file mode 100644 index 988c1b1b32..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_output/namespace.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Namespace attribution for scenario lifecycle progress.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.models import ScenarioRunOutputContext - - -def scenario_activity_message(*, activity: str, context: ScenarioRunOutputContext) -> str: - """Label scenario work with the effective namespace and its source.""" - return ( - f"{activity} (namespace: {context.namespace.value or '(unset)'}; " - f"source: {context.namespace.source})" - ) - - -def write_namespace_completion(*, context: ScenarioRunOutputContext, succeeded: bool) -> None: - """Emit the explicit terminal state for the namespaced run.""" - message: str = scenario_activity_message( - activity="Scenario execution complete" if succeeded else "Scenario execution failed", - context=context, - ) - context.progress_stream.write(f"{message}\n") - context.progress_stream.flush() diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_output/result_output.py b/src/sqlbuild/cli/commands/_helpers/scenario_output/result_output.py deleted file mode 100644 index 6f7279885a..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_output/result_output.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Scenario result row rendering shared across warehouse, local, and snapshot paths.""" - -from __future__ import annotations - -from pathlib import Path -from typing import TextIO - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.scenario_output.scenario_runtime import ( - write_local_scenario_runtime_target, - write_scenario_runtime_target, -) -from sqlbuild.cli.commands.constants import ( - FAILED_STATUS, - SUCCESS_STATUS, -) -from sqlbuild.compiler.planner.models import ScenarioArtifactName, ScenarioExecutionPlan -from sqlbuild.compiler.planner.types import ScenarioArtifactKind -from sqlbuild.executor.scenario.models import ( - ScenarioAssertionExpectationExecutionResult, - ScenarioExpectedExpectationExecutionResult, - ScenarioRunResult, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.coded_error_text import format_coded_error - -_SCENARIO_NAME_WIDTH: int = 64 -_EXPECTATION_LABEL_WIDTH: int = 10 -_EXPECTATION_NAME_WIDTH: int = 50 - - -def complete_scenario_run( - *, - scenario_status: TransientStatusReporter, - status_is_tty: bool, - target_dir: Path, - adapter: BaseAdapter, - scenario_plan: ScenarioExecutionPlan | None, - result: ScenarioRunResult, - progress_stream: TextIO, - use_color: bool, -) -> None: - """Render one completed scenario row and persist its runtime target.""" - - if status_is_tty: - scenario_status.close() - if scenario_plan is not None and result.local_status is None: - write_scenario_runtime_target( - target_dir=target_dir, - adapter=adapter, - scenario_plan=scenario_plan, - result=result, - ) - if scenario_plan is not None and result.local_status is not None: - write_local_scenario_runtime_target( - target_dir=target_dir, - adapter=adapter, - scenario_plan=scenario_plan, - result=result, - ) - write_scenario_result(result=result, stream=progress_stream, use_color=use_color) - - -def write_scenario_result(*, result: ScenarioRunResult, stream: TextIO, use_color: bool) -> None: - """Render one scenario result with expectations and retained artifacts.""" - - status_text: str = ( - result.local_status.value - if result.local_status is not None - else "PASS" - if result.status == SUCCESS_STATUS - else "FAIL" - ) - style: CliStyle = CliStyle(use_color=use_color) - status: str = style.status(status=status_text) - duration: str = "" if result.duration_ms is None else f" {result.duration_ms / 1000:.2f}s" - stream.write(f"{result.scenario_name:<{_SCENARIO_NAME_WIDTH}} {status}{duration}\n") - if result.error_message: - rendered_error_message: str = render_result_error( - error_code=result.error_code, - error_message=result.error_message, - error_help=result.error_help, - use_color=use_color, - ) - error_line: str - for error_line in rendered_error_message.splitlines(): - stream.write(f" {error_line}\n") - if not result.retained and result.local_status is None: - stream.write(" Rerun with --retain to inspect scenario-owned artifacts.\n") - _write_expectations(result=result, stream=stream, use_color=use_color) - if result.retained and result.relation_map is not None: - stream.write(" Retained relations:\n") - artifact: ScenarioArtifactName - for artifact in result.relation_map.artifacts: - kind: str = ScenarioArtifactKind(artifact.identity.kind).value - stream.write( - f" {kind:<6} {artifact.identity.logical_name} -> {artifact.physical_name}\n" - ) - if result.retained and result.local_duckdb_path is not None: - stream.write(f" Retained local DuckDB: {result.local_duckdb_path.as_posix()}\n") - _write_expectation_failures(result=result, stream=stream) - stream.flush() - - -def render_result_error( - *, - error_code: str | None, - error_message: str, - error_help: str | None = None, - use_color: bool = False, -) -> str: - """Render a scenario result error with optional code and help.""" - - if error_code is None: - return error_message - return format_coded_error( - code=error_code, - message=error_message, - help=error_help, - use_color=use_color, - ) - - -def _write_expectations(*, result: ScenarioRunResult, stream: TextIO, use_color: bool) -> None: - style: CliStyle = CliStyle(use_color=use_color) - expected: ScenarioExpectedExpectationExecutionResult - for expected in result.expected_results: - status_text: str = "FAIL" if expected.status == FAILED_STATUS else "PASS" - status: str = style.status(status=status_text) - detail: str = "" - if expected.status == FAILED_STATUS: - detail = f" {expected.mismatched_row_count} mismatched" - item_name: str = f"expected {expected.model_name}" - stream.write( - f" {'expect':<{_EXPECTATION_LABEL_WIDTH}}" - f"{item_name:<{_EXPECTATION_NAME_WIDTH}} " - f"{status}{detail}\n" - ) - if expected.error_message is not None: - rendered_error: str = render_result_error( - error_code=expected.error_code, - error_message=expected.error_message, - error_help=expected.error_help, - use_color=use_color, - ) - stream.write(f"{'':>14}{rendered_error}\n") - assertion: ScenarioAssertionExpectationExecutionResult - for assertion in result.assertion_results: - status_text = "FAIL" if assertion.status == FAILED_STATUS else "PASS" - status = style.status(status=status_text) - row_label: str = "row" if assertion.failing_row_count == 1 else "rows" - detail = "" - if assertion.status == FAILED_STATUS: - detail = f" {assertion.failing_row_count} {row_label}" - item_name = f"assertion {assertion.name}" - stream.write( - f" {'expect':<{_EXPECTATION_LABEL_WIDTH}}" - f"{item_name:<{_EXPECTATION_NAME_WIDTH}} " - f"{status}{detail}\n" - ) - if assertion.error_message is not None: - rendered_error = render_result_error( - error_code=assertion.error_code, - error_message=assertion.error_message, - error_help=assertion.error_help, - use_color=use_color, - ) - stream.write(f"{'':>14}{rendered_error}\n") - - -def _write_expectation_failures(*, result: ScenarioRunResult, stream: TextIO) -> None: - expected: ScenarioExpectedExpectationExecutionResult - for expected in result.expected_results: - if expected.status != FAILED_STATUS: - continue - stream.write( - f" expected {expected.model_name}: actual={expected.actual_row_count} " - f"expected={expected.expected_row_count} mismatched={expected.mismatched_row_count}\n" - ) - assertion: ScenarioAssertionExpectationExecutionResult - for assertion in result.assertion_results: - if assertion.status != FAILED_STATUS: - continue - stream.write( - f" assertion {assertion.name}: failing_rows={assertion.failing_row_count}\n" - ) - if assertion.sample_rows: - stream.write(f" sample: {assertion.sample_rows[0]}\n") diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_output/run_presentation.py b/src/sqlbuild/cli/commands/_helpers/scenario_output/run_presentation.py deleted file mode 100644 index f231b54ecf..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_output/run_presentation.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Progress framing shared by warehouse and local scenario test runs.""" - -from __future__ import annotations - -import time - -from sqlbuild._native import ( - format_scenario_execution_json, - format_summary_footer, - write_execution_json_output, -) -from sqlbuild.cli.commands._helpers.scenario_output.namespace import ( - scenario_activity_message, - write_namespace_completion, -) -from sqlbuild.cli.commands.models import ScenarioRunOutputContext, ScenarioRunPresentation -from sqlbuild.executor.scenario.models import ScenarioRunResult -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter - - -def begin_scenario_run( - *, context: ScenarioRunOutputContext, scenario_count: int -) -> ScenarioRunPresentation: - """Write the scenario section header and start the run clock.""" - - style: CliStyle = CliStyle(use_color=context.use_color) - context.progress_stream.write( - f"\n{style.success_strong(f'Scenario ({scenario_count} selected)')}\n\n" - ) - status_is_tty: bool = ( - hasattr(context.progress_stream, "isatty") and context.progress_stream.isatty() - ) - activity: str = scenario_activity_message(activity="Running scenarios...", context=context) - if not status_is_tty: - context.progress_stream.write(f"{activity}\n\n") - context.progress_stream.flush() - return ScenarioRunPresentation( - scenario_status=TransientStatusReporter( - stream=context.progress_stream, use_color=context.use_color - ), - status_is_tty=status_is_tty, - activity=activity, - started=time.monotonic(), - ) - - -def finish_scenario_run( - *, - context: ScenarioRunOutputContext, - presentation: ScenarioRunPresentation, - results: tuple[ScenarioRunResult, ...], - counts: tuple[tuple[str, int], ...], - succeeded: bool, - local: bool, -) -> None: - """Write the timed summary, the namespace completion line, and optional JSON.""" - - presentation.scenario_status.close() - elapsed_seconds: float = time.monotonic() - presentation.started - context.progress_stream.write( - "\n" - + format_summary_footer( - counts=counts, use_color=context.use_color, elapsed=f"{elapsed_seconds:.2f}s" - ) - + "\n" - ) - context.progress_stream.flush() - write_namespace_completion(context=context, succeeded=succeeded) - write_execution_json_output( - payload=format_scenario_execution_json( - results=results, - local=local, - run_namespace=context.namespace.value, - namespace_source=context.namespace.source, - duration_ms=elapsed_seconds * 1000, - ), - json_output=context.json_output, - json_output_path=context.json_output_path, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_output/scenario_runtime.py b/src/sqlbuild/cli/commands/_helpers/scenario_output/scenario_runtime.py deleted file mode 100644 index 629ac637c2..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/scenario_output/scenario_runtime.py +++ /dev/null @@ -1,326 +0,0 @@ -"""Write scenario runtime SQL artifacts to target/run.""" - -from __future__ import annotations - -import shutil -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import FunctionPlanEntry, ScenarioExecutionPlan -from sqlbuild.executor.scenario.main.expected_comparison_sql import ( - build_scenario_expected_comparison_sql, -) -from sqlbuild.executor.scenario.models import ( - ScenarioCleanupExecutionResult, - ScenarioLocalSnapshotLoadedRelation, - ScenarioRunResult, -) - -_RUN_DIR: str = "run" -_SCENARIOS_DIR: str = "scenarios" -_FIXTURES_DIR: str = "fixtures" -_FUNCTIONS_DIR: str = "functions" -_SEEDS_DIR: str = "seeds" -_MODELS_DIR: str = "models" -_EXPECTATIONS_DIR: str = "expectations" -_CLEANUP_DIR: str = "cleanup" -_LOCAL_DIR: str = "local" -_SQL_FILE_SUFFIX: str = ".sql" - - -def write_scenario_runtime_target( - *, - target_dir: Path, - adapter: BaseAdapter, - scenario_plan: ScenarioExecutionPlan, - result: ScenarioRunResult, -) -> None: - """Write executed scenario lifecycle SQL under target/run/scenarios.""" - - scenario_run_dir: Path = target_dir / _RUN_DIR / _SCENARIOS_DIR / scenario_plan.name - if scenario_run_dir.exists(): - shutil.rmtree(scenario_run_dir) - - _write_cleanup_result( - scenario_run_dir=scenario_run_dir, - file_name="prepare.sql", - cleanup_result=result.prepare_cleanup_result, - ) - - for fixture_result in result.fixture_results: - sql: str = _sql_events_text(fixture_result.lifecycle_events) - if not sql: - continue - fixture_path: Path = ( - scenario_run_dir - / _FIXTURES_DIR - / f"{fixture_result.kind.value}__{_safe_file_stem(fixture_result.logical_name)}.sql" - ) - _write_sql(path=fixture_path, sql=sql) - - for seed_result in result.seed_results: - sql = _sql_events_text(seed_result.lifecycle_events) - if not sql: - continue - seed_path: Path = scenario_run_dir / _SEEDS_DIR / f"{seed_result.seed_name}.sql" - _write_sql(path=seed_path, sql=sql) - - for model_result in result.model_results: - sql = _sql_events_text(model_result.lifecycle_events) - if not sql: - continue - relative_path: Path | None = _model_relative_path( - scenario_plan=scenario_plan, - model_name=model_result.model_name, - ) - if relative_path is None: - continue - model_path: Path = scenario_run_dir / _model_output_path(relative_path) - _write_sql(path=model_path, sql=sql) - - for expected_expectation in scenario_plan.expected_expectations: - expected_path: Path = ( - scenario_run_dir - / _EXPECTATIONS_DIR - / f"expected__{expected_expectation.model_name}.sql" - ) - actual_relation: str = resolve_relation_location_qualified_name( - adapter=adapter, - location=expected_expectation.actual_destination, - ) - _write_sql( - path=expected_path, - sql=build_scenario_expected_comparison_sql( - actual_sql=f"SELECT * FROM {actual_relation}", - expected_sql=expected_expectation.expected_sql, - set_difference_operator=adapter.render_set_difference_operator(), - ), - ) - - for assertion_expectation in scenario_plan.assertion_expectations: - assertion_path: Path = ( - scenario_run_dir / _EXPECTATIONS_DIR / f"assertion__{assertion_expectation.name}.sql" - ) - _write_sql( - path=assertion_path, - sql=( - "SELECT COUNT(*) FROM " - f"({assertion_expectation.sql}) AS __scenario_assertion_failures;\n\n" - "SELECT * FROM " - f"({assertion_expectation.sql}) AS __scenario_assertion_failures\n" - "LIMIT 10;" - ), - ) - - _write_cleanup_result( - scenario_run_dir=scenario_run_dir, - file_name="final.sql", - cleanup_result=result.cleanup_result, - ) - - -def write_local_scenario_runtime_target( - *, - target_dir: Path, - adapter: BaseAdapter, - scenario_plan: ScenarioExecutionPlan, - result: ScenarioRunResult, -) -> None: - """Write local scenario replay SQL/debug artifacts under target/run/scenarios.""" - - local_plan: ScenarioExecutionPlan = result.local_execution_plan or scenario_plan - scenario_run_dir: Path = target_dir / _RUN_DIR / _SCENARIOS_DIR / scenario_plan.name - local_run_dir: Path = scenario_run_dir / _LOCAL_DIR - if local_run_dir.exists(): - shutil.rmtree(local_run_dir) - - _write_local_fixture_artifacts( - local_run_dir=local_run_dir, - loaded_relations=result.local_snapshot_relations, - ) - _write_local_function_artifacts( - local_run_dir=local_run_dir, - adapter=adapter, - scenario_plan=local_plan, - result=result, - ) - _write_local_model_artifacts(local_run_dir=local_run_dir, scenario_plan=local_plan) - _write_local_expectation_artifacts( - local_run_dir=local_run_dir, - adapter=adapter, - scenario_plan=local_plan, - ) - - -def _write_cleanup_result( - *, - scenario_run_dir: Path, - file_name: str, - cleanup_result: ScenarioCleanupExecutionResult | None, -) -> None: - if cleanup_result is None: - return - sql: str = _sql_events_text(cleanup_result.lifecycle_events) - if not sql: - return - _write_sql(path=scenario_run_dir / _CLEANUP_DIR / file_name, sql=sql) - - -def _write_local_fixture_artifacts( - *, local_run_dir: Path, loaded_relations: tuple[ScenarioLocalSnapshotLoadedRelation, ...] -) -> None: - loaded_relation: ScenarioLocalSnapshotLoadedRelation - for loaded_relation in loaded_relations: - fixture_path: Path = ( - local_run_dir - / _FIXTURES_DIR - / f"{loaded_relation.kind.value}__{_safe_file_stem(loaded_relation.logical_name)}.sql" - ) - _write_sql( - path=fixture_path, - sql=( - f"-- loaded from {loaded_relation.file_path.as_posix()}\n" - f"-- rows: {loaded_relation.row_count}\n" - f'CREATE TABLE "{loaded_relation.table_name}" AS SELECT * FROM ' - f"'{loaded_relation.file_path.as_posix()}';" - ), - ) - - -def _write_local_function_artifacts( - *, - local_run_dir: Path, - adapter: BaseAdapter, - scenario_plan: ScenarioExecutionPlan, - result: ScenarioRunResult, -) -> None: - executed_function_names: frozenset[str] = frozenset( - function_result.function_name for function_result in result.function_results - ) - function_entry: FunctionPlanEntry - for function_entry in scenario_plan.function_entries: - if function_entry.name not in executed_function_names: - continue - function_path: Path = local_run_dir / _function_output_path( - relative_path=function_entry.relative_path, - language=function_entry.language, - ) - _write_sql( - path=function_path, - sql="\n\n".join( - _format_statement(statement) - for statement in adapter.render_create_function( - destination=function_entry.destination.qualified_name - or function_entry.destination.name, - arguments=function_entry.arguments, - returns=function_entry.returns, - body_sql=function_entry.body_sql, - return_columns=function_entry.return_columns, - language=function_entry.language, - runtime_version=function_entry.runtime_version, - entry_point=function_entry.entry_point, - packages=function_entry.packages, - ) - ), - ) - - -def _write_local_model_artifacts( - *, local_run_dir: Path, scenario_plan: ScenarioExecutionPlan -) -> None: - for entry in scenario_plan.model_entries: - model_path: Path = local_run_dir / _model_output_path(entry.relative_path) - _write_sql(path=model_path, sql=entry.resolved_sql) - - -def _write_local_expectation_artifacts( - *, local_run_dir: Path, adapter: BaseAdapter, scenario_plan: ScenarioExecutionPlan -) -> None: - for expected_expectation in scenario_plan.expected_expectations: - expected_path: Path = ( - local_run_dir / _EXPECTATIONS_DIR / f"expected__{expected_expectation.model_name}.sql" - ) - actual_relation: str = resolve_relation_location_qualified_name( - adapter=adapter, - location=expected_expectation.actual_destination, - ) - _write_sql( - path=expected_path, - sql=build_scenario_expected_comparison_sql( - actual_sql=f"SELECT * FROM {actual_relation}", - expected_sql=expected_expectation.expected_sql, - set_difference_operator=adapter.render_set_difference_operator(), - ), - ) - for assertion_expectation in scenario_plan.assertion_expectations: - assertion_path: Path = ( - local_run_dir / _EXPECTATIONS_DIR / f"assertion__{assertion_expectation.name}.sql" - ) - _write_sql( - path=assertion_path, - sql=( - "SELECT COUNT(*) FROM " - f"({assertion_expectation.sql}) AS __scenario_assertion_failures;\n\n" - "SELECT * FROM " - f"({assertion_expectation.sql}) AS __scenario_assertion_failures\n" - "LIMIT 10;" - ), - ) - - -def _sql_events_text(events: tuple[LifeCycleEvent, ...]) -> str: - sql_events: tuple[LifeCycleEvent, ...] = tuple( - event for event in events if event.kind == LifeCycleEventKind.SQL - ) - return "\n\n".join(_format_statement(event.content) for event in sql_events) - - -def _model_relative_path(*, scenario_plan: ScenarioExecutionPlan, model_name: str) -> Path | None: - for entry in scenario_plan.model_entries: - if entry.name == model_name: - return entry.relative_path - return None - - -def _model_output_path(relative_path: Path) -> Path: - parts: tuple[str, ...] = relative_path.parts - if parts and parts[0] == _MODELS_DIR: - return Path(*parts) - return Path(_MODELS_DIR) / relative_path - - -def _function_output_path(*, relative_path: Path, language: FunctionLanguage) -> Path: - parts: tuple[str, ...] = relative_path.parts - language_dir: str = language.value - function_language_path_part_count: int = 2 - if ( - len(parts) >= function_language_path_part_count - and parts[0] == _FUNCTIONS_DIR - and parts[1] == language_dir - ): - return Path(*parts).with_suffix(_SQL_FILE_SUFFIX) - return (Path(_FUNCTIONS_DIR) / language_dir / relative_path).with_suffix(_SQL_FILE_SUFFIX) - - -def _safe_file_stem(name: str) -> str: - return name.replace("/", "__").replace("\\", "__") - - -def _write_sql(*, path: Path, sql: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(sql.rstrip() + "\n", encoding="utf-8") - - -def _format_statement(statement: str) -> str: - stripped: str = statement.rstrip() - if not stripped: - return statement - if stripped.endswith(";"): - return stripped - return f"{stripped};" diff --git a/src/sqlbuild/cli/commands/classes/direct_python_lifecycle_state.py b/src/sqlbuild/cli/commands/classes/direct_python_lifecycle_state.py deleted file mode 100644 index dfc405b320..0000000000 --- a/src/sqlbuild/cli/commands/classes/direct_python_lifecycle_state.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Mutable direct Python lifecycle state.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any, TextIO, cast - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.build_python_nodes.python_lifecycle_selection import ( - load_result_key_or_none, - python_node_result_names, - sql_loader_functions_for_lifecycle_handoff, -) -from sqlbuild.cli.commands._helpers.build_python_nodes.python_node_output import ( - write_python_node_results, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction, DiscoveredProjectInputs -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.types import PythonNodeStatus -from sqlbuild.executor.build.types import ExecutionStatus -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult - - -class DirectPythonLifecycleState: - """Mutable state for direct-mode Python lifecycle execution.""" - - def __init__(self, **kwargs: object) -> None: - self.plan_output = cast(PlanOutput, kwargs["plan_output"]) - self.discovered_inputs = cast(DiscoveredProjectInputs, kwargs["discovered_inputs"]) - self.adapter = cast(BaseAdapter, kwargs["adapter"]) - self.progress_stream = cast(TextIO, kwargs["progress_stream"]) - self.use_color = cast(bool, kwargs["use_color"]) - self.base_on_node_complete = cast(Callable[[object], None], kwargs["base_on_node_complete"]) - self.read_side_connection = kwargs["read_side_connection"] - self.read_side_tracker = cast(Any | None, kwargs["read_side_tracker"]) - self.ingress_python_results = cast( - tuple[PythonNodeExecutionResult, ...], kwargs["ingress_python_results"] - ) - self.ingress_load_results = cast( - tuple[LoadExecutionResult, ...], kwargs["ingress_load_results"] - ) - self.ingress_loader_names = cast(frozenset[str], kwargs["ingress_loader_names"]) - - @property - def ingress_failed(self) -> bool: - return any( - result.status == PythonNodeStatus.FAILED for result in self.ingress_python_results - ) or any(result.status == ExecutionStatus.FAILED for result in self.ingress_load_results) - - @property - def python_results(self) -> tuple[PythonNodeExecutionResult, ...]: - read_side_results: tuple[PythonNodeExecutionResult, ...] = ( - () if self.read_side_tracker is None else self.read_side_tracker.results - ) - return (*self.ingress_python_results, *read_side_results) - - @property - def loader_functions(self) -> tuple[DiscoveredLoaderFunction, ...]: - return sql_loader_functions_for_lifecycle_handoff( - discovered_inputs=self.discovered_inputs, - ingress_loader_names=self.ingress_loader_names, - ) - - @property - def precompleted_keys(self) -> frozenset[CompiledObjectKey]: - return frozenset( - key - for load_result in self.ingress_load_results - if (key := load_result_key_or_none(plan=self.plan_output, result=load_result)) - is not None - ) - - @property - def blocked_keys(self) -> frozenset[CompiledObjectKey]: - return frozenset( - key - for load_result in self.ingress_load_results - if load_result.status != ExecutionStatus.SUCCESS - if (key := load_result_key_or_none(plan=self.plan_output, result=load_result)) - is not None - ) - - def on_node_complete(self, node_result: object) -> None: - self.base_on_node_complete(node_result) - if self.read_side_tracker is None: - return - previous_names: frozenset[str] = python_node_result_names(self.read_side_tracker.results) - self.read_side_tracker.record_sql_result(node_result) - new_results: tuple[PythonNodeExecutionResult, ...] = tuple( - result - for result in self.read_side_tracker.results - if result.node_name not in previous_names - ) - if new_results: - for result in new_results: - self.base_on_node_complete(result) - write_python_node_results( - stream=self.progress_stream, - results=new_results, - use_color=self.use_color, - ) - - def finalize(self) -> None: - if self.read_side_connection is not None: - self.adapter.close(self.read_side_connection) - self.read_side_connection = None - if self.read_side_tracker is None: - return - finalized_results: tuple[PythonNodeExecutionResult, ...] = ( - self.read_side_tracker.finalize_unrun_python_nodes() - ) - if finalized_results: - for result in finalized_results: - self.base_on_node_complete(result) - write_python_node_results( - stream=self.progress_stream, - results=finalized_results, - use_color=self.use_color, - ) diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index 03e718d228..bf49cd9353 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -26,33 +26,12 @@ SQL_FILE_SUFFIX: str = ".sql" SUCCESS_STATUS: str = "success" FAILED_STATUS: str = "failed" -DEFAULT_MAX_SNAPSHOT_ROWS_PER_RELATION: int = 10_000 -DEFAULT_MAX_SNAPSHOT_TOTAL_ROWS: int = 50_000 -DEFAULT_MAX_SNAPSHOT_BYTES_PER_RELATION: int = 5_000_000 -DEFAULT_MAX_SNAPSHOT_TOTAL_BYTES: int = 25_000_000 SCENARIO_CLI_MISSING_SUBCOMMAND: str = "C450" -SCENARIO_CLI_NONE_DISCOVERED: str = "C451" -SCENARIO_CLI_LOCAL_RETAIN_UNSUPPORTED: str = "C452" -SCENARIO_CLI_UNKNOWN_SELECTOR: str = "C453" -SCENARIO_CLI_LOCAL_SNAPSHOT_FLAG_REQUIRED: str = "C454" -SCENARIO_CLI_SQL_VALIDATION_REQUIRED: str = "C455" -SCENARIO_CLI_CAPTURE_DIALECT_REQUIRED: str = "C456" -SCENARIO_CLI_UNSUPPORTED_GRAPH_SELECTOR: str = "C457" -SCENARIO_CLI_INVALID_NAMESPACE: str = "C458" SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" -GRAPH_SELECTOR_EXPANSION_MARKER: str = "+" -GRAPH_SELECTOR_PATH_MARKER: str = "~" DBT_JSON_OUTPUT_OPTION: str = "--json" DBT_VERBOSE_OPTIONS: frozenset[str] = frozenset({"--verbose", "-v"}) DBT_CLI_OUTPUT_OPTIONS: frozenset[str] = frozenset({DBT_JSON_OUTPUT_OPTION, *DBT_VERBOSE_OPTIONS}) DBT_NO_CONNECTION_OPTION: str = "--no-connection" -COST_HISTORY_SELECTOR: str = "history" -COST_LATEST_SELECTOR: str = "latest" -COST_DESCENDING_ORDER: str = "desc" -COST_DEFAULT_HISTORY_SORT: str = "completed" -COST_DEFAULT_DETAIL_SORT: str = "cost" -COST_DEFAULT_HISTORY_LIMIT: int = 10 -ISO_DATE_LENGTH: int = 10 COST_NARROW_TERMINAL_WIDTH: int = 120 COST_HISTORY_NUMERIC_COLUMNS: frozenset[int] = frozenset({4, 5}) COST_RESOURCE_NUMERIC_START_COLUMN: int = 2 diff --git a/src/sqlbuild/cli/commands/main/execution/_audit.py b/src/sqlbuild/cli/commands/main/execution/_audit.py deleted file mode 100644 index de3e0401ac..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_audit.py +++ /dev/null @@ -1,60 +0,0 @@ -"""CLI audit command entry point.""" - -from __future__ import annotations - -from sqlbuild._native import write_execution_header -from sqlbuild.cli.commands._helpers.audit.execution import ( - execute_audit_plan, - prepare_audit_execution, - write_audit_plan_header, -) -from sqlbuild.cli.commands._helpers.audit.invocation import resolve_audit_invocation -from sqlbuild.cli.commands._helpers.audit.outputs import ( - resolve_audit_exit_code, - write_audit_completion_output, -) -from sqlbuild.cli.commands._helpers.audit.planning import compile_audit_plan -from sqlbuild.cli.commands.models import ( - AuditCommandRequest, - AuditExecutionPreparation, - AuditInvocation, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.executor.auditing.models import AuditExecutionResult - - -def run_audit(request: AuditCommandRequest) -> int: - """Execute the audit command.""" - - invocation: AuditInvocation = resolve_audit_invocation(request=request) - pipeline_result: CompilePipelineResult = compile_audit_plan( - request=request, - invocation=invocation, - ) - preparation: AuditExecutionPreparation = prepare_audit_execution( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - invocation.progress_stream.write("\n") - write_execution_header( - stream=invocation.progress_stream, - command="sqb audit", - target=None, - concurrency=preparation.effective_concurrency, - use_color=invocation.use_color, - ) - write_audit_plan_header(invocation=invocation, pipeline_result=pipeline_result) - results: tuple[AuditExecutionResult, ...] = execute_audit_plan( - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - ) - write_audit_completion_output( - request=request, - invocation=invocation, - results=results, - configured_concurrency=preparation.effective_concurrency, - worker_count=preparation.worker_count, - ) - return resolve_audit_exit_code(results) diff --git a/src/sqlbuild/cli/commands/main/execution/_build.py b/src/sqlbuild/cli/commands/main/execution/_build.py deleted file mode 100644 index 3a9f7b505a..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_build.py +++ /dev/null @@ -1,238 +0,0 @@ -"""CLI build command entry point.""" - -from __future__ import annotations - -import sys -import time -from datetime import UTC, datetime -from typing import Any - -from sqlbuild.cli.commands._helpers.build_execution.checks import run_post_build_python_checks -from sqlbuild.cli.commands._helpers.build_execution.completion import complete_direct_build -from sqlbuild.cli.commands._helpers.build_execution.execution import ( - execute_build_plan, - prepare_build_execution, -) -from sqlbuild.cli.commands._helpers.build_execution.outputs import ( - write_build_plan_text, -) -from sqlbuild.cli.commands._helpers.build_execution.phase_timings import ( - finalize_exceptional_with_timings, - finalize_no_work_with_timings, - write_partial_build_phase_timings, -) -from sqlbuild.cli.commands._helpers.build_planning.compile_target import write_build_compile_target -from sqlbuild.cli.commands._helpers.build_planning.defer_clone import ( - run_defer_clone_boundary_prephase, -) -from sqlbuild.cli.commands._helpers.build_planning.invocation import resolve_build_invocation -from sqlbuild.cli.commands._helpers.build_planning.plan_policies import ( - enforce_build_plan_policies, -) -from sqlbuild.cli.commands._helpers.build_planning.planning import compile_build_plan -from sqlbuild.cli.commands._helpers.cost.collection import ( - finalize_build_cost, -) -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildCostFinalization, - BuildExecutionPreparation, - BuildInvocation, - BuildRunOutcome, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.cost.types import CostStatus -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker -from sqlbuild.diagnostics.main.process_resource_reporting import process_resource_reporting -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult -from sqlbuild.provider.main.session import build_provider_session -from sqlbuild.runtime.execution_limits.main.execution_deadline_scope import ( - execution_deadline_scope, -) -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig - - -def run_build(request: BuildCommandRequest) -> int: - """Execute the build command.""" - - timing_tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker() - with process_resource_reporting(enabled=request.debug), timing_tracker.scope(): - try: - command_started_at: float = time.monotonic() - invocation: BuildInvocation = resolve_build_invocation(request=request) - limits: ExecutionLimitsConfig = invocation.execution_limits - with execution_deadline_scope( - max_duration_seconds=limits.max_duration_seconds, - max_duration=limits.max_duration, - target_name=invocation.effective_target_name, - remediation=limits.remediation, - ): - return _run_build( - request=request, - invocation=invocation, - command_started_at=command_started_at, - ) - except BaseException: - if request.verbose or request.debug: - try: - write_partial_build_phase_timings( - stream=sys.stderr, - timings=timing_tracker.snapshot(), - use_color=False, - ) - except BaseException: - pass - raise - - -def _run_build( - *, request: BuildCommandRequest, invocation: BuildInvocation, command_started_at: float -) -> int: - provider_session: Any = build_provider_session( - discovered_providers=invocation.discovered_inputs.providers - ) - try: - if invocation.effective_defer_clone_from is not None: - _ = run_defer_clone_boundary_prephase( - request=request, - invocation=invocation, - origin_target_name=invocation.effective_defer_clone_from, - ) - pipeline_result: CompilePipelineResult = compile_build_plan( - request=request, - invocation=invocation, - ) - write_build_plan_text( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - enforce_build_plan_policies( - request=request, - invocation=invocation, - plan=pipeline_result.plan_output, - ) - write_build_compile_target( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - ) - if finalize_no_work_with_timings( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - command_started_at=command_started_at, - ): - return 0 - build_started_at: datetime = datetime.now(UTC) - _ = finalize_build_cost( - BuildCostFinalization( - project_dir=invocation.effective_project_dir, - adapter_name=invocation.adapter_name, - adapter=invocation.adapter, - connection_config=invocation.connection_config, - target_name=pipeline_result.project.effective_target_name, - target_database=pipeline_result.project.effective_target_database, - run_id=pipeline_result.project.run_id, - build_status="running", - started_at=build_started_at, - completed_at=build_started_at, - config=invocation.discovered_inputs.project_config.cost, - output_stream=invocation.progress_stream, - use_color=invocation.use_color, - collect=False, - render=False, - cost_status=CostStatus.PENDING, - cost_message="Build cost collection has not completed.", - ) - ) - try: - preparation, outcome, check_results = _execute_direct_build( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - providers=provider_session.providers, - ) - except BaseException as error: - _ = finalize_exceptional_with_timings( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - build_started_at=build_started_at, - command_started_at=command_started_at, - error=error, - ) - raise - return complete_direct_build( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - outcome=outcome, - check_results=check_results, - build_started_at=build_started_at, - command_started_at=command_started_at, - ) - finally: - provider_session.close() - - -def _execute_direct_build( - *, - request: BuildCommandRequest, - invocation: BuildInvocation, - pipeline_result: CompilePipelineResult, - providers: Any, -) -> tuple[ - BuildExecutionPreparation, - BuildRunOutcome, - tuple[PythonCheckExecutionResult, ...], -]: - with CostContext.scope( - run_id=pipeline_result.project.run_id, - resource_type="run", - resource_name=(pipeline_result.project.effective_target_name or invocation.adapter_name), - ledger_path=( - invocation.effective_project_dir - / "target" - / "executions" - / pipeline_result.project.run_id - / "statements.jsonl" - ), - phase="build", - ): - preparation: BuildExecutionPreparation = prepare_build_execution( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - providers=providers, - ) - try: - outcome: BuildRunOutcome = execute_build_plan( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - preparation=preparation, - providers=providers, - ) - with CostContext.resource_scope( - resource_type="run", - resource_name=( - pipeline_result.project.effective_target_name or invocation.adapter_name - ), - phase="post_build_checks", - ): - check_results: tuple[PythonCheckExecutionResult, ...] = ( - run_post_build_python_checks( - request=request, - invocation=invocation, - pipeline_result=pipeline_result, - outcome=outcome, - providers=providers, - preparation=preparation, - ) - ) - finally: - preparation.callbacks.close() - return preparation, outcome, check_results diff --git a/src/sqlbuild/cli/commands/main/execution/_scenario.py b/src/sqlbuild/cli/commands/main/execution/_scenario.py deleted file mode 100644 index 951cc0013f..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_scenario.py +++ /dev/null @@ -1,14 +0,0 @@ -"""CLI scenario command entry point.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.scenario_execution.runner import ( - run_scenario as run_scenario_command, -) -from sqlbuild.cli.commands.models import ScenarioTestCommandRequest - - -def run_scenario(request: ScenarioTestCommandRequest) -> int: - """Run the scenario command.""" - - return run_scenario_command(request=request) diff --git a/src/sqlbuild/cli/commands/main/inspection/_contract.py b/src/sqlbuild/cli/commands/main/inspection/_contract.py deleted file mode 100644 index 0f5f0171ee..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_contract.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Contract comparison and repository adoption command.""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild._native import ( - build_compiled_project, - build_project_graph_from_compiled_project, - discover_project_inputs, -) -from sqlbuild.cli.commands._helpers.contract.progress import ( - inspect_contracts_with_progress, - write_contract_updates_with_progress, -) -from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( - resolve_adapter_connection_context, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import AdapterConnectionContext, ContractCommandRequest -from sqlbuild.compiler.compile.models import ( - CompileAnalysisSelection, - CompiledObjectKey, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledSourceView, -) -from sqlbuild.compiler.contract_adoption.models import ContractAdoptionResult, ContractEvidence -from sqlbuild.compiler.contract_adoption.types import ContractAction -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.project_graph_selection import select_project_graph -from sqlbuild.compiler.pipeline.main.reject_unit_test_selectors import reject_unit_test_selectors -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.presentation.main.supports_color import supports_color - - -def run_contract(request: ContractCommandRequest) -> int: - """Compare or adopt selected contracts from one read-only target.""" - - if request.overwrite and (request.action != ContractAction.GENERATE or not request.write): - raise CliUserError("contract --overwrite requires generate --write", code="C471") - project_dir: Path = request.project_dir or Path.cwd() - discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - _ = reject_unit_test_selectors( - select=request.select, exclude=request.exclude, discovered_inputs=discovered - ) - if request.from_target not in discovered.project_config.targets: - raise CliUserError( - f"unknown contract source target '{request.from_target}'", - code="C472", - ) - context: AdapterConnectionContext = resolve_adapter_connection_context( - discovered_inputs=discovered, - effective_project_dir=project_dir, - selected_target=None, - cli_vars=request.cli_vars, - ) - project: CompiledProjectView = build_compiled_project( - discovered_inputs=discovered, - adapter=context.adapter, - selected_target=request.from_target, - cli_vars=request.cli_vars, - resolved_connection=context.connection_config, - analysis_selection=CompileAnalysisSelection(no_cache=True), - ) - graph: ProjectGraph = build_project_graph_from_compiled_project(project=project) - selected: frozenset[CompiledObjectKey] = select_project_graph( - graph=graph, select=request.select, exclude=request.exclude - ) - selected_models: tuple[CompiledModelView, ...] = tuple( - model for model in graph.project.models if model.key in selected - ) - selected_sources: tuple[CompiledSourceView, ...] = tuple( - source for source in graph.project.sources if source.key in selected - ) - if not selected_models and not selected_sources: - raise CliUserError("contract selection contains no models or sources", code="C473") - - progress_stream: TextIO = sys.stderr if request.json_output else sys.stdout - use_progress_color: bool = not request.no_color and not request.json_output and supports_color() - evidence: tuple[ContractEvidence, ...] = inspect_contracts_with_progress( - context=context, - request=request, - selected_models=selected_models, - selected_sources=selected_sources, - progress_stream=progress_stream, - use_progress_color=use_progress_color, - ) - result: ContractAdoptionResult = ContractAdoptionResult( - from_target=request.from_target, evidence=evidence - ) - if request.action == ContractAction.GENERATE and request.write: - result = write_contract_updates_with_progress( - project_dir=project_dir, - graph=graph, - result=result, - request=request, - adapter=context.adapter, - resolved_connection=context.connection_config, - progress_stream=progress_stream, - use_progress_color=use_progress_color, - ) - _write_output(request=request, result=result) - return 1 if result.findings else 0 - - -def _write_output(*, request: ContractCommandRequest, result: ContractAdoptionResult) -> None: - if request.json_output: - print( - json.dumps( - _result_json_payload(result=result), - indent=2, - sort_keys=True, - ) - ) - return - print(f"Contract comparison from target '{result.from_target}'") - for item in result.evidence: - label: str = f"{item.resource_type.value}:{item.resource_name}" - if not item.findings: - print(f" OK {label}") - continue - print(f" DIFF {label}") - for finding in item.findings: - print(f" {finding.kind.value}: {finding.message}") - if result.written_paths: - print("\nUpdated repository declarations:") - for path in result.written_paths: - print(f" {path}") - print(f"\n{len(result.findings)} contract difference(s)") - - -def _result_json_payload(*, result: ContractAdoptionResult) -> dict[str, object]: - resources: list[dict[str, object]] = [] - for item in result.evidence: - findings: list[dict[str, object]] = [] - for finding in item.findings: - findings.append( - { - "kind": str(finding.kind), - "column": finding.column_name, - "declared_type": finding.declared_type, - "physical_type": finding.physical_type, - "message": finding.message, - } - ) - resources.append( - { - "resource_type": str(item.resource_type), - "resource_name": item.resource_name, - "relation": ".".join( - value for value in (item.database, item.schema, item.relation) if value - ), - "findings": findings, - } - ) - return { - "from_target": result.from_target, - "differences": len(result.findings), - "written_paths": [str(path) for path in result.written_paths], - "resources": resources, - } diff --git a/src/sqlbuild/cli/commands/main/inspection/_cost.py b/src/sqlbuild/cli/commands/main/inspection/_cost.py deleted file mode 100644 index ea5c7f7cac..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_cost.py +++ /dev/null @@ -1,12 +0,0 @@ -"""CLI run-cost history and drill-down entry point.""" - -from sqlbuild.cli.commands._helpers.cost.command import run_cost_command -from sqlbuild.cli.commands._helpers.cost.refresh import refresh_pending_cost_run -from sqlbuild.cli.commands.models import CostCommandRequest - - -def run_cost(request: CostCommandRequest) -> int: - """Render persisted cost history or one run breakdown.""" - - _ = refresh_pending_cost_run(request) - return run_cost_command(request) diff --git a/src/sqlbuild/cli/commands/main/inspection/_query.py b/src/sqlbuild/cli/commands/main/inspection/_query.py deleted file mode 100644 index ff44bbbeb3..0000000000 --- a/src/sqlbuild/cli/commands/main/inspection/_query.py +++ /dev/null @@ -1,108 +0,0 @@ -"""CLI query command entry point.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild._native import ( - discover_project_inputs, - resolve_project_connection_config, -) -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.adapter.contract.models import QueryResult -from sqlbuild.cli.commands._helpers.query.output import render_query_result -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands.exceptions import CliUserError, QueryExecutionError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) - - -def run_query( - *, - project_dir: Path | None, - sql: str | None, - query_file: Path | None = None, - selected_target: str | None = None, - output_format: str = "long", - limit: int | None = 20, -) -> int: - """Execute ad hoc SQL against the active project connection.""" - - query_sql: str = _resolve_query_sql(sql=sql, query_file=query_file) - if limit is not None and limit < 0: - raise CliUserError( - "query --limit must be greater than or equal to 0", - code="C101", - help="pass a non-negative integer or use --no-limit", - ) - - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - project_dir=effective_project_dir, - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - selected_target=selected_target, - ) - connection: object = adapter.connect(connection_config) - try: - result: QueryResult = adapter.query(connection=connection, sql=query_sql, limit=limit) - except AdapterUserError: - raise - except Exception as error: - raise QueryExecutionError(str(getattr(error, "message", error))) from error - finally: - adapter.close(connection) - - sys.stdout.write(render_query_result(result=result, output_format=output_format, limit=limit)) - sys.stdout.flush() - return 0 - - -def _resolve_query_sql(*, sql: str | None, query_file: Path | None) -> str: - if sql is not None and query_file is not None: - raise CliUserError( - "query accepts either positional SQL or --file, not both", - code="C104", - ) - if query_file is not None: - query_sql: str = _read_query_file(query_file=query_file) - if not query_sql.strip(): - raise CliUserError("query SQL must not be empty", code="C103") - return query_sql - if sql is None: - raise CliUserError( - "query requires SQL", - code="C102", - help="pass SQL as the query argument or use --file PATH", - ) - query_sql = sql - if not query_sql.strip(): - raise CliUserError("query SQL must not be empty", code="C103") - return query_sql - - -def _read_query_file(*, query_file: Path) -> str: - if not query_file.exists(): - raise CliUserError(f"query file does not exist: {query_file}", code="C105") - if not query_file.is_file(): - raise CliUserError(f"query file path is not a file: {query_file}", code="C106") - try: - return query_file.read_text(encoding="utf-8") - except (OSError, UnicodeError) as error: - raise CliUserError( - f"query file could not be read as UTF-8: {query_file}", - code="C107", - ) from error diff --git a/src/sqlbuild/cli/commands/main/project/_dag.py b/src/sqlbuild/cli/commands/main/project/_dag.py deleted file mode 100644 index 2502ffef2c..0000000000 --- a/src/sqlbuild/cli/commands/main/project/_dag.py +++ /dev/null @@ -1,71 +0,0 @@ -"""CLI dag command entry point.""" - -from __future__ import annotations - -import json -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import discover_project_inputs -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) - - -def run_dag( - *, - project_dir: Path | None, - no_sql_validation: bool = False, - json_output: bool = False, - cli_vars: dict[str, object] | None = None, -) -> int: - """Execute the dag command.""" - - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ), - project_dir=effective_project_dir, - ) - graph: ProjectGraph = _native.build_project_graph( - discovered_inputs=discovered_inputs, - adapter=adapter, - no_sql_validation=no_sql_validation, - cli_vars=cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - ), - ) - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=discovered_inputs - ) - dag_json: str = _native.build_dag_json( - graph=graph, - project_name=discovered_inputs.project_config.name, - python_graph=python_graph, - ) - if json_output: - print(dag_json) - else: - payload: dict[str, object] = json.loads(dag_json) - print( - "DAG ready " - f"({len(payload['nodes'])} nodes, {len(payload['edges'])} edges, " - f"{len(payload['checks'])} checks)" - ) - return 0 diff --git a/src/sqlbuild/cli/commands/main/project/_format.py b/src/sqlbuild/cli/commands/main/project/_format.py deleted file mode 100644 index cb4e5fb99a..0000000000 --- a/src/sqlbuild/cli/commands/main/project/_format.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Run the format command.""" - -from __future__ import annotations - -import difflib -import sys -import time -from dataclasses import replace -from pathlib import Path - -from sqlbuild._native import discover_project_inputs -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.lint.runs import ( - prepare_lint_run, - render_lint_result, - render_lint_result_json, -) -from sqlbuild.cli.commands._helpers.lint.selection import resolve_lint_inputs -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint.main.has_fixture_typed_null_candidates import ( - has_fixture_typed_null_candidates, -) -from sqlbuild.lint.main.run_format import run_format -from sqlbuild.lint.models import LintConfig, LintRunResult -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.supports_color import supports_color - - -def run_format_command( - *, - project_dir: Path | None, - select: tuple[str, ...] = (), - exclude: tuple[str, ...] = (), - check: bool = False, - diff: bool = False, - fixtures_only: bool = False, - fix: bool = False, - json_output: bool = False, - no_color: bool = False, -) -> int: - """Apply autofixes in place; non-zero when faults remain after formatting.""" - - base_dir: Path = project_dir if project_dir is not None else Path.cwd() - prepared: tuple[LintConfig, str | None] = prepare_lint_run(project_dir=base_dir) - config: LintConfig = prepared[0] - value_renderer: BaseAdapter | None = None - selected_paths: frozenset[Path] | None = None - discovered_inputs: DiscoveredProjectInputs | None = None - has_fixture_candidates: bool = False - if not select and not exclude: - has_fixture_candidates = has_fixture_typed_null_candidates(project_dir=base_dir) - if select or exclude or has_fixture_candidates or fix: - try: - value_renderer, selected_paths, discovered_inputs = resolve_lint_inputs( - project_dir=base_dir, - select=select, - exclude=exclude, - ) - except Exception: - if select or exclude or fix: - raise - else: - has_fixture_candidates = bool(discovered_inputs.test_files) - config = replace( - config, - dialect=value_renderer.sql_analysis_dialect_name or "generic", - ) - fixture_inputs: DiscoveredProjectInputs | None = None - if fix: - discovered_inputs = discover_project_inputs(project_dir=base_dir) - if discovered_inputs is not None and has_fixture_candidates: - local_overrides: frozenset[str] = discovered_inputs.local_config.setting_overrides - sql_analysis_enabled: bool = ( - discovered_inputs.local_config.settings.sql_analysis - if SQL_ANALYSIS_SETTING_KEY in local_overrides - else discovered_inputs.project_config.settings.sql_analysis - ) - if sql_analysis_enabled: - fixture_inputs = discovered_inputs - format_started_at: float = time.monotonic() - format_status: TransientStatusReporter = TransientStatusReporter( - stream=sys.stderr, - use_color=not no_color and supports_color(), - ) - format_status.start("Formatting SQL START") - try: - result: LintRunResult = run_format( - project_dir=base_dir, - config=config, - value_renderer=value_renderer, - selected_paths=selected_paths, - discovered_inputs=discovered_inputs if fix else fixture_inputs, - fixtures_only=fixtures_only, - write=not (check or diff), - fix=fix, - ) - except Exception: - format_status.error("Formatting SQL ERROR") - raise - elapsed_seconds: float = time.monotonic() - format_started_at - _ = _complete_format_status(status=format_status, result=result, elapsed=elapsed_seconds) - if diff: - _ = _render_format_diff(result=result) - if json_output: - _ = render_lint_result_json(result=result) - else: - _ = render_lint_result( - result=result, - root=base_dir, - use_color=not no_color and supports_color(), - show_formatted=True, - formatted_heading="Would format files:" if check or diff else "Formatted files:", - ) - return 1 if result.faults or (check and result.formatted_files) else 0 - - -def _complete_format_status( - *, status: TransientStatusReporter, result: LintRunResult, elapsed: float -) -> None: - if result.faults: - status.error(f"Formatting SQL ERROR ({elapsed:.2f}s; {len(result.faults)} faults)") - else: - status.complete( - message=( - f"Formatting SQL OK ({elapsed:.2f}s; " - f"{result.files_checked} files checked, {len(result.formatted_files)} changed)" - ) - ) - - -def _render_format_diff(*, result: LintRunResult) -> None: - """Print stable unified diffs for files the formatter would change.""" - - for change in result.format_changes: - print( - "".join( - difflib.unified_diff( - change.before.splitlines(keepends=True), - change.after.splitlines(keepends=True), - fromfile=str(change.file_path), - tofile=str(change.file_path), - ) - ), - end="", - ) diff --git a/src/sqlbuild/cli/commands/main/workspace/_init.py b/src/sqlbuild/cli/commands/main/workspace/_init.py deleted file mode 100644 index 3ad4cb9877..0000000000 --- a/src/sqlbuild/cli/commands/main/workspace/_init.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Scaffold a new blank SQLBuild project.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import CliDocument, update_sqlbuild_skills -from sqlbuild.cli.commands._helpers.init.scaffold import scaffold_blank_project -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.supports_color import supports_color - - -def run_init(project_dir: Path | None) -> int: - """Create a minimal SQLBuild project in the current directory.""" - - base_dir: Path = project_dir if project_dir is not None else Path.cwd() - project_file: Path = base_dir / "sqlbuild_project.toml" - - if project_file.exists(): - from sqlbuild.cli.commands.exceptions import CliUserError - - raise CliUserError( - "sqlbuild_project.toml already exists in this directory", - code="C900", - ) - - project_name: str = base_dir.name.replace("-", "_").replace(" ", "_").lower() - - _ = scaffold_blank_project(base_dir=base_dir, project_name=project_name) - - _ = update_sqlbuild_skills(project_dir=base_dir) - - use_color: bool = supports_color() - style: CliStyle = CliStyle(use_color=use_color) - doc: CliDocument = CliDocument(style) - doc.header(text="SQLBuild project created") - doc.blank() - doc.field(label="Project", value=project_name) - doc.field(label="Config", value="sqlbuild_project.toml", value_padding=" ") - doc.blank() - doc.section("Next steps") - doc.line(" 1. Add sources to sources/") - doc.line(" 2. Add seeds to seeds/") - doc.line(" 3. Add Python loaders, tasks, assets, checks, and factories to python/") - doc.line( - " 4. Add SQL hooks to hooks/sql/, Python hooks to hooks/python/, " - "functions to functions/, or macros to macros/" - ) - doc.line(" 5. Add models to models/staging/ and models/marts/") - doc.line( - " 6. Add tests to tests/unit/ or tests/scenarios/, reusable audits to " - "audits/generic/, or standalone audits to audits/singular/" - ) - doc.command_line(prefix=" 7. ", command="sqb compile") - doc.command_line(prefix=" 8. ", command="sqb build") - print(doc.render(), end="") - return 0 diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 3429c98306..06a5233463 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -44,29 +44,15 @@ PlanningProgressReporter, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter - from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks - from sqlbuild.cli.commands.classes.direct_python_lifecycle_state import ( - DirectPythonLifecycleState, - ) - from sqlbuild.cli.progress.classes.audit_progress_reporter import AuditProgressReporter from sqlbuild.compiler.compile.models import ( - CompiledSqlScenario, CompilerDiagnostic, ) from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - from sqlbuild.compiler.pipeline.models import ( - CompilePipelineResult, - PythonPlanEntry, - ) from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.refactoring.models import RefactorProject - from sqlbuild.executor.build.models import BuildExecutionResult from sqlbuild.executor.janitor.models import JanitorPlan - from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.integrations.dbt.models import DbtInitRequest - from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter - from sqlbuild.runtime.contracts.types import NodeStartCallback @dataclass(frozen=True) @@ -101,31 +87,6 @@ class AuditInvocation: planning_progress: PlanningProgressReporter -@dataclass(frozen=True) -class AuditExecutionPreparation: - """Prepared nested progress and execution reporters for audit runs.""" - - progress: AuditProgressReporter - execution_connection_progress: ConnectionProgressReporter - effective_concurrency: int - worker_count: int - - -@dataclass(frozen=True) -class BuildRunContext: - """Resolved inputs needed to render one build execution context.""" - - command: str - project: CompiledProjectView - plan: PlanOutput - discovered_inputs: DiscoveredProjectInputs - python_plan_entries: tuple[PythonPlanEntry, ...] - connection_config: dict[str, object] - concurrency: int - full_refresh: bool - selector_files: tuple[SelectorFileSummary, ...] - - @dataclass(frozen=True) class BuildCommandRequest: """CLI inputs for one build command invocation.""" @@ -166,16 +127,6 @@ class BuildCommandRequest: selector_files: tuple[SelectorFileSummary, ...] = () -@dataclass(frozen=True) -class DirectLifecycleCallbacks: - """Node progress callbacks and output settings for direct Python lifecycle.""" - - on_node_complete: Callable[[object], None] - progress_stream: TextIO - use_color: bool - on_node_start: NodeStartCallback | None = None - - @dataclass(frozen=True) class BuildInvocation: """Resolved project, adapter, and reporter context for the build command.""" @@ -230,41 +181,6 @@ class DeferClonePrephaseOutputContext: use_color: bool = False -@dataclass(frozen=True) -class BuildExecutionPreparation: - """Prepared callbacks, concurrency, cursors, and python lifecycle for execution.""" - - callbacks: BuildProgressCallbacks - effective_concurrency: int - execution_connection_progress: ConnectionProgressReporter - python_lifecycle: DirectPythonLifecycleState - start_cursor_ts: datetime | None - end_cursor_ts: datetime | None - start_cursor_int: int | None - end_cursor_int: int | None - - -@dataclass(frozen=True) -class BuildRunOutcome: - """Build pipeline execution result and finalized python node results.""" - - result: BuildExecutionResult - python_results: tuple[PythonNodeExecutionResult, ...] - - -@dataclass(frozen=True) -class BuildPhaseTimings: - """Final monotonic phase durations for verbose build output.""" - - compile_seconds: float | None = None - planning_seconds: float | None = None - connection_preparation_seconds: float | None = None - schema_preparation_seconds: float | None = None - execution_seconds: float | None = None - cost_collection_seconds: float | None = None - total_seconds: float | None = None - - @dataclass(frozen=True) class CheckCommandRequest: """CLI inputs for one check command invocation.""" @@ -578,35 +494,6 @@ class AdapterConnectionContext: connection_config: dict[str, object] -@dataclass(frozen=True) -class ScenarioRunNamespace: - """Effective artifact namespace and its configuration source.""" - - value: str | None = None - source: str = "unset" - - -@dataclass(frozen=True) -class ScenarioRunOutputContext: - """Progress stream and JSON output settings for one scenario CLI run.""" - - progress_stream: TextIO - use_color: bool - namespace: ScenarioRunNamespace = ScenarioRunNamespace() - json_output: bool = False - json_output_path: Path | None = None - - -@dataclass(frozen=True) -class ScenarioRunPresentation: - """Live status and timing shared by one scenario test run's progress output.""" - - scenario_status: TransientStatusReporter - status_is_tty: bool - activity: str - started: float - - @dataclass(frozen=True) class ScenarioSnapshotLimitInputs: """CLI snapshot capture-limit overrides for one scenario run.""" @@ -653,24 +540,6 @@ class ScenarioCaptureCommandRequest: limit_inputs: ScenarioSnapshotLimitInputs = ScenarioSnapshotLimitInputs() -@dataclass(frozen=True) -class LocalSnapshotSyncInputs: - """Resolved project, adapters, and scenarios for a local snapshot sync run.""" - - project_dir: Path - discovered_inputs: DiscoveredProjectInputs - local_pipeline_result: CompilePipelineResult - local_scenarios: tuple[CompiledSqlScenario, ...] - local_adapter: BaseAdapter - project_adapter: BaseAdapter - project_adapter_name: str - capture_dialect: str - project_connection_config: dict[str, object] - project_name: str - no_sql_validation: bool - refresh: bool - - @dataclass(frozen=True) class SeedCommandRequest: """CLI inputs for one seed command invocation.""" diff --git a/src/sqlbuild/compiler/compile/main/effective_target.py b/src/sqlbuild/compiler/compile/main/_effective_target.py similarity index 100% rename from src/sqlbuild/compiler/compile/main/effective_target.py rename to src/sqlbuild/compiler/compile/main/_effective_target.py diff --git a/src/sqlbuild/compiler/contract_adoption/main/write.py b/src/sqlbuild/compiler/contract_adoption/main/_write.py similarity index 100% rename from src/sqlbuild/compiler/contract_adoption/main/write.py rename to src/sqlbuild/compiler/contract_adoption/main/_write.py diff --git a/src/sqlbuild/compiler/discovery/main/sql_analysis_off_guidance.py b/src/sqlbuild/compiler/discovery/main/_sql_analysis_off_guidance.py similarity index 100% rename from src/sqlbuild/compiler/discovery/main/sql_analysis_off_guidance.py rename to src/sqlbuild/compiler/discovery/main/_sql_analysis_off_guidance.py diff --git a/src/sqlbuild/compiler/pipeline/main/static_command.py b/src/sqlbuild/compiler/pipeline/main/_static_command.py similarity index 100% rename from src/sqlbuild/compiler/pipeline/main/static_command.py rename to src/sqlbuild/compiler/pipeline/main/_static_command.py diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py b/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py index cefe5a92da..6560c5b9bd 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/reference_renames.py @@ -6,11 +6,9 @@ from datetime import UTC, datetime from itertools import islice, product -from sqlbuild._native import compute_query_hash +from sqlbuild._native import compute_query_hash, neutralize_lint_interpolation from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import ReferenceRename -from sqlbuild.lint.main.scan_interpolation_sites import scan_interpolation_sites -from sqlbuild.lint.models import InterpolationSite _REF_CALL: re.Pattern[str] = re.compile( r"^__ref\s*\(\s*(?P['\"])(?P[^'\"]+)(?P=quote)\s*\)$" @@ -61,14 +59,17 @@ def _ref_name_spans( *, query_sql: str, dialect: str, names: frozenset[str] ) -> tuple[_NameSpan, ...]: spans: list[_NameSpan] = [] - site: InterpolationSite - for site in scan_interpolation_sites(body=query_sql, dialect=dialect): - match: re.Match[str] | None = _REF_CALL.match(site.original_text) + original_start: int + original_text: str + for _, _, _, original_start, _, original_text in neutralize_lint_interpolation( + query_sql, dialect + )[1]: + match: re.Match[str] | None = _REF_CALL.match(original_text) if match is not None and match.group("name") in names: spans.append( ( - site.original_start + match.start("name"), - site.original_start + match.end("name"), + original_start + match.start("name"), + original_start + match.end("name"), match.group("name"), ) ) diff --git a/src/sqlbuild/compiler/planner/main/commands/audit.py b/src/sqlbuild/compiler/planner/main/commands/_audit.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/commands/audit.py rename to src/sqlbuild/compiler/planner/main/commands/_audit.py diff --git a/src/sqlbuild/compiler/planner/main/cursor_window/typed_cursor_overrides.py b/src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/cursor_window/typed_cursor_overrides.py rename to src/sqlbuild/compiler/planner/main/cursor_window/_typed_cursor_overrides.py diff --git a/src/sqlbuild/compiler/planner/main/selection/loader_dag.py b/src/sqlbuild/compiler/planner/main/selection/_loader_dag.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/selection/loader_dag.py rename to src/sqlbuild/compiler/planner/main/selection/_loader_dag.py diff --git a/src/sqlbuild/compiler/python_nodes/main/graph.py b/src/sqlbuild/compiler/python_nodes/main/_graph.py similarity index 100% rename from src/sqlbuild/compiler/python_nodes/main/graph.py rename to src/sqlbuild/compiler/python_nodes/main/_graph.py diff --git a/src/sqlbuild/compiler/python_nodes/main/run_lifecycle.py b/src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py similarity index 100% rename from src/sqlbuild/compiler/python_nodes/main/run_lifecycle.py rename to src/sqlbuild/compiler/python_nodes/main/_run_lifecycle.py diff --git a/src/sqlbuild/executor/auditing/main/current_audit_result_projection.py b/src/sqlbuild/executor/auditing/main/_current_audit_result_projection.py similarity index 100% rename from src/sqlbuild/executor/auditing/main/current_audit_result_projection.py rename to src/sqlbuild/executor/auditing/main/_current_audit_result_projection.py diff --git a/src/sqlbuild/executor/python_nodes/main/checks.py b/src/sqlbuild/executor/python_nodes/main/_checks.py similarity index 100% rename from src/sqlbuild/executor/python_nodes/main/checks.py rename to src/sqlbuild/executor/python_nodes/main/_checks.py diff --git a/src/sqlbuild/executor/python_nodes/main/ingress.py b/src/sqlbuild/executor/python_nodes/main/_ingress.py similarity index 100% rename from src/sqlbuild/executor/python_nodes/main/ingress.py rename to src/sqlbuild/executor/python_nodes/main/_ingress.py diff --git a/src/sqlbuild/lint/_helpers/expansion.py b/src/sqlbuild/lint/_helpers/expansion.py index 9a11a2c204..08429e283c 100644 --- a/src/sqlbuild/lint/_helpers/expansion.py +++ b/src/sqlbuild/lint/_helpers/expansion.py @@ -25,20 +25,14 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.lint._helpers.headers import lint_body_ranges, lint_file_role -from sqlbuild.lint._helpers.native_sql import check_lint_not_stopped -from sqlbuild.lint._helpers.sqlbuild_tokens import ( - neutralize_context_interpolation, - neutralize_generic_audit_parameters, - neutralize_interpolation, - sentinel_spans, -) +from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.constants import ( HEADER_KIND_SCENARIO, HEADER_KIND_TEST, RELATION_IDENTITY_TEMPLATE, TEMPLATE_INTERPOLATION_START, ) -from sqlbuild.lint.exceptions import ProjectCompileError +from sqlbuild.lint.exceptions import NativeLintError, ProjectCompileError from sqlbuild.lint.models import ( ExpandedLintFile, HeaderSpan, @@ -60,42 +54,6 @@ r"^__(ref|source|seed)\s*\(\s*['\"]([^'\"]+)['\"]\s*\)$", re.IGNORECASE ) -_CTE_DEFINITION_PATTERN: re.Pattern[str] = re.compile( - r'(?:\bWITH(?:\s+RECURSIVE)?|,)\s*["`\[]?(?P[A-Za-z_][A-Za-z0-9_]*)' - r'["`\]]?(?:\s*\([^)]*\))?\s+AS\s*\(', - re.IGNORECASE | re.DOTALL, -) -_OPAQUE_CTE_PREFIX_PATTERN: re.Pattern[str] = re.compile( - r"\b[A-Za-z_][A-Za-z0-9_]*\s+AS\s*\(\s*(?:--[^\n]*\n\s*)?$", - re.IGNORECASE, -) - - -def _externally_referenced_ctes( - *, - expanded: str, - interpolation_sites: tuple[InterpolationSite, ...], - pre_expansion_body: str, - pre_expansion_sites: tuple[InterpolationSite, ...], -) -> tuple[str, ...]: - names: set[str] = { - match.group("name").lower() for match in _CTE_DEFINITION_PATTERN.finditer(expanded) - } - referenced: set[str] = set() - for name in names: - for site in interpolation_sites: - if re.search(rf"\b{re.escape(name)}\b", site.original_text, re.IGNORECASE) is not None: - referenced.add(name) - break - for site in pre_expansion_sites: - prefix: str = pre_expansion_body[: site.neutralized_start] - if _OPAQUE_CTE_PREFIX_PATTERN.search(prefix) is None: - continue - referenced.update( - match.group("name").lower() for match in _CTE_DEFINITION_PATTERN.finditer(prefix) - ) - return tuple(sorted(referenced.intersection(names))) - def build_lint_expansion_context( *, @@ -202,11 +160,13 @@ def expand_lint_body( pre_expansion_sites: tuple[InterpolationSite, ...] = () expansion_input: str = authored_body if _is_generic_audit_path(role=role): - expansion_input, pre_expansion_sites = neutralize_generic_audit_parameters( - body=authored_body + expansion_input, pre_expansion_sites = _sites( + _native.neutralize_lint_audit_parameters(authored_body) ) elif _is_sql_hook_path(role=role): - expansion_input, pre_expansion_sites = neutralize_context_interpolation(body=authored_body) + expansion_input, pre_expansion_sites = _sites( + _native.neutralize_lint_context_interpolation(authored_body) + ) expanded: str expansion_passes: tuple[tuple[ExpansionSpan, ...], ...] try: @@ -273,23 +233,14 @@ def prepare_lint_sql_batch( def finish_lint_body( *, pending: PendingLintBody, prepared: NativePreparedSql | None, dialect: str ) -> LintBody: - """Build the lint body from one body's native preparation, or the Python fallback.""" + """Build the lint body from one body's native preparation.""" - expanded: str = pending.expanded - neutralized: str - sites: tuple[InterpolationSite, ...] + del dialect if prepared is None: - neutralized, sites = neutralize_interpolation(body=expanded, dialect=dialect) - externally_referenced_ctes: tuple[str, ...] = _externally_referenced_ctes( - expanded=expanded, - interpolation_sites=sites, - pre_expansion_body=pending.expansion_input, - pre_expansion_sites=pending.pre_expansion_sites, - ) - else: - neutralized = prepared[0] - sites = tuple(InterpolationSite(*site) for site in prepared[1]) - externally_referenced_ctes = tuple(prepared[2]) + raise NativeLintError(f"{pending.file_path} could not be prepared for native SQL lint") + neutralized: str = prepared[0] + sites: tuple[InterpolationSite, ...] = tuple(InterpolationSite(*site) for site in prepared[1]) + externally_referenced_ctes: tuple[str, ...] = tuple(prepared[2]) dependency_identifiers: tuple[str, ...] = tuple( site.sentinel for site in sites @@ -301,9 +252,9 @@ def finish_lint_body( body_end=pending.body_end, lint_text=neutralized, passes=( - sentinel_spans(sites=pending.pre_expansion_sites), + _sentinel_spans(sites=pending.pre_expansion_sites), *pending.expansion_passes, - sentinel_spans(sites=sites), + _sentinel_spans(sites=sites), ), external_identifiers=pending.external_identifiers, dependency_identifiers=dependency_identifiers, @@ -322,6 +273,26 @@ def finish_lint_body( ) +def _sites( + neutralized: tuple[str, list[tuple[str, int, int, int, int, str]]], +) -> tuple[str, tuple[InterpolationSite, ...]]: + return neutralized[0], tuple(InterpolationSite(*site) for site in neutralized[1]) + + +def _sentinel_spans(*, sites: tuple[InterpolationSite, ...]) -> tuple[ExpansionSpan, ...]: + """Express sentinel substitutions as expansion spans for offset mapping.""" + + return tuple( + ExpansionSpan( + source_start=site.original_start, + source_end=site.original_end, + output_start=site.neutralized_start, + output_end=site.neutralized_end, + ) + for site in sites + ) + + def _is_generic_audit_path(*, role: LintFileRole) -> bool: """Return whether a lint input is an authored generic-audit definition.""" @@ -348,7 +319,7 @@ def expand_file_bodies( ) -> ExpandedLintFile: """Expand one file's bodies up to the first body that cannot be expanded.""" - check_lint_not_stopped() + LintStopContext.check() bodies: list[PendingLintBody] = [] role: LintFileRole = lint_file_role( file_path=file_path, project_dir=project_dir, relative_path=relative_path @@ -401,7 +372,7 @@ def prepare_expanded_files( ) -> PreparedLintFiles: """Prepare every expanded body natively at once, then report failures in file order.""" - check_lint_not_stopped() + LintStopContext.check() pending: list[PendingLintBody] = [] for expanded in expanded_files: pending.extend(expanded.bodies) diff --git a/src/sqlbuild/lint/_helpers/fixes.py b/src/sqlbuild/lint/_helpers/fixes.py index 2921cfd340..39c8893de8 100644 --- a/src/sqlbuild/lint/_helpers/fixes.py +++ b/src/sqlbuild/lint/_helpers/fixes.py @@ -8,12 +8,11 @@ from dataclasses import dataclass, field, replace from pathlib import Path -from sqlbuild._native import build_compiled_project +from sqlbuild._native import build_compiled_project, run_lint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledProjectView, TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint._helpers.native_format import with_newline_style from sqlbuild.lint.constants import ( LINT_ENGINE_NATIVE, SAFE_RULE_FIX_CODES, @@ -24,11 +23,9 @@ from sqlbuild.lint.main.compile_facts import compile_facts from sqlbuild.lint.main.compiled_relation_keys import compiled_relation_keys from sqlbuild.lint.main.fix_verdict import fix_verdict -from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import ( CompileFacts, FixVerdict, - FormatChange, LintConfig, LintEdit, LintRunResult, @@ -295,36 +292,6 @@ def _decline( ) -def persist_changes( - *, files: dict[Path, str], updated: dict[Path, str], newlines: dict[Path, str], write: bool -) -> tuple[FormatChange, ...]: - changes: list[FormatChange] = [] - for path, contents in sorted(updated.items()): - after: str = with_newline_style(contents=contents, newline=newlines[path]) - before: str = with_newline_style(contents=files[path], newline=newlines[path]) - if after == before: - continue - changes.append(FormatChange(file_path=path, before=before, after=after)) - if write: - write_atomically(path=path, contents=after) - return tuple(changes) - - -def finalize_fix_reports( - *, reports: tuple[RuleFixResult, ...], declined_paths: set[Path] -) -> tuple[RuleFixResult, ...]: - return tuple( - replace( - report, - status="refused", - reason="Final layout formatting failed; original file retained", - ) - if report.status == _APPLIED and report.file_path in declined_paths - else report - for report in reports - ) - - def write_atomically(*, path: Path, contents: str) -> None: descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) temporary_path: Path = Path(temporary_name) diff --git a/src/sqlbuild/lint/_helpers/fixture_null_candidates.py b/src/sqlbuild/lint/_helpers/fixture_null_candidates.py deleted file mode 100644 index 35c6cd7f0f..0000000000 --- a/src/sqlbuild/lint/_helpers/fixture_null_candidates.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Cheap source scan for contract-aware fixture formatting candidates.""" - -from __future__ import annotations - -import re -from pathlib import Path - -from sqlbuild.lint.models import FixtureNullCandidateScan - -_FIXTURE_TYPED_NULL_PATTERN: re.Pattern[str] = re.compile( - r"(?:CAST\s*\(\s*NULL\s+AS\b|NULL\s*::)", re.IGNORECASE -) -_FIXTURE_CTE_MARKERS: tuple[str, ...] = ("__ref__", "__source__", "__seed__") -_IGNORED_SEARCH_DIRECTORIES: frozenset[str] = frozenset({".git", ".venv", "target", "node_modules"}) -_MODEL_FIXTURE_PATTERN: re.Pattern[str] = re.compile( - r"\b__ref__(?P[A-Za-z_][A-Za-z0-9_]*)\s+AS\s*\(", - re.IGNORECASE, -) - - -def has_fixture_typed_null_candidates(*, project_dir: Path) -> bool: - """Return whether source files may contain safe fixture-null autofixes.""" - - return bool(scan_fixture_typed_null_candidates(project_dir=project_dir).paths) - - -def scan_fixture_typed_null_candidates( - *, project_dir: Path, selected_paths: frozenset[Path] | None = None -) -> FixtureNullCandidateScan: - """Return selected SQL-test paths and model names with possible typed-null fixtures.""" - - paths: set[Path] = set() - model_names: set[str] = set() - tests_root: Path = project_dir / "tests" / "unit" - if not tests_root.is_dir(): - return FixtureNullCandidateScan() - for file_path in tests_root.rglob("*.sql"): - if any(part in _IGNORED_SEARCH_DIRECTORIES for part in file_path.parts): - continue - resolved_path: Path = file_path.resolve() - if selected_paths is not None and resolved_path not in selected_paths: - continue - try: - contents: str = file_path.read_text(encoding="utf-8") - except OSError: - continue - folded: str = contents.casefold() - if not any(marker in folded for marker in _FIXTURE_CTE_MARKERS): - continue - if _FIXTURE_TYPED_NULL_PATTERN.search(contents) is not None: - paths.add(resolved_path) - model_names.update( - match.group("name") for match in _MODEL_FIXTURE_PATTERN.finditer(contents) - ) - return FixtureNullCandidateScan( - paths=frozenset(paths), - model_names=frozenset(model_names), - ) diff --git a/src/sqlbuild/lint/_helpers/headers.py b/src/sqlbuild/lint/_helpers/headers.py index f34848ff89..b98fdfeed0 100644 --- a/src/sqlbuild/lint/_helpers/headers.py +++ b/src/sqlbuild/lint/_helpers/headers.py @@ -1,20 +1,16 @@ -"""Lintable body ranges of SQL files by project role, and externally resolved descriptions.""" +"""Lintable body ranges of SQL files by project role.""" from __future__ import annotations from pathlib import Path, PurePath from sqlbuild._native import measurement_body_ranges, sql_body_ranges -from sqlbuild.compiler.compile.classes.model_description_resolution import ( - ModelDescriptionResolution, -) from sqlbuild.compiler.compile.constants import HOOK_DIRECTORY_NAME from sqlbuild.compiler.discovery.main.named_declaration_role_kind import ( named_declaration_role_kind, ) from sqlbuild.compiler.scopes.types import DeclarationKind -from sqlbuild.lint.constants import RULE_DESCRIPTION_PRESENT -from sqlbuild.lint.models import HeaderSpan, LintFileRole, LintViolation +from sqlbuild.lint.models import HeaderSpan, LintFileRole def lint_file_role( @@ -54,22 +50,3 @@ def lint_body_ranges( if measurement_ranges: return measurement_ranges return sql_body_ranges(contents=contents, headers=headers) - - -def drop_resolved_description_faults( - *, violations: list[LintViolation], project_dir: Path -) -> list[LintViolation]: - """Drop `description-present` faults for models described outside their header.""" - - if not any(violation.code == RULE_DESCRIPTION_PRESENT for violation in violations): - return violations - described: frozenset[Path] = frozenset( - path.resolve() - for path in ModelDescriptionResolution.externally_described_models(project_dir=project_dir) - ) - return [ - violation - for violation in violations - if violation.code != RULE_DESCRIPTION_PRESENT - or violation.file_path.resolve() not in described - ] diff --git a/src/sqlbuild/lint/_helpers/native_format.py b/src/sqlbuild/lint/_helpers/native_format.py deleted file mode 100644 index a8b5af6bf0..0000000000 --- a/src/sqlbuild/lint/_helpers/native_format.py +++ /dev/null @@ -1,263 +0,0 @@ -"""Safe SQLBuild-owned formatting over the native Polyglot boundary.""" - -from __future__ import annotations - -import json -import re -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -import sqlbuild._native as _native -from sqlbuild._native import scan_headers -from sqlbuild.compiler.compile.constants import OMITTED_CEREMONIAL_SELECT_SQL -from sqlbuild.compiler.discovery.main.omitted_ceremonial_select import ( - omitted_ceremonial_select_offset, -) -from sqlbuild.compiler.sql_analysis.constants import POLYGLOT_MAX_FUNCTION_CALL_DEPTH -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint._helpers.headers import lint_body_ranges -from sqlbuild.lint._helpers.sqlbuild_tokens import neutralize_interpolation, restore_interpolation -from sqlbuild.lint.constants import ( - CARRIAGE_RETURN_LINE_FEED, - HEADER_KIND_SCENARIO, - HEADER_KIND_TEST, - LINE_FEED, - LINT_ENGINE_NATIVE, - RULE_FORMAT_UNSAFE, - VIOLATION_SEVERITY_FAULT, -) -from sqlbuild.lint.exceptions import InterpolationRestorationError, NativeLintError -from sqlbuild.lint.models import HeaderSpan, InterpolationSite, LintConfig, LintViolation - -_NATIVE_FORMAT_API_VERSION: int = 1 -_FIXTURE_HEADER_KINDS: frozenset[str] = frozenset({HEADER_KIND_TEST, HEADER_KIND_SCENARIO}) -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() -_OMITTED_SELECT_SUFFIX_PATTERN: re.Pattern[str] = re.compile(r"\s*\bSELECT\s+1\s*\Z", re.IGNORECASE) - - -@dataclass(frozen=True) -class _PreparedBody: - """One authored SQL body prepared for batched native formatting.""" - - start: int - end: int - trailing: str - cache_key: tuple[str, str, str] - interpolation_sites: tuple[InterpolationSite, ...] - omits_select: bool = False - - -def with_newline_style(*, contents: str, newline: str) -> str: - """Normalize generated contents back to the authored newline convention.""" - - normalized: str = contents.replace(CARRIAGE_RETURN_LINE_FEED, LINE_FEED) - if newline == LINE_FEED: - return normalized - return normalized.replace(LINE_FEED, newline) - - -def format_native_sql_bodies( - *, - files: dict[Path, str], - config: LintConfig, - project_dir: Path, -) -> tuple[dict[Path, str], list[LintViolation]]: - """Format SQL bodies and report every declined body as a named fault.""" - - prepared_by_path: dict[Path, tuple[_PreparedBody, ...]] = {} - faults: list[LintViolation] = [] - requests_by_key: dict[tuple[str, str, str], dict[str, object]] = {} - for file_path, contents in sorted(files.items()): - headers: tuple[HeaderSpan, ...] = scan_headers(contents=contents) - body_ranges: tuple[tuple[int, int], ...] = lint_body_ranges( - contents=contents, - headers=headers, - file_path=file_path, - project_dir=project_dir, - ) - allows_omitted_select: bool = any( - header.kind in _FIXTURE_HEADER_KINDS for header in headers - ) - prepared_bodies: list[_PreparedBody] = [] - for body_start, body_end in body_ranges: - body: str = contents[body_start:body_end] - trailing: str = body[len(body.rstrip()) :] - core: str = body[: len(body) - len(trailing)] if trailing else body - omitted_select_offset: int | None = ( - omitted_ceremonial_select_offset( - sql=core, - syntax=_GENERIC_SQL_SYNTAX, - ) - if allows_omitted_select - else None - ) - if omitted_select_offset is not None: - trailing = core[omitted_select_offset:] + trailing - core = core[:omitted_select_offset] - neutralized_core, sites = neutralize_interpolation( - body=core, dialect=config.dialect, for_formatting=True - ) - neutralized: str = ( - neutralized_core - if omitted_select_offset is None - else f"{neutralized_core}{OMITTED_CEREMONIAL_SELECT_SQL}" - ) - token_widths: dict[str, int] = _sentinel_widths(sites=sites) - cache_key: tuple[str, str, str] = ( - neutralized, - config.dialect, - repr(sorted(token_widths.items())), - ) - requests_by_key.setdefault( - cache_key, - { - "version": _NATIVE_FORMAT_API_VERSION, - "sql": neutralized, - "dialect": config.dialect, - "max_function_call_depth": POLYGLOT_MAX_FUNCTION_CALL_DEPTH, - "line_width": config.line_width, - "token_widths": token_widths, - }, - ) - prepared_bodies.append( - _PreparedBody( - start=body_start, - end=body_end, - trailing=trailing, - cache_key=cache_key, - interpolation_sites=sites, - omits_select=omitted_select_offset is not None, - ) - ) - prepared_by_path[file_path] = tuple(prepared_bodies) - response_cache: dict[tuple[str, str, str], dict[str, Any]] = _format_responses( - requests_by_key=requests_by_key - ) - formatted_files: dict[Path, str] = {} - for file_path, contents in sorted(files.items()): - updated: str = contents - for prepared in reversed(prepared_by_path[file_path]): - response: dict[str, Any] = response_cache[prepared.cache_key] - raw_sql: object = response.get("sql") - changed: object = response.get("changed") - formatted: object = response.get("formatted") - if ( - not isinstance(raw_sql, str) - or not isinstance(changed, bool) - or not isinstance(formatted, bool) - ): - raise NativeLintError("native formatter returned invalid SQL or changed state") - if not formatted: - faults.append( - _format_fault( - file_path=file_path, - contents=contents, - start=prepared.start, - reason=str( - response.get("reason") or "native formatter declined without a reason" - ), - ) - ) - continue - if prepared.omits_select: - suffix: re.Match[str] | None = _OMITTED_SELECT_SUFFIX_PATTERN.search(raw_sql) - if suffix is None: - faults.append( - _format_fault( - file_path=file_path, - contents=contents, - start=prepared.start, - reason="native formatter moved the omitted ceremonial SELECT 1", - ) - ) - continue - raw_sql = raw_sql[: suffix.start()] - changed = True - if not changed: - continue - try: - restored: str = restore_interpolation( - fixed=raw_sql, - sites=prepared.interpolation_sites, - ) - except InterpolationRestorationError as error: - faults.append( - _format_fault( - file_path=file_path, - contents=contents, - start=prepared.start, - reason=str(error), - ) - ) - continue - updated = ( - f"{updated[: prepared.start]}{restored}{prepared.trailing}{updated[prepared.end :]}" - ) - if updated != contents: - formatted_files[file_path] = updated - return formatted_files, faults - - -def _sentinel_widths(*, sites: tuple[InterpolationSite, ...]) -> dict[str, int]: - """Map each sentinel to the printed width of the SQLBuild syntax it stands in for.""" - - return { - site.sentinel.upper(): _printed_width(text=site.original_text) - for site in sites - if len(site.original_text) != len(site.sentinel) or LINE_FEED in site.original_text - } - - -def _printed_width(*, text: str) -> int: - lines: list[str] = text.splitlines() or [""] - return max(map(len, lines)) - - -def _format_fault(*, file_path: Path, contents: str, start: int, reason: str) -> LintViolation: - return LintViolation( - file_path=file_path, - line=contents.count(LINE_FEED, 0, start) + 1, - column=1, - code=RULE_FORMAT_UNSAFE, - message=f"File left unchanged because its SQL body could not be formatted: {reason}", - severity=VIOLATION_SEVERITY_FAULT, - engine=LINT_ENGINE_NATIVE, - remediation=( - "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. " - "No header, SQL body, or fixture changes were written to this file." - ), - ) - - -def _format_responses( - *, requests_by_key: dict[tuple[str, str, str], dict[str, object]] -) -> dict[tuple[str, str, str], dict[str, Any]]: - if not requests_by_key: - return {} - ordered_requests: tuple[tuple[tuple[str, str, str], dict[str, object]], ...] = tuple( - sorted(requests_by_key.items()) - ) - try: - decoded: object = json.loads( - _native.format_sql_batch_json( - json.dumps( - [request for _, request in ordered_requests], - sort_keys=True, - separators=(",", ":"), - ) - ) - ) - except (TypeError, ValueError) as error: - raise NativeLintError(str(error)) from error - if not isinstance(decoded, list) or len(decoded) != len(ordered_requests): - raise NativeLintError("native formatter returned an invalid batch response") - responses: dict[tuple[str, str, str], dict[str, Any]] = {} - for (cache_key, _), raw_response in zip(ordered_requests, decoded, strict=True): - if not isinstance(raw_response, dict): - raise NativeLintError("native formatter returned a non-object response") - response: dict[str, Any] = {str(key): value for key, value in raw_response.items()} - if response.get("version") != _NATIVE_FORMAT_API_VERSION: - raise NativeLintError("native formatter returned an unsupported response version") - responses[cache_key] = response - return responses diff --git a/src/sqlbuild/lint/_helpers/native_sql.py b/src/sqlbuild/lint/_helpers/native_sql.py deleted file mode 100644 index 631a5c1506..0000000000 --- a/src/sqlbuild/lint/_helpers/native_sql.py +++ /dev/null @@ -1,521 +0,0 @@ -"""Python boundary for SQLBuild-owned Polyglot-backed native SQL lint.""" - -from __future__ import annotations - -import re -from bisect import bisect_right -from pathlib import Path -from threading import Event -from typing import Any, cast - -import orjson - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import ( - ASSERT_TEST_CTE_PREFIX, - DBT_REF_TEST_CTE_PREFIX, - EXPECTED_TEST_CTE_PREFIX, - MACRO_ACTUAL_TEST_CTE_NAME, - MACRO_EXPECTED_TEST_CTE_NAME, - MACRO_TEST_CTE_PREFIX, - OMITTED_CEREMONIAL_SELECT_SQL, - REF_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, - TABLE_FN_ACTUAL_TEST_CTE_NAME, - TABLE_FN_EXPECTED_TEST_CTE_NAME, - UDF_ACTUAL_TEST_CTE_NAME, - UDF_EXPECTED_TEST_CTE_NAME, -) -from sqlbuild.compiler.compile.main.map_expanded_offset import map_expanded_offset -from sqlbuild.compiler.compile.models import MappedOffset -from sqlbuild.compiler.discovery.main.omitted_ceremonial_select import ( - omitted_ceremonial_select_offset, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint._helpers.sqlbuild_tokens import interpolation_text_at -from sqlbuild.lint.classes.stop_context import LintStopContext -from sqlbuild.lint.constants import ( - GENERATED_SQL_MESSAGE_SUFFIX, - GENERATED_SQL_MESSAGE_TEMPLATE, - LINT_ENGINE_NATIVE, - VIOLATION_SEVERITY_FAULT, - VIOLATION_SEVERITY_WARNING, -) -from sqlbuild.lint.exceptions import LintStoppedError, NativeLintError -from sqlbuild.lint.models import LintBody, LintConfig, LintEdit, LintViolation - -_NATIVE_LINT_API_VERSION: int = 1 -_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() -_UNUSED_CTE_CODE: str = "SQBRSQL005" -_LONG_LITERAL_CODE: str = "SQBRSQL044" -_PARSE_ERROR_POSITION_PATTERN: re.Pattern[str] = re.compile( - r"^Parse error at line (?P\d+), column (?P\d+):" -) -_SQLBUILD_HARNESS_CTE_PREFIXES: tuple[str, ...] = ( - EXPECTED_TEST_CTE_PREFIX, - ASSERT_TEST_CTE_PREFIX, - REF_TEST_CTE_PREFIX, - SOURCE_TEST_CTE_PREFIX, - SEED_TEST_CTE_PREFIX, - DBT_REF_TEST_CTE_PREFIX, - MACRO_TEST_CTE_PREFIX, - MACRO_ACTUAL_TEST_CTE_NAME, - MACRO_EXPECTED_TEST_CTE_NAME, - UDF_ACTUAL_TEST_CTE_NAME, - UDF_EXPECTED_TEST_CTE_NAME, - TABLE_FN_ACTUAL_TEST_CTE_NAME, - TABLE_FN_EXPECTED_TEST_CTE_NAME, -) -type _NativeCacheKey = tuple[ - str, - str, - tuple[str, ...] | None, - tuple[str, ...], - tuple[str, ...], - tuple[str, ...], - tuple[str, ...], - bool, - bool, - bool, - int, - int, - tuple[tuple[str, tuple[tuple[str, ...], ...]], ...], -] -type _NativeResult = dict[str, Any] | NativeLintError - - -def run_native_sql_lint( - *, - bodies: tuple[LintBody, ...], - contents_by_path: dict[Path, str], - config: LintConfig, -) -> dict[Path, tuple[LintViolation, ...]]: - """Lint expanded bodies in Rust and map diagnostics to authored source.""" - - violations_by_file: dict[Path, list[LintViolation]] = {} - line_starts_by_path: dict[Path, tuple[int, ...]] = {} - requests: dict[_NativeCacheKey, dict[str, object]] = {} - omitted_select_offsets: dict[str, int | None] = {} - for body in bodies: - cache_key: _NativeCacheKey = _native_cache_key(body=body, config=config) - if cache_key in requests: - continue - omitted_select_offset: int | None = _omitted_select_offset( - body=body, offsets=omitted_select_offsets - ) - payload: dict[str, object] = { - "version": _NATIVE_LINT_API_VERSION, - "sql": ( - body.lint_text - if omitted_select_offset is None - else f"{body.lint_text[:omitted_select_offset]}{OMITTED_CEREMONIAL_SELECT_SQL}" - f"{body.lint_text[omitted_select_offset:]}" - ), - "dialect": config.dialect, - } - if config.enabled_native_rules is not None: - payload["enabled_rules"] = list(config.enabled_native_rules) - if config.ignored_native_rules: - payload["ignored_rules"] = list(config.ignored_native_rules) - if body.external_identifiers: - payload["external_identifiers"] = list(body.external_identifiers) - if body.dependency_identifiers: - payload["dependency_identifiers"] = list(body.dependency_identifiers) - if body.externally_referenced_ctes: - payload["externally_referenced_ctes"] = list(body.externally_referenced_ctes) - if body.allows_ceremonial_select: - payload["allows_ceremonial_select"] = True - if body.allows_dynamic_output_star: - payload["allows_dynamic_output_star"] = True - if body.allows_empty_fixture_star: - payload["allows_empty_fixture_star"] = True - payload["max_literal_length"] = config.max_literal_length - payload["max_ranking_order_by"] = config.max_ranking_order_by - if cache_key[-1]: - payload["relation_keys"] = dict(cache_key[-1]) - requests[cache_key] = payload - check_lint_not_stopped() - response_cache: dict[_NativeCacheKey, _NativeResult] = _native_responses(requests=requests) - - for body in bodies: - check_lint_not_stopped() - response: _NativeResult = response_cache[_native_cache_key(body=body, config=config)] - contents: str = contents_by_path[body.file_path] - line_starts: tuple[int, ...] | None = line_starts_by_path.get(body.file_path) - if line_starts is None: - line_starts = source_line_starts(contents) - line_starts_by_path[body.file_path] = line_starts - if isinstance(response, NativeLintError): - parse_violation: LintViolation | None = _parse_failure_violation( - error=response, body=body, line_starts=line_starts - ) - if parse_violation is None: - raise response - violations_by_file.setdefault(body.file_path, []).append(parse_violation) - continue - raw_diagnostics: object = response.get("diagnostics") - if not isinstance(raw_diagnostics, list): - raise NativeLintError("native lint response is missing a diagnostics list") - body_omitted_select_offset: int | None = _omitted_select_offset( - body=body, offsets=omitted_select_offsets - ) - if body_omitted_select_offset is not None: - raw_diagnostics = _authored_prefix_diagnostics( - raw_diagnostics=raw_diagnostics, offset=body_omitted_select_offset - ) - for raw_diagnostic in raw_diagnostics: - try: - violation: LintViolation | None = _authored_violation( - raw_diagnostic=raw_diagnostic, - body=body, - contents=contents, - line_starts=line_starts, - dialect=config.dialect, - ) - except NativeLintError as error: - violation = _parse_failure_violation( - error=error, body=body, line_starts=line_starts - ) - if violation is not None: - violations_by_file.setdefault(body.file_path, []).append(violation) - return {path: tuple(entries) for path, entries in violations_by_file.items()} - - -def _omitted_select_offset(*, body: LintBody, offsets: dict[str, int | None]) -> int | None: - """Where a fixture body omits its ceremonial `SELECT 1`, so the parser sees one statement.""" - - if not body.allows_ceremonial_select: - return None - if body.lint_text not in offsets: - offsets[body.lint_text] = omitted_ceremonial_select_offset( - sql=body.lint_text, - syntax=_GENERIC_SQL_SYNTAX, - ) - return offsets[body.lint_text] - - -def _authored_prefix_diagnostics(*, raw_diagnostics: list[object], offset: int) -> list[object]: - """Keep diagnostics on authored SQL before the added `SELECT 1`, clamped to end there.""" - - kept: list[object] = [] - for raw_diagnostic in raw_diagnostics: - if not isinstance(raw_diagnostic, dict): - kept.append(raw_diagnostic) - continue - diagnostic: dict[str, object] = dict(cast("dict[str, object]", raw_diagnostic)) - start: object = diagnostic.get("start") - end: object = diagnostic.get("end") - if isinstance(start, int) and start >= offset: - continue - if isinstance(end, int) and end > offset: - diagnostic["end"] = offset - fix: object = diagnostic.get("fix") - fix_end: object = ( - cast("dict[str, object]", fix).get("end") if isinstance(fix, dict) else None - ) - if isinstance(fix_end, int) and fix_end > offset: - diagnostic["fix"] = None - diagnostic["fix_unavailable_reason"] = "fix overlaps generated SQL" - kept.append(diagnostic) - return kept - - -def check_lint_not_stopped() -> None: - """Abandon a speculative lint run on this thread once its owner no longer needs it.""" - - stop: Event | None = LintStopContext.active.get() - if stop is not None and stop.is_set(): - raise LintStoppedError - - -def _native_cache_key(*, body: LintBody, config: LintConfig) -> _NativeCacheKey: - return ( - body.lint_text, - config.dialect, - config.enabled_native_rules, - config.ignored_native_rules, - body.external_identifiers, - body.dependency_identifiers, - body.externally_referenced_ctes, - body.allows_ceremonial_select, - body.allows_dynamic_output_star, - body.allows_empty_fixture_star, - config.max_literal_length, - config.max_ranking_order_by, - _body_relation_keys(body=body, config=config), - ) - - -def _body_relation_keys( - *, body: LintBody, config: LintConfig -) -> tuple[tuple[str, tuple[tuple[str, ...], ...]], ...]: - """Declared keys of the relations one body reads, by their neutralized sentinel.""" - - return tuple( - (sentinel, config.relation_keys[name]) - for sentinel, name in body.dependency_relations - if config.relation_keys.get(name) - ) - - -def _native_responses( - *, requests: dict[_NativeCacheKey, dict[str, object]] -) -> dict[_NativeCacheKey, _NativeResult]: - """Analyze unique SQL bodies in one bounded native batch.""" - - keys: tuple[_NativeCacheKey, ...] = tuple(requests) - payloads: tuple[dict[str, object], ...] = tuple(requests.values()) - try: - decoded: object = orjson.loads( - _native.lint_sql_batch_json( - orjson.dumps(payloads, option=orjson.OPT_SORT_KEYS).decode() - ) - ) - except (TypeError, ValueError, RuntimeError) as error: - return dict.fromkeys(keys, NativeLintError(str(error))) - if not isinstance(decoded, list) or len(decoded) != len(keys): - return dict.fromkeys( - keys, NativeLintError("native lint engine returned an invalid batch response") - ) - results: tuple[_NativeResult, ...] = tuple(_native_result(response=value) for value in decoded) - return dict(zip(keys, results, strict=True)) - - -def _native_result(*, response: object) -> _NativeResult: - if not isinstance(response, dict): - return NativeLintError("native lint engine returned a non-object response") - payload: dict[str, object] = cast(dict[str, object], response) - if isinstance(payload.get("Err"), str): - return NativeLintError(cast(str, payload["Err"])) - decoded: object = payload.get("Ok") - if not isinstance(decoded, dict): - return NativeLintError("native lint engine returned a non-object response") - result: dict[str, Any] = cast(dict[str, Any], decoded) - if result.get("version") != _NATIVE_LINT_API_VERSION: - return NativeLintError("native lint engine returned an unsupported response version") - if not isinstance(result.get("diagnostics"), list): - return NativeLintError("native lint response is missing a diagnostics list") - return result - - -def _parse_failure_violation( - *, error: NativeLintError, body: LintBody, line_starts: tuple[int, ...] -) -> LintViolation: - """Convert a per-body native parse rejection into an authored project fault.""" - - match: re.Match[str] | None = _PARSE_ERROR_POSITION_PATTERN.match(error.message) - expanded_line: int = int(match.group("line")) if match else 1 - expanded_column: int = int(match.group("column")) if match else 1 - expanded_lines: list[str] = body.lint_text.splitlines(keepends=True) - expanded_offset: int = 0 - if 1 <= expanded_line <= len(expanded_lines): - expanded_offset = sum(len(line) for line in expanded_lines[: expanded_line - 1]) - expanded_offset += min(expanded_column - 1, len(expanded_lines[expanded_line - 1])) - mapped: MappedOffset = map_expanded_offset(offset=expanded_offset, passes=body.passes) - absolute_offset: int = body.body_start + mapped.offset - line, column = source_position(offset=absolute_offset, line_starts=line_starts) - return LintViolation( - file_path=body.file_path, - line=line, - column=column, - code=error.code, - message=f"Native SQL parser could not analyze this body: {error.message}", - severity=VIOLATION_SEVERITY_FAULT, - engine=LINT_ENGINE_NATIVE, - remediation="Use SQL syntax supported by the native parser or report the parser gap.", - ) - - -def _authored_violation( - *, - raw_diagnostic: object, - body: LintBody, - contents: str, - line_starts: tuple[int, ...], - dialect: str, -) -> LintViolation | None: - if not isinstance(raw_diagnostic, dict): - raise NativeLintError("native lint diagnostic must be an object") - diagnostic: dict[str, object] = cast("dict[str, object]", raw_diagnostic) - code: object = diagnostic.get("code") - message: object = diagnostic.get("message") - remediation: object = diagnostic.get("remediation") - start: object = diagnostic.get("start") - end: object = diagnostic.get("end") - raw_fix: object = diagnostic.get("fix") - raw_fix_unavailable_reason: object = diagnostic.get("fix_unavailable_reason") - if ( - not isinstance(code, str) - or not isinstance(message, str) - or not isinstance(remediation, str) - or not isinstance(start, int) - or isinstance(start, bool) - or not isinstance(end, int) - or isinstance(end, bool) - or start < 0 - or end < start - or end > len(body.lint_text) - ): - raise NativeLintError("native lint diagnostic has invalid code, message, or source span") - if raw_fix_unavailable_reason is not None and not isinstance(raw_fix_unavailable_reason, str): - raise NativeLintError("native lint diagnostic has an invalid fix refusal reason") - mapped: MappedOffset = map_expanded_offset(offset=start, passes=body.passes) - absolute_offset: int = body.body_start + mapped.offset - if code == _LONG_LITERAL_CODE and mapped.generated: - return None - if code == _UNUSED_CTE_CODE and contents.startswith( - _SQLBUILD_HARNESS_CTE_PREFIXES, absolute_offset - ): - return None - line, column = source_position(offset=absolute_offset, line_starts=line_starts) - end_position: tuple[int, int] | None = _authored_end_position( - start=start, - end=end, - mapped_start=mapped, - absolute_start=absolute_offset, - body=body, - contents=contents, - line_starts=line_starts, - dialect=dialect, - ) - fix: LintEdit | None = _authored_fix( - raw_fix=raw_fix, - code=code, - body=body, - ) - fix_unavailable_reason: str | None = raw_fix_unavailable_reason - if raw_fix is not None and fix is None: - fix_unavailable_reason = "fix overlaps generated SQL or a non-contiguous authored region" - return LintViolation( - file_path=body.file_path, - line=line, - column=column, - code=code, - message=_violation_message( - message=message, - mapped=mapped, - contents=contents, - absolute_offset=absolute_offset, - dialect=dialect, - ), - severity=VIOLATION_SEVERITY_WARNING, - engine=LINT_ENGINE_NATIVE, - end_line=end_position[0] if end_position is not None else None, - end_column=end_position[1] if end_position is not None else None, - remediation=remediation, - fix=fix, - fix_unavailable_reason=fix_unavailable_reason, - ) - - -def _authored_fix(*, raw_fix: object, code: str, body: LintBody) -> LintEdit | None: - """Map a native edit only when both boundaries remain contiguous authored source.""" - - if raw_fix is None: - return None - if not isinstance(raw_fix, dict): - raise NativeLintError("native lint diagnostic fix must be an object") - payload: dict[str, object] = cast("dict[str, object]", raw_fix) - start: object = payload.get("start") - end: object = payload.get("end") - replacement: object = payload.get("replacement") - if ( - not isinstance(start, int) - or isinstance(start, bool) - or not isinstance(end, int) - or isinstance(end, bool) - or not isinstance(replacement, str) - or start < 0 - or end <= start - or end > len(body.lint_text) - ): - raise NativeLintError("native lint diagnostic has an invalid fix edit") - region_start: int = start - region_end: int = end - for spans in reversed(body.passes): - if any( - region_start < span.output_end - and span.output_start < region_end - or ( - span.output_start == span.output_end - and region_start <= span.output_start < region_end - ) - for span in spans - ): - return None - region_start = map_expanded_offset(offset=region_start, passes=(spans,)).offset - region_end = map_expanded_offset(offset=region_end, passes=(spans,)).offset - mapped_start: MappedOffset = map_expanded_offset(offset=start, passes=body.passes) - mapped_last: MappedOffset = map_expanded_offset(offset=end - 1, passes=body.passes) - if ( - mapped_start.generated - or mapped_last.generated - or mapped_last.offset - mapped_start.offset != end - start - 1 - ): - return None - return LintEdit( - file_path=body.file_path, - code=code, - start=body.body_start + mapped_start.offset, - end=body.body_start + mapped_last.offset + 1, - replacement=replacement, - ) - - -def _authored_end_position( - *, - start: int, - end: int, - mapped_start: MappedOffset, - absolute_start: int, - body: LintBody, - contents: str, - line_starts: tuple[int, ...], - dialect: str, -) -> tuple[int, int] | None: - """Map a native exclusive end offset only when authored continuity is provable.""" - - if mapped_start.generated: - token: str | None = interpolation_text_at( - body=contents, start=absolute_start, dialect=dialect - ) - if token is None: - return None - return source_position(offset=absolute_start + len(token), line_starts=line_starts) - if end <= start: - return None - mapped_last: MappedOffset = map_expanded_offset(offset=end - 1, passes=body.passes) - if mapped_last.generated or mapped_last.offset - mapped_start.offset != end - start - 1: - return None - absolute_end: int = body.body_start + mapped_last.offset + 1 - return source_position(offset=absolute_end, line_starts=line_starts) - - -def _violation_message( - *, message: str, mapped: MappedOffset, contents: str, absolute_offset: int, dialect: str -) -> str: - if not mapped.generated: - return message - token: str | None = interpolation_text_at(body=contents, start=absolute_offset, dialect=dialect) - if token is None: - return f"{message} {GENERATED_SQL_MESSAGE_SUFFIX}" - return f"{message} {GENERATED_SQL_MESSAGE_TEMPLATE.format(token=token)}" - - -def source_line_starts(contents: str) -> tuple[int, ...]: - """Return the offset where each line of contents starts.""" - - starts: list[int] = [0] - index: int = contents.find("\n") - while index != -1: - starts.append(index + 1) - index = contents.find("\n", index + 1) - return tuple(starts) - - -def source_position(*, offset: int, line_starts: tuple[int, ...]) -> tuple[int, int]: - """Map a zero-based offset to a one-based line and column.""" - - line_index: int = bisect_right(line_starts, offset) - 1 - return line_index + 1, offset - line_starts[line_index] + 1 diff --git a/src/sqlbuild/lint/_helpers/project_files.py b/src/sqlbuild/lint/_helpers/project_files.py deleted file mode 100644 index 316ff051ae..0000000000 --- a/src/sqlbuild/lint/_helpers/project_files.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Collect authored SQL files for lint and format runs.""" - -from __future__ import annotations - -from collections.abc import Callable -from functools import cache -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint.constants import LINT_DIRECTORY_NAMES - - -def collect_project_files( - *, - project_dir: Path, - selected_paths: frozenset[Path] | None = None, - source_files: dict[Path, str] | None = None, - discovered_inputs: DiscoveredProjectInputs | None = None, -) -> dict[Path, str]: - """Return authored SQL files keyed by path, sharing identical text discovery already holds.""" - - resolved_directory: Callable[[Path], Path] = cache(_resolved_path) - if source_files is not None: - return { - file_path: contents - for file_path, contents in source_files.items() - if selected_paths is None - or _resolved_file(path=file_path, resolved_directory=resolved_directory) - in selected_paths - } - discovered: dict[Path, str] = ( - {} if discovered_inputs is None else discovered_sql_contents(discovered_inputs) - ) - files: dict[Path, str] = {} - directory_name: str - for directory_name in LINT_DIRECTORY_NAMES: - root: Path = project_dir / directory_name - if not root.is_dir(): - continue - file_path: Path - for file_path in sorted(root.rglob("*.sql")): - if ( - selected_paths is not None - and _resolved_file(path=file_path, resolved_directory=resolved_directory) - not in selected_paths - ): - continue - with file_path.open("r", encoding="utf-8", newline="") as handle: - contents: str = handle.read() - known: str | None = discovered.get(file_path) - files[file_path] = known if known is not None and known == contents else contents - return files - - -def _resolved_file(*, path: Path, resolved_directory: Callable[[Path], Path]) -> Path: - """Resolve a file as Path.resolve() does, resolving each parent directory only once.""" - - if path.is_symlink(): - return path.resolve() - return resolved_directory(path.parent) / path.name - - -def _resolved_path(path: Path) -> Path: - return path.resolve() - - -def discovered_sql_contents(discovered_inputs: DiscoveredProjectInputs) -> dict[Path, str]: - """Return the text discovery read for every SQL file it retains, keyed by path.""" - - contents: dict[Path, str] = {} - for group in ( - discovered_inputs.model_files, - discovered_inputs.enum_files, - discovered_inputs.constant_files, - discovered_inputs.model_schema_files, - discovered_inputs.sql_function_files, - discovered_inputs.sql_hook_files, - discovered_inputs.test_files, - discovered_inputs.scenario_files, - discovered_inputs.audit_files, - ): - contents.update({item.file_path: item.contents for item in group}) - return contents - - -def sort_violations(violations: list) -> tuple: - """Return violations sorted by file path then position.""" - - return tuple(sorted(violations, key=lambda item: (str(item.file_path), item.line, item.column))) diff --git a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py deleted file mode 100644 index eb7b02dc82..0000000000 --- a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py +++ /dev/null @@ -1,394 +0,0 @@ -"""Neutralize SQLBuild interpolation for native SQL analysis and formatting.""" - -from __future__ import annotations - -import re -from functools import cache - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import ( - MACRO_TOKEN, - SQL_INTERPOLATION_TOKEN, -) -from sqlbuild.compiler.compile.models import ExpansionSpan -from sqlbuild.lint.constants import ( - BACKTICK_CHARACTER, - CLOSING_PAREN_CHARACTER, - DOLLAR_QUOTE_CHARACTER, - EXPECTED_SENTINEL_OCCURRENCES, - IDENTIFIER_EXTRA_CHARACTER, - INTERPOLATION_NAME_EXTRA_CHARACTERS, - OPENING_PAREN_CHARACTER, - SENTINEL_TEMPLATE, - SINGLE_QUOTE_CHARACTER, - SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN, - SQL_ARGUMENT_RAW_PARAMETER_PATTERN, - SQL_ESCAPE_CHARACTER, - SQL_QUOTE_CHARACTERS, - TEMPLATE_INTERPOLATION_END, - TEMPLATE_INTERPOLATION_START, -) -from sqlbuild.lint.exceptions import InterpolationRestorationError -from sqlbuild.lint.models import InterpolationSite - -_SQLBUILD_REFERENCE_FUNCTION_NAMES: tuple[str, ...] = ( - "__dbt_ref", - "__table_fn", - "__source", - "__seed", - "__udf", - "__ref", -) -_SQLBUILD_FUNCTION_NAMES: tuple[str, ...] = ( - "__cursor_start", - "__cursor_end", - "__empty_fixture", - *_SQLBUILD_REFERENCE_FUNCTION_NAMES, -) -_AUDIT_PARAMETER_SENTINEL_TEMPLATE: str = "__sqlbuild_audit_parameter_{index}__" -_CONTEXT_SENTINEL_TEMPLATE: str = "__sqlbuild_context_parameter_{index}__" -_LINE_COMMENT_START: str = "--" -_BLOCK_COMMENT_START: str = "/*" -_BLOCK_COMMENT_END: str = "*/" -_DOLLAR_QUOTE_SCAN_PATTERN: str = ( - r"(?(?:[A-Za-z_][A-Za-z0-9_]*)?)\$" - r"(?:[\s\S]*?\$(?P=dollar_tag)\$|[\s\S]*\Z)" -) -_NON_CODE_SCAN_PATTERN: str = ( - r"--[^\n]*(?:\n|\Z)" - r"|/\*[\s\S]*?(?:\*/|\Z)" - r"|'(?:\\.|''|[^'\\])*(?:'|\Z)" - r'|"(?:\\.|""|[^"\\])*(?:"|\Z)' - rf"|{_DOLLAR_QUOTE_SCAN_PATTERN}" -) -_BACKTICK_SCAN_PATTERN: str = r"|`(?:``|[^`])*(?:`|\Z)" -_DOLLAR_QUOTE_PATTERN: re.Pattern[str] = re.compile(_DOLLAR_QUOTE_SCAN_PATTERN) - - -@cache -def _interpolation_scan_pattern( - *, backtick_identifiers: bool, for_formatting: bool -) -> re.Pattern[str]: - """Keep lint fallback lexical policy identical to the native ASCII fast path.""" - - names: tuple[str, ...] = ( - _SQLBUILD_FUNCTION_NAMES if for_formatting else _SQLBUILD_REFERENCE_FUNCTION_NAMES - ) - functions: str = "|".join(re.escape(name) for name in sorted(names)) - whitespace: str = r"\s*" if for_formatting else "" - sites: str = rf"|(?P@@|\$\{{|@|(?:{functions}){whitespace}\()" - backticks: str = _BACKTICK_SCAN_PATTERN if backtick_identifiers else "" - flags: re.RegexFlag = re.IGNORECASE if for_formatting else re.NOFLAG - return re.compile(_NON_CODE_SCAN_PATTERN + backticks + sites, flags) - - -@cache -def _backtick_identifiers(dialect: str) -> bool: - """Return whether lint treats backticks as identifier quotes, matching the native lexer.""" - - return _native.lint_backtick_identifiers(dialect) - - -def neutralize_context_interpolation(*, body: str) -> tuple[str, tuple[InterpolationSite, ...]]: - """Replace runtime CTX interpolation with parseable generated sentinels.""" - - sites: list[InterpolationSite] = [] - pieces: list[str] = [] - output_length: int = 0 - copied_to: int = 0 - token: str = f"{SQL_INTERPOLATION_TOKEN}CTX:" - while True: - start: int = body.find(token, copied_to) - if start < 0: - break - end: int = _interpolation_name_end(body=body, start=start + len(SQL_INTERPOLATION_TOKEN)) - literal: str = body[copied_to:start] - sentinel: str = _CONTEXT_SENTINEL_TEMPLATE.format(index=len(sites)) - pieces.extend((literal, sentinel)) - output_length += len(literal) - sites.append( - InterpolationSite( - sentinel=sentinel, - neutralized_start=output_length, - neutralized_end=output_length + len(sentinel), - original_start=start, - original_end=end, - original_text=body[start:end], - ) - ) - output_length += len(sentinel) - copied_to = end - pieces.append(body[copied_to:]) - return "".join(pieces), tuple(sites) - - -def neutralize_generic_audit_parameters(*, body: str) -> tuple[str, tuple[InterpolationSite, ...]]: - """Replace unbound generic-audit arguments with parseable generated sentinels.""" - - matches: list[tuple[int, int, bool]] = [] - for match in SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN.finditer(body): - matches.append((match.start(), match.end(), True)) - for match in SQL_ARGUMENT_RAW_PARAMETER_PATTERN.finditer(body): - matches.append((match.start(), match.end(), False)) - matches.sort(key=lambda item: item[0]) - - sites: list[InterpolationSite] = [] - pieces: list[str] = [] - output_length: int = 0 - copied_to: int = 0 - for start, end, quoted in matches: - literal: str = body[copied_to:start] - identifier: str = _AUDIT_PARAMETER_SENTINEL_TEMPLATE.format(index=len(sites)) - sentinel: str = f"'{identifier}'" if quoted else identifier - pieces.extend((literal, sentinel)) - output_length += len(literal) - sites.append( - InterpolationSite( - sentinel=sentinel, - neutralized_start=output_length, - neutralized_end=output_length + len(sentinel), - original_start=start, - original_end=end, - original_text=body[start:end], - ) - ) - output_length += len(sentinel) - copied_to = end - pieces.append(body[copied_to:]) - return "".join(pieces), tuple(sites) - - -def neutralize_interpolation( - *, body: str, dialect: str, for_formatting: bool = False -) -> tuple[str, tuple[InterpolationSite, ...]]: - """Neutralize references for lint, or protect every intrinsic spelling for formatting.""" - - if not _contains_interpolation_candidate(body=body): - return body, () - backtick_identifiers: bool = _backtick_identifiers(dialect) - scan_pattern: re.Pattern[str] = _interpolation_scan_pattern( - backtick_identifiers=backtick_identifiers, for_formatting=for_formatting - ) - sites: list[InterpolationSite] = [] - pieces: list[str] = [] - neutralized_length: int = 0 - copied_to: int = 0 - match: re.Match[str] - for match in scan_pattern.finditer(body): - if match.group("site") is None or match.start() < copied_to: - continue - site_start: int = match.start() - site_end: int | None = _interpolation_site_end( - body=body, start=site_start, backtick_identifiers=backtick_identifiers - ) - if site_end is None: - continue - literal: str = body[copied_to:site_start] - sentinel: str = SENTINEL_TEMPLATE.format(index=len(sites)) - pieces.append(literal) - pieces.append(sentinel) - neutralized_length += len(literal) - sites.append( - InterpolationSite( - sentinel=sentinel, - neutralized_start=neutralized_length, - neutralized_end=neutralized_length + len(sentinel), - original_start=site_start, - original_end=site_end, - original_text=body[site_start:site_end], - ) - ) - neutralized_length += len(sentinel) - copied_to = site_end - pieces.append(body[copied_to:]) - return "".join(pieces), tuple(sites) - - -def _contains_interpolation_candidate(*, body: str) -> bool: - return ( - MACRO_TOKEN in body - or TEMPLATE_INTERPOLATION_START in body - or any(name in body.lower() for name in _SQLBUILD_FUNCTION_NAMES) - ) - - -def restore_interpolation(*, fixed: str, sites: tuple[InterpolationSite, ...]) -> str: - """Swap sentinels back to their original text, failing on any ambiguity.""" - - restored: str = fixed - site: InterpolationSite - for site in sites: - pattern: re.Pattern[str] = re.compile(re.escape(site.sentinel), re.IGNORECASE) - occurrences: int = len(tuple(pattern.finditer(restored))) - if occurrences != EXPECTED_SENTINEL_OCCURRENCES: - raise InterpolationRestorationError( - f"formatted SQL contains {occurrences} occurrences of sentinel " - f"'{site.sentinel}' standing in for '{site.original_text}'; expected " - f"exactly {EXPECTED_SENTINEL_OCCURRENCES}" - ) - restored = pattern.sub( - lambda _match, original_text=site.original_text: original_text, - restored, - ) - return restored - - -def interpolation_text_at(*, body: str, start: int, dialect: str) -> str | None: - """Return the interpolation token beginning at an offset, if there is one.""" - - site_end: int | None = _interpolation_site_end( - body=body, start=start, backtick_identifiers=_backtick_identifiers(dialect) - ) - if site_end is None: - return None - return body[start:site_end] - - -def sentinel_spans(*, sites: tuple[InterpolationSite, ...]) -> tuple[ExpansionSpan, ...]: - """Express sentinel substitutions as expansion spans for offset mapping.""" - - spans: list[ExpansionSpan] = [] - site: InterpolationSite - for site in sites: - spans.append( - ExpansionSpan( - source_start=site.original_start, - source_end=site.original_end, - output_start=site.neutralized_start, - output_end=site.neutralized_end, - ) - ) - return tuple(spans) - - -def _interpolation_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: - character: str = body[start] - if character == MACRO_TOKEN: - if body.startswith(SQL_INTERPOLATION_TOKEN, start): - return _interpolation_name_end(body=body, start=start + len(SQL_INTERPOLATION_TOKEN)) - return _macro_site_end(body=body, start=start, backtick_identifiers=backtick_identifiers) - if character == TEMPLATE_INTERPOLATION_START[0]: - return _template_site_end(body=body, start=start) - if ( - character == IDENTIFIER_EXTRA_CHARACTER - and body[start : _identifier_end(body=body, start=start)].lower() - in _SQLBUILD_FUNCTION_NAMES - ): - return _sqlbuild_function_site_end( - body=body, start=start, backtick_identifiers=backtick_identifiers - ) - return None - - -def _sqlbuild_function_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: - name_end: int = _identifier_end(body=body, start=start) - while name_end < len(body) and body[name_end].isspace(): - name_end += 1 - if name_end >= len(body) or body[name_end] != OPENING_PAREN_CHARACTER: - return None - call_end: int | None = _matching_paren_end( - body=body, opening_index=name_end, backtick_identifiers=backtick_identifiers - ) - return call_end - - -def _template_site_end(*, body: str, start: int) -> int | None: - end_index: int = body.find( - TEMPLATE_INTERPOLATION_END, start + len(TEMPLATE_INTERPOLATION_START) - ) - if end_index < 0: - return None - return end_index + len(TEMPLATE_INTERPOLATION_END) - - -def _macro_site_end(*, body: str, start: int, backtick_identifiers: bool) -> int | None: - name_start: int = start + len(MACRO_TOKEN) - if name_start >= len(body): - return None - if body[name_start] == SINGLE_QUOTE_CHARACTER: - closing_index: int = body.find(SINGLE_QUOTE_CHARACTER, name_start + 1) - if closing_index < 0: - return None - return closing_index + 1 - if not _is_identifier_start(character=body[name_start]): - return None - name_end: int = _identifier_end(body=body, start=name_start) - if name_end >= len(body) or body[name_end] != OPENING_PAREN_CHARACTER: - return name_end - return _matching_paren_end( - body=body, opening_index=name_end, backtick_identifiers=backtick_identifiers - ) - - -def _matching_paren_end(*, body: str, opening_index: int, backtick_identifiers: bool) -> int | None: - depth: int = 0 - index: int = opening_index - length: int = len(body) - quote_character: str | None = None - while index < length: - if quote_character is None and body.startswith(_LINE_COMMENT_START, index): - newline_index: int = body.find("\n", index + len(_LINE_COMMENT_START)) - index = length if newline_index < 0 else newline_index + 1 - continue - if quote_character is None and body.startswith(_BLOCK_COMMENT_START, index): - comment_end: int = body.find(_BLOCK_COMMENT_END, index + len(_BLOCK_COMMENT_START)) - index = length if comment_end < 0 else comment_end + len(_BLOCK_COMMENT_END) - continue - if quote_character is None and body.startswith(DOLLAR_QUOTE_CHARACTER, index): - dollar_quote: re.Match[str] | None = _DOLLAR_QUOTE_PATTERN.match(body, index) - if dollar_quote is not None: - index = dollar_quote.end() - continue - character: str = body[index] - if quote_character is not None: - if ( - character == SQL_ESCAPE_CHARACTER - and quote_character in SQL_QUOTE_CHARACTERS - and index + 1 < length - ): - index += 2 - continue - if character == quote_character: - if index + 1 < length and body[index + 1] == quote_character: - index += 2 - continue - quote_character = None - index += 1 - continue - if character in SQL_QUOTE_CHARACTERS or ( - backtick_identifiers and character == BACKTICK_CHARACTER - ): - quote_character = character - index += 1 - continue - if character == OPENING_PAREN_CHARACTER: - depth += 1 - if character == CLOSING_PAREN_CHARACTER: - depth -= 1 - if depth == 0: - return index + 1 - index += 1 - return None - - -def _interpolation_name_end(*, body: str, start: int) -> int: - index: int = start - while index < len(body) and ( - body[index].isalnum() or body[index] in INTERPOLATION_NAME_EXTRA_CHARACTERS - ): - index += 1 - return index - - -def _identifier_end(*, body: str, start: int) -> int: - index: int = start - while index < len(body) and ( - body[index].isalnum() or body[index] == IDENTIFIER_EXTRA_CHARACTER - ): - index += 1 - return index - - -def _is_identifier_start(*, character: str) -> bool: - return character.isalpha() or character == IDENTIFIER_EXTRA_CHARACTER diff --git a/src/sqlbuild/lint/_helpers/suppressions.py b/src/sqlbuild/lint/_helpers/suppressions.py deleted file mode 100644 index 9b62f460e1..0000000000 --- a/src/sqlbuild/lint/_helpers/suppressions.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Reason-required local suppression handling for lint diagnostics.""" - -from __future__ import annotations - -import re -from dataclasses import dataclass, replace -from pathlib import Path - -from sqlbuild.errors.setting_help.main.setting_help import setting_help -from sqlbuild.errors.setting_help.main.setting_note import setting_note -from sqlbuild.lint.constants import ( - ALLOW_EXCEPTIONS_KEY, - LINT_ENGINE_NATIVE, - PROJECT_CONFIG_FILENAME_KEY, - RULES_SECTION_KEY, - VIOLATION_SEVERITY_FAULT, - VIOLATION_SEVERITY_WARNING, -) -from sqlbuild.lint.models import LintEdit, LintViolation - -_SUPPRESSION_PREFIX: str = "sqb: ignore" -_SUPPRESSION_PATTERN: re.Pattern[str] = re.compile( - r"^\s*--\s*sqb:\s*ignore\s+(?P[A-Za-z0-9_-]+)\s+because\s+(?P\S.*)\s*$" -) -_SUPPRESSION_DIAGNOSTIC_CODE: str = "SQBRSQL000" - - -@dataclass(frozen=True) -class _Suppression: - code: str - directive_line: int - target_line: int - - -def apply_suppressions( - *, - violations: list[LintViolation], - contents_by_path: dict[Path, str], - allow_suppressions: bool, -) -> list[LintViolation]: - """Remove matching next-line diagnostics, or reject every directive when forbidden.""" - - retained: list[LintViolation] = list(violations) - for file_path, contents in contents_by_path.items(): - suppressions, invalid = _parse_suppressions(file_path=file_path, contents=contents) - if not allow_suppressions: - directive_lines: set[int] = {violation.line for violation in invalid} - directive_lines.update(suppression.directive_line for suppression in suppressions) - retained.extend( - _forbidden_violation(file_path=file_path, line=line, contents=contents) - for line in sorted(directive_lines) - ) - continue - retained.extend(invalid) - for suppression in suppressions: - match_index: int | None = next( - ( - index - for index, violation in enumerate(retained) - if violation.file_path == file_path - and violation.line == suppression.target_line - and violation.code == suppression.code - and violation.engine == LINT_ENGINE_NATIVE - and violation.code.startswith("SQBRSQL") - ), - None, - ) - if match_index is None: - retained.append( - _suppression_violation( - file_path=file_path, - line=suppression.directive_line, - message=f"Unused suppression for {suppression.code}", - remediation="Remove the stale suppression directive.", - contents=contents, - fixable=True, - ) - ) - else: - retained.pop(match_index) - return retained - - -def _parse_suppressions( - *, file_path: Path, contents: str -) -> tuple[tuple[_Suppression, ...], tuple[LintViolation, ...]]: - lines: list[str] = contents.splitlines() - suppressions: list[_Suppression] = [] - invalid: list[LintViolation] = [] - for index, line in enumerate(lines): - if _SUPPRESSION_PREFIX not in line.lower(): - continue - match: re.Match[str] | None = _SUPPRESSION_PATTERN.match(line) - if match is None: - invalid.append( - _suppression_violation( - file_path=file_path, - line=index + 1, - message="Suppression directive is invalid", - remediation="Use '-- sqb: ignore CODE because '.", - contents=contents, - fixable=False, - ) - ) - continue - target_index: int = index + 1 - while target_index < len(lines) and ( - not lines[target_index].strip() or lines[target_index].lstrip().startswith("--") - ): - target_index += 1 - suppressions.append( - _Suppression( - code=match.group("code"), - directive_line=index + 1, - target_line=target_index + 1, - ) - ) - return tuple(suppressions), tuple(invalid) - - -def _forbidden_violation(*, file_path: Path, line: int, contents: str) -> LintViolation: - violation: LintViolation = _suppression_violation( - file_path=file_path, - line=line, - message="Inline Rule suppressions are not allowed in this project", - remediation=( - "Remove the directive and fix the finding it suppresses; " - + setting_note( - file_name=PROJECT_CONFIG_FILENAME_KEY, - section=RULES_SECTION_KEY, - key=ALLOW_EXCEPTIONS_KEY, - value=False, - ) - + ". " - + setting_help( - purpose="To allow inline suppressions", - file_name=PROJECT_CONFIG_FILENAME_KEY, - section=RULES_SECTION_KEY, - key=ALLOW_EXCEPTIONS_KEY, - value=True, - ) - ), - contents=contents, - fixable=False, - ) - return replace( - violation, - severity=VIOLATION_SEVERITY_FAULT, - fix_unavailable_reason="removing the directive exposes the finding it suppresses", - ) - - -def _suppression_violation( - *, - file_path: Path, - line: int, - message: str, - remediation: str, - contents: str, - fixable: bool, -) -> LintViolation: - start, end = _line_range(contents=contents, line=line) - return LintViolation( - file_path=file_path, - line=line, - column=1, - code=_SUPPRESSION_DIAGNOSTIC_CODE, - message=message, - severity=VIOLATION_SEVERITY_WARNING, - engine=LINT_ENGINE_NATIVE, - remediation=remediation, - fix=( - LintEdit( - file_path=file_path, - code=_SUPPRESSION_DIAGNOSTIC_CODE, - start=start, - end=end, - replacement="", - ) - if fixable - else None - ), - fix_unavailable_reason=( - None if fixable else "a valid suppression reason requires user intent" - ), - ) - - -def _line_range(*, contents: str, line: int) -> tuple[int, int]: - lines: list[str] = contents.splitlines(keepends=True) - start: int = sum(len(entry) for entry in lines[: line - 1]) - return start, start + len(lines[line - 1]) diff --git a/src/sqlbuild/lint/classes/stop_context.py b/src/sqlbuild/lint/classes/stop_context.py index 11d2c0b9c0..3b6215ebd6 100644 --- a/src/sqlbuild/lint/classes/stop_context.py +++ b/src/sqlbuild/lint/classes/stop_context.py @@ -4,8 +4,18 @@ from threading import Event from typing import ClassVar +from sqlbuild.lint.exceptions import LintStoppedError + class LintStopContext: """Own the stop signal that lets a speculative lint run be abandoned between files.""" active: ClassVar[ContextVar[Event | None]] = ContextVar("active_lint_stop", default=None) + + @staticmethod + def check() -> None: + """Abandon a speculative lint run on this thread once its owner no longer needs it.""" + + stop: Event | None = LintStopContext.active.get() + if stop is not None and stop.is_set(): + raise LintStoppedError diff --git a/src/sqlbuild/lint/constants.py b/src/sqlbuild/lint/constants.py index 947f1197ff..a5a45b6d9f 100644 --- a/src/sqlbuild/lint/constants.py +++ b/src/sqlbuild/lint/constants.py @@ -2,8 +2,6 @@ from __future__ import annotations -import re - from sqlbuild.lint.types import LintSeverity LINT_ENGINE_SQLBUILD: str = "sqlbuild" @@ -12,39 +10,9 @@ VIOLATION_SEVERITY_FAULT: LintSeverity = LintSeverity.FAULT VIOLATION_SEVERITY_WARNING: LintSeverity = LintSeverity.WARNING -HEADER_KIND_MODEL: str = "MODEL" HEADER_KIND_SCENARIO: str = "SCENARIO" HEADER_KIND_TEST: str = "TEST" -HEADER_KIND_AUDIT: str = "AUDIT" -HEADER_KIND_FUNCTION: str = "FUNCTION" -HEADER_KIND_ENUM: str = "ENUM" -HEADER_KIND_CONSTANT: str = "CONSTANT" -HEADER_KIND_HOOK: str = "HOOK" -HEADER_KIND_SCHEMA: str = "SCHEMA" - -DSL_HEADER_KINDS: frozenset[str] = frozenset( - { - HEADER_KIND_MODEL, - HEADER_KIND_SCENARIO, - HEADER_KIND_TEST, - HEADER_KIND_AUDIT, - HEADER_KIND_FUNCTION, - HEADER_KIND_ENUM, - HEADER_KIND_CONSTANT, - HEADER_KIND_HOOK, - HEADER_KIND_SCHEMA, - } -) -IDENTIFIER_SEPARATOR_CHARACTER: str = "_" - -DESCRIPTION_HEADER_KINDS: frozenset[str] = frozenset({HEADER_KIND_MODEL, HEADER_KIND_SCENARIO}) -DESCRIPTION_REQUIRED_HEADER_KINDS: frozenset[str] = frozenset( - {HEADER_KIND_MODEL, HEADER_KIND_SCENARIO, HEADER_KIND_FUNCTION, HEADER_KIND_HOOK} -) - -DEFAULT_MAX_DESCRIPTION_LINES: int = 10 -DEFAULT_LINE_WIDTH: int = 100 DEFAULT_MAX_LITERAL_LENGTH: int = 100 DEFAULT_MAX_RANKING_ORDER_BY: int = 5 MAX_LITERAL_LENGTH_THRESHOLD: str = "max_literal_length" @@ -62,64 +30,7 @@ } ) -LINT_DIRECTORY_NAMES: tuple[str, ...] = ( - "models", - "tests", - "audits", - "functions", - "enums", - "constants", - "hooks", - "schemas", -) -PARENT_PATH_SEGMENT: str = ".." -SQL_FILE_SUFFIX: str = ".sql" - -RULE_DESCRIPTION_PRESENT: str = "description-present" -RULE_DESCRIPTION_LENGTH: str = "description-length" -RULE_LEADING_COMMENT_DESCRIPTION: str = "leading-comment-description" -RULE_HEADER_WHITESPACE: str = "header-whitespace" -RULE_HEADER_PARSE: str = "header-parse" -RULE_FORMAT_UNSAFE: str = "format-unsafe" - -PROJECT_CONFIG_FILENAME_KEY: str = "sqlbuild_project.toml" -FORMAT_SECTION_KEY: str = "format" -RULES_SECTION_KEY: str = "rules" -ALLOW_EXCEPTIONS_KEY: str = "allow_exceptions" -MAX_DESCRIPTION_LINES_KEY: str = "max_description_lines" -LINE_WIDTH_KEY: str = "line_width" - -ADAPTER_DIALECT_TRANSLATIONS: dict[str, str] = { - "duckdb": "duckdb", - "motherduck": "duckdb", - "snowflake": "snowflake", - "bigquery": "bigquery", - "databricks": "databricks", - "postgres": "postgres", - "sqlserver": "tsql", -} -ADAPTER_CONFIG_KEY: str = "adapter" - -SQL_QUOTE_CHARACTERS: frozenset[str] = frozenset({"'", '"'}) -BACKTICK_CHARACTER: str = "`" -DOLLAR_QUOTE_CHARACTER: str = "$" -SQL_ESCAPE_CHARACTER: str = "\\" -SINGLE_QUOTE_CHARACTER: str = "'" -OPENING_PAREN_CHARACTER: str = "(" -CLOSING_PAREN_CHARACTER: str = ")" -IDENTIFIER_EXTRA_CHARACTER: str = "_" -INTERPOLATION_NAME_EXTRA_CHARACTERS: frozenset[str] = frozenset({"_", ":", "."}) -SENTINEL_TEMPLATE: str = "__sqb_lint_{index}__" -EXPECTED_SENTINEL_OCCURRENCES: int = 1 TEMPLATE_INTERPOLATION_START: str = "${" -TEMPLATE_INTERPOLATION_END: str = "}" - -GENERATED_SQL_MESSAGE_TEMPLATE: str = "(in SQL generated by {token})" -GENERATED_SQL_MESSAGE_SUFFIX: str = "(in generated SQL)" - -LINE_FEED: str = "\n" -CARRIAGE_RETURN_LINE_FEED: str = "\r\n" -TAB_CHARACTER: str = "\t" RELATION_IDENTITY_TEMPLATE: str = "{kind}:{name}" RELATION_KIND_SOURCE: str = "source" RELATION_KIND_SEED: str = "seed" @@ -129,10 +40,3 @@ FIX_DEPENDENCIES_CHANGED: str = "The fix would change model dependencies" FIX_LINEAGE_CHANGED: str = "The fix would change column lineage" FIX_NEW_DIAGNOSTICS: str = "The fix would add compile diagnostics: " - -SQL_ARGUMENT_QUOTED_PARAMETER_PATTERN: re.Pattern[str] = re.compile( - r"(?[A-Za-z_][A-Za-z0-9_]*)'" -) -SQL_ARGUMENT_RAW_PARAMETER_PATTERN: re.Pattern[str] = re.compile( - r"(?[A-Za-z_][A-Za-z0-9_]*)(?![A-Za-z0-9_])(?!\s*\()" -) diff --git a/src/sqlbuild/lint/exceptions.py b/src/sqlbuild/lint/exceptions.py index bc3dd9fd03..30c96576e2 100644 --- a/src/sqlbuild/lint/exceptions.py +++ b/src/sqlbuild/lint/exceptions.py @@ -21,12 +21,6 @@ class NativeLintError(LintError): code: str = "L003" -class InterpolationRestorationError(LintError): - """A formatted SQL body no longer contains exactly one of each sentinel.""" - - code: str = "L002" - - class ProjectCompileError(LintError): """The project could not be compiled, so its SQL cannot be linted.""" diff --git a/src/sqlbuild/lint/main/collect_project_files.py b/src/sqlbuild/lint/main/collect_project_files.py deleted file mode 100644 index dde2fd282e..0000000000 --- a/src/sqlbuild/lint/main/collect_project_files.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Stable internal entry for SQL analysis project-file collection.""" - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint._helpers.project_files import collect_project_files as _collect_project_files - - -def collect_project_files( - *, - project_dir: Path, - selected_paths: frozenset[Path] | None, - discovered_inputs: DiscoveredProjectInputs | None = None, -) -> dict[Path, str]: - """Collect supported SQL-analysis inputs, sharing text discovery already holds.""" - - return _collect_project_files( - project_dir=project_dir, - selected_paths=selected_paths, - discovered_inputs=discovered_inputs, - ) diff --git a/src/sqlbuild/lint/main/has_fixture_typed_null_candidates.py b/src/sqlbuild/lint/main/has_fixture_typed_null_candidates.py deleted file mode 100644 index e65cbbd865..0000000000 --- a/src/sqlbuild/lint/main/has_fixture_typed_null_candidates.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public lint entry for fixture-null candidate discovery.""" - -from pathlib import Path - -from sqlbuild.lint._helpers.fixture_null_candidates import ( - has_fixture_typed_null_candidates as _has_fixture_typed_null_candidates, -) - - -def has_fixture_typed_null_candidates(*, project_dir: Path) -> bool: - """Return whether source files may contain safe fixture-null autofixes.""" - - return _has_fixture_typed_null_candidates(project_dir=project_dir) diff --git a/src/sqlbuild/lint/main/run_format.py b/src/sqlbuild/lint/main/run_format.py deleted file mode 100644 index 83674b12b5..0000000000 --- a/src/sqlbuild/lint/main/run_format.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Run the format pass over a SQLBuild project.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import ( - format_native_headers, - lint_native_headers, - prepare_native_header_cache, - reject_unparseable_header_rewrites, - safe_format_files, - scan_headers, -) -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.classes.fixture_null_autofix import FixtureNullAutofix -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.lint._helpers.fixes import finalize_fix_reports, persist_changes, plan_rule_fixes -from sqlbuild.lint._helpers.headers import drop_resolved_description_faults -from sqlbuild.lint._helpers.native_format import format_native_sql_bodies -from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations -from sqlbuild.lint._helpers.suppressions import apply_suppressions -from sqlbuild.lint.exceptions import LintError -from sqlbuild.lint.models import ( - FormatChange, - HeaderSpan, - LintConfig, - LintRunResult, - LintViolation, -) - - -def run_format( - *, - project_dir: Path, - config: LintConfig, - value_renderer: TypedSqlValueRenderer | None = None, - selected_paths: frozenset[Path] | None = None, - discovered_inputs: DiscoveredProjectInputs | None = None, - fixtures_only: bool = False, - write: bool = True, - fix: bool = False, -) -> LintRunResult: - """Format all DSL files in place and report the violations that remain.""" - - if discovered_inputs is not None and value_renderer is None: - raise LintError("fixture-null formatting requires the project adapter") - files: dict[Path, str] = collect_project_files( - project_dir=project_dir, selected_paths=selected_paths - ) - safe_files, newline_by_path = safe_format_files(files=files, config=config) - safe_files, rule_fixes, rule_faults = plan_rule_fixes( - project_dir=project_dir, - files=safe_files, - config=config, - adapter=value_renderer, - discovered_inputs=discovered_inputs, - enabled=fix, - ) - updated_contents, format_faults = _apply_fixes( - files=safe_files, - config=config, - project_dir=project_dir, - fixture_inputs=( - None - if discovered_inputs is None or value_renderer is None - else (discovered_inputs, value_renderer.sql_lexical_syntax) - ), - fixtures_only=fixtures_only, - ) - format_faults.extend(rule_faults) - updated_contents: dict[Path, str] = {**safe_files, **updated_contents} - declined_paths: set[Path] = {fault.file_path for fault in format_faults} - updated_contents: dict[Path, str] = { - path: contents for path, contents in updated_contents.items() if path not in declined_paths - } - changes: tuple[FormatChange, ...] = persist_changes( - files=files, updated=updated_contents, newlines=newline_by_path, write=write - ) - violations: list[LintViolation] = ( - [] - if fixtures_only - else _lint_final_contents( - files=files, - updated_contents=updated_contents, - config=config, - project_dir=project_dir, - value_renderer=value_renderer, - ) - ) - violations.extend(format_faults) - violations = drop_resolved_description_faults(violations=violations, project_dir=project_dir) - final_contents: dict[Path, str] = { - path: updated_contents.get(path, contents) for path, contents in files.items() - } - return LintRunResult( - files_checked=len(files), - violations=sort_violations( - apply_suppressions( - violations=violations, - contents_by_path=final_contents, - allow_suppressions=config.allow_suppressions, - ) - ), - formatted_files=tuple(change.file_path for change in changes), - format_changes=changes, - rule_fixes=finalize_fix_reports(reports=rule_fixes, declined_paths=declined_paths), - source_texts=final_contents, - ) - - -def _apply_fixes( - *, - files: dict[Path, str], - config: LintConfig, - project_dir: Path, - fixture_inputs: tuple[DiscoveredProjectInputs, SqlLexicalSyntax] | None, - fixtures_only: bool, -) -> tuple[dict[Path, str], list[LintViolation]]: - """Return contents after native header and supported SQL body fixes.""" - - updated: dict[Path, str] = ( - FixtureNullAutofix.apply( - files=files, - project_dir=project_dir, - discovered_inputs=fixture_inputs[0], - sql_lexical_syntax=fixture_inputs[1], - ) - if fixture_inputs is not None - else {} - ) - if fixtures_only: - return updated, [] - file_path: Path - contents: str - current_files: dict[Path, str] = { - file_path: updated.get(file_path, contents) for file_path, contents in files.items() - } - faults: list[LintViolation] = [] - _ = prepare_native_header_cache(files=current_files) - for file_path, contents in sorted(current_files.items()): - native_result: tuple[str, tuple[LintViolation, ...]] = format_native_headers( - contents=contents, - file_path=file_path, - config=config, - ) - faults.extend(native_result[1]) - if native_result[0] != contents: - updated[file_path] = native_result[0] - if not config.native_enabled: - return reject_unparseable_header_rewrites( - updated=updated, - config=config, - faults=faults, - ) - current_files = { - file_path: updated.get(file_path, contents) for file_path, contents in files.items() - } - native_formatted, native_faults = format_native_sql_bodies( - files=current_files, - config=config, - project_dir=project_dir, - ) - faults.extend(native_faults) - updated.update(native_formatted) - post_native_files: dict[Path, str] = { - file_path: updated.get(file_path, contents) for file_path, contents in files.items() - } - fixture_formatted: dict[Path, str] = ( - FixtureNullAutofix.apply( - files=post_native_files, - project_dir=project_dir, - discovered_inputs=fixture_inputs[0], - sql_lexical_syntax=fixture_inputs[1], - ) - if fixture_inputs is not None - else {} - ) - if fixture_formatted: - final_native, final_faults = format_native_sql_bodies( - files=fixture_formatted, - config=config, - project_dir=project_dir, - ) - faults.extend(final_faults) - updated.update(fixture_formatted) - updated.update(final_native) - return reject_unparseable_header_rewrites( - updated=updated, - config=config, - faults=faults, - ) - - -def _lint_final_contents( - *, - files: dict[Path, str], - updated_contents: dict[Path, str], - config: LintConfig, - project_dir: Path, - value_renderer: TypedSqlValueRenderer | None, -) -> list[LintViolation]: - """Return formatter-owned header diagnostics without enforcing configured Rules.""" - - del project_dir, value_renderer - violations: list[LintViolation] = [] - file_path: Path - contents: str - for file_path, contents in sorted(files.items()): - final_contents: str = updated_contents.get(file_path, contents) - headers: tuple[HeaderSpan, ...] = scan_headers(contents=final_contents) - violations.extend( - lint_native_headers( - contents=final_contents, - file_path=file_path, - headers=headers, - config=config, - ) - ) - return violations diff --git a/src/sqlbuild/lint/main/run_lint.py b/src/sqlbuild/lint/main/run_lint.py deleted file mode 100644 index 650dc1cd29..0000000000 --- a/src/sqlbuild/lint/main/run_lint.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Run the lint pass over a SQLBuild project.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild._native import lint_native_headers, scan_headers -from sqlbuild.compiler.compile.constants import MODEL_DIRECTORY_NAME -from sqlbuild.compiler.compile.models import CompiledSqlExpansion, SqlExpansionContext -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.constants import ( - DECLARATION_GROUP_DIRECTORY, - INHERITED_DECLARATION_DIRECTORIES, - LOCAL_DECLARATION_DIRECTORIES, -) -from sqlbuild.lint._helpers.expansion import ( - build_lint_expansion_context, - expand_file_bodies, - prepare_expanded_files, -) -from sqlbuild.lint._helpers.native_sql import run_native_sql_lint -from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations -from sqlbuild.lint._helpers.suppressions import apply_suppressions -from sqlbuild.lint.models import ( - ExpandedLintFile, - HeaderSpan, - LintBody, - LintConfig, - LintRunResult, - LintViolation, - PreparedLintFiles, -) - - -def run_lint( - *, - project_dir: Path, - config: LintConfig, - value_renderer: TypedSqlValueRenderer | None = None, - selected_paths: frozenset[Path] | None = None, - discovered_inputs: DiscoveredProjectInputs | None = None, - dynamic_output_paths: frozenset[Path] = frozenset(), - compiled_expansions: dict[Path, CompiledSqlExpansion] | None = None, - expansion_context: SqlExpansionContext | None = None, - source_files: dict[Path, str] | None = None, - skip_unexpandable: bool = False, -) -> LintRunResult: - """Lint all DSL files without modifying anything; optionally skip unexpandable SQL.""" - - files: dict[Path, str] = collect_project_files( - project_dir=project_dir, - selected_paths=selected_paths, - source_files=source_files, - discovered_inputs=discovered_inputs, - ) - violations: list[LintViolation] = [] - context: SqlExpansionContext | None = expansion_context or _expansion_context( - project_dir=project_dir, - native_enabled=config.native_enabled, - value_renderer=value_renderer, - discovered_inputs=discovered_inputs, - ) - expanded_files: list[ExpandedLintFile] = [] - failure: Exception | None = None - file_path: Path - contents: str - for file_path, contents in sorted(files.items()): - try: - header_violations: tuple[LintViolation, ...] - expanded: ExpandedLintFile | None - header_violations, expanded = _scan_file( - file_path=file_path, - contents=contents, - project_dir=project_dir, - config=config, - context=context, - dynamic_output_paths=dynamic_output_paths, - compiled_expansions=compiled_expansions, - ) - except Exception as error: - failure = error - break - violations.extend(header_violations) - if expanded is None: - continue - expanded_files.append(expanded) - if expanded.failure is not None and not skip_unexpandable: - break - prepared: PreparedLintFiles = prepare_expanded_files( - expanded_files=tuple(expanded_files), - dialect=config.dialect, - skip_unexpandable=skip_unexpandable, - ) - if failure is not None: - raise failure - bodies: tuple[LintBody, ...] = prepared.bodies - unexpandable: dict[Path, str] = prepared.unexpandable - - if bodies and config.native_enabled: - native_violations: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( - bodies=bodies, - contents_by_path=files, - config=config, - ) - for entries in native_violations.values(): - violations.extend(entries) - return LintRunResult( - files_checked=len(files), - violations=sort_violations( - apply_suppressions( - violations=violations, - contents_by_path=files, - allow_suppressions=config.allow_suppressions, - ) - ), - formatted_files=(), - source_texts=files, - unexpandable=unexpandable, - ) - - -def _expansion_context( - *, - project_dir: Path, - native_enabled: bool, - value_renderer: TypedSqlValueRenderer | None, - discovered_inputs: DiscoveredProjectInputs | None, -) -> SqlExpansionContext | None: - if not native_enabled: - return None - return build_lint_expansion_context( - project_dir=project_dir, - value_renderer=value_renderer, - discovered_inputs=discovered_inputs, - ) - - -def _scan_file( - *, - file_path: Path, - contents: str, - project_dir: Path, - config: LintConfig, - context: SqlExpansionContext | None, - dynamic_output_paths: frozenset[Path], - compiled_expansions: dict[Path, CompiledSqlExpansion] | None, -) -> tuple[tuple[LintViolation, ...], ExpandedLintFile | None]: - """Header violations of one file and, when expanding, its bodies awaiting preparation.""" - - relative_path: Path = file_path.relative_to(project_dir) - relative_parts: tuple[str, ...] = relative_path.parts - declaration_directories: frozenset[str] = ( - INHERITED_DECLARATION_DIRECTORIES - | LOCAL_DECLARATION_DIRECTORIES - | {DECLARATION_GROUP_DIRECTORY} - ) - headers: tuple[HeaderSpan, ...] = scan_headers( - contents=contents, - first_only=( - relative_parts[:1] == (MODEL_DIRECTORY_NAME,) - and not declaration_directories.intersection(relative_parts[1:-1]) - ), - ) - header_violations: tuple[LintViolation, ...] = ( - tuple( - lint_native_headers( - contents=contents, - file_path=file_path, - headers=headers, - config=config, - ) - ) - if config.header_rules_enabled - else () - ) - if context is None: - return header_violations, None - return header_violations, expand_file_bodies( - file_path=file_path, - contents=contents, - headers=headers, - context=context, - dialect=config.dialect, - project_dir=project_dir, - relative_path=relative_path, - dynamic_output_paths=dynamic_output_paths, - compiled_expansions=compiled_expansions, - ) diff --git a/src/sqlbuild/lint/main/scan_fixture_typed_null_candidates.py b/src/sqlbuild/lint/main/scan_fixture_typed_null_candidates.py deleted file mode 100644 index c5529ddeb6..0000000000 --- a/src/sqlbuild/lint/main/scan_fixture_typed_null_candidates.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public lint entry for selected fixture-null candidate discovery.""" - -from pathlib import Path - -from sqlbuild.lint._helpers.fixture_null_candidates import ( - scan_fixture_typed_null_candidates as _scan_fixture_typed_null_candidates, -) -from sqlbuild.lint.models import FixtureNullCandidateScan - - -def scan_fixture_typed_null_candidates( - *, project_dir: Path, selected_paths: frozenset[Path] | None = None -) -> FixtureNullCandidateScan: - """Return selected SQL-test fixture-null candidates and referenced model names.""" - - return _scan_fixture_typed_null_candidates( - project_dir=project_dir, - selected_paths=selected_paths, - ) diff --git a/src/sqlbuild/lint/main/scan_interpolation_sites.py b/src/sqlbuild/lint/main/scan_interpolation_sites.py deleted file mode 100644 index 0540c6608a..0000000000 --- a/src/sqlbuild/lint/main/scan_interpolation_sites.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public lexical scan for SQLBuild interpolation sites in authored SQL.""" - -from __future__ import annotations - -from sqlbuild.lint._helpers.sqlbuild_tokens import neutralize_interpolation -from sqlbuild.lint.models import InterpolationSite - - -def scan_interpolation_sites(*, body: str, dialect: str) -> tuple[InterpolationSite, ...]: - """Return every reference call, macro, and template site outside comments and strings.""" - - return neutralize_interpolation(body=body, dialect=dialect)[1] diff --git a/src/sqlbuild/lint/models.py b/src/sqlbuild/lint/models.py index c036768590..53e54892c7 100644 --- a/src/sqlbuild/lint/models.py +++ b/src/sqlbuild/lint/models.py @@ -108,14 +108,6 @@ class PreparedLintFiles: unexpandable: dict[Path, str] -@dataclass(frozen=True) -class FixtureNullCandidateScan: - """Cheaply discovered SQL-test fixtures that may contain removable typed nulls.""" - - paths: frozenset[Path] = field(default_factory=frozenset) - model_names: frozenset[str] = field(default_factory=frozenset) - - @dataclass(frozen=True) class LintEdit: """One deterministic authored-source edit proposed by a lint rule.""" @@ -156,15 +148,6 @@ class RuleFixResult: reason: str -@dataclass(frozen=True) -class FormatChange: - """One deterministic file-formatting change.""" - - file_path: Path - before: str - after: str - - @dataclass(frozen=True) class LintConfig: """Resolved lint and format configuration for one run.""" @@ -191,8 +174,6 @@ class LintRunResult: files_checked: int violations: tuple[LintViolation, ...] formatted_files: tuple[Path, ...] - format_changes: tuple[FormatChange, ...] = () - rule_fixes: tuple[RuleFixResult, ...] = () source_texts: Mapping[Path, str] = field(default_factory=dict, repr=False, compare=False) unexpandable: Mapping[Path, str] = field(default_factory=dict) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py index 8ae88128f3..9e65027fea 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rule_threads.py @@ -7,6 +7,7 @@ from pathlib import Path from threading import Event +from sqlbuild._native import run_lint from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlExpansion, @@ -18,7 +19,6 @@ from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.classes.stop_context import LintStopContext from sqlbuild.lint.main.build_expansion_context import build_expansion_context -from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult from sqlbuild.rule_engine._helpers.engine.custom_dispatch import evaluate_native, start_custom_rules from sqlbuild.rule_engine._helpers.run.rules import ( diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 9d08af870c..e4bed6d647 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -8,10 +8,12 @@ from pathlib import Path from sqlbuild._native import ( + collect_lint_project_files, encode_sql_rule_findings, load_sql_rule_cache, lookup_sql_rule_cache, project_with, + run_lint, selected_sql_rule_codes, sql_file_rule_key, sql_model_rule_keys, @@ -43,10 +45,8 @@ ) from sqlbuild.lint.exceptions import NativeLintError from sqlbuild.lint.main.build_expansion_context import build_expansion_context -from sqlbuild.lint.main.collect_project_files import collect_project_files from sqlbuild.lint.main.compiled_relation_keys import compiled_relation_keys from sqlbuild.lint.main.declared_relation_keys import declared_relation_keys -from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation from sqlbuild.lint.types import RelationKeys from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue, select_rules @@ -246,9 +246,7 @@ def _prepare_sql_rule_inputs( return _SqlRuleInputs( identities=identities, project_files=( - collect_project_files( - project_dir=project_dir, selected_paths=None, discovered_inputs=discovered_inputs - ) + collect_lint_project_files(project_dir=project_dir, selected_paths=None) if identities and collect_files else None ), @@ -470,11 +468,7 @@ def _run_sql_rules( project_files = ( prepared_inputs.project_files if prepared_inputs is not None and prepared_inputs.project_files is not None - else collect_project_files( - project_dir=project_dir, - selected_paths=None, - discovered_inputs=discovered_inputs, - ) + else collect_lint_project_files(project_dir=project_dir, selected_paths=None) ) lint_key: str = sql_rule_lint_key(config=lint_config) file_keys: dict[str, str | None] = { diff --git a/src/sqlbuild/spec/contracts/main/parse_scenario_run_namespace.py b/src/sqlbuild/spec/contracts/main/parse_scenario_run_namespace.py deleted file mode 100644 index 422794b3ae..0000000000 --- a/src/sqlbuild/spec/contracts/main/parse_scenario_run_namespace.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Validation for the optional scenario artifact run namespace.""" - -from __future__ import annotations - -import re - -from sqlbuild.spec.contracts.exceptions import SpecConfigError - - -def parse_scenario_run_namespace(value: object) -> str: - """Return a valid namespace or reject it without normalization.""" - - if not isinstance(value, str) or re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", value) is None: - raise SpecConfigError( - "scenario.run_namespace must be 1–128 characters: " - "ASCII letters, digits, '-', '_' or '.'" - ) - return value diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index dec0bf7ea8..2cd5640d92 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -63,7 +63,7 @@ "dense_models_10000_all_rules", 10000, 77.0, - 13 * _GIB // 4, + 15 * _GIB // 4, "f2a574af6e8068a0634fa3c078b168dff9a475ada5d71502482ad2926c90e1c3", ), ), diff --git a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py index 91cb48d23d..f2845910f7 100644 --- a/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py +++ b/tests/integration/src/sqlbuild/cli/commands/_helpers/runtime/test_command_invocation.py @@ -9,7 +9,6 @@ import pytest from sqlbuild.cli.commands import models as command_models -from sqlbuild.cli.commands._helpers.audit.invocation import resolve_audit_invocation from sqlbuild.cli.commands._helpers.runtime.command_invocation import ( resolve_command_invocation, resolve_reported_command_invocation, @@ -28,7 +27,10 @@ ( CommandInvocationTestCase( description="audit json output reports progress on stderr", - resolve=resolve_audit_invocation, + resolve=partial( + resolve_reported_command_invocation, + invocation_type=command_models.AuditInvocation, + ), request_type=command_models.AuditCommandRequest, json_output=True, expected_invocation_type_name="AuditInvocation", diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index cddca28fcd..4a59de8ac9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -16,16 +16,16 @@ from sqlbuild._native import ( discover_project_inputs, format_load_execution_json, + run_audit_command, + run_build_command, run_load_command, + run_scenario_test_command, run_test_command, select_load_entries, ) from sqlbuild.adapter.contract.types import CursorKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.main.execution._audit import run_audit -from sqlbuild.cli.commands.main.execution._build import run_build -from sqlbuild.cli.commands.main.execution._scenario import run_scenario from sqlbuild.cli.commands.main.project._plan import run_plan from sqlbuild.cli.commands.models import ( AuditCommandRequest, @@ -809,8 +809,8 @@ def test_given_selected_model_references_managed_source_when_build_or_run_then_a ) -> None: write_repo_files(tmp_path, _BUILD_RUN_AUTO_LOAD_PROJECT_FILES) - exit_code: int = run_build( - BuildCommandRequest(project_dir=tmp_path, no_color=True, select=("stg_orders",)) + exit_code: int = run_build_command( + request=BuildCommandRequest(project_dir=tmp_path, no_color=True, select=("stg_orders",)) ) captured: CaptureResult[str] = capsys.readouterr() @@ -856,8 +856,8 @@ def test_given_self_managed_source_when_running_build_then_auto_loads_before_mod ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_build( - BuildCommandRequest(project_dir=tmp_path, no_color=True, select=("stg_orders",)) + exit_code: int = run_build_command( + request=BuildCommandRequest(project_dir=tmp_path, no_color=True, select=("stg_orders",)) ) captured: CaptureResult[str] = capsys.readouterr() @@ -978,8 +978,8 @@ def test_given_managed_source_environment_when_building_or_running_then_reads_co finally: setup_connection.close() - exit_code: int = run_build( - BuildCommandRequest( + exit_code: int = run_build_command( + request=BuildCommandRequest( project_dir=tmp_path, no_color=True, select=("stg_orders",), @@ -1229,9 +1229,11 @@ def test_given_no_managed_source_read_ambiguity_when_building_then_source_deferr setup_connection.close() command_runners: dict[str, Callable[..., int]] = { - "audit": lambda **kwargs: run_audit(AuditCommandRequest(**kwargs)), - "build": lambda **kwargs: run_build(BuildCommandRequest(**kwargs)), - "scenario": lambda **kwargs: run_scenario(ScenarioTestCommandRequest(**kwargs)), + "audit": lambda **kwargs: run_audit_command(request=AuditCommandRequest(**kwargs)), + "build": lambda **kwargs: run_build_command(request=BuildCommandRequest(**kwargs)), + "scenario": lambda **kwargs: run_scenario_test_command( + request=ScenarioTestCommandRequest(**kwargs) + ), "test": lambda **kwargs: run_test_command(request=TestCommandRequest(**kwargs)), } exit_code: int = command_runners[test_case.command]( @@ -1360,8 +1362,8 @@ def test_given_source_deferral_when_writing_artifacts_then_sql_uses_deferred_rel setup_connection.close() command_runners: dict[str, Callable[..., int]] = { - "audit": lambda **kwargs: run_audit(AuditCommandRequest(**kwargs)), - "build": lambda **kwargs: run_build(BuildCommandRequest(**kwargs)), + "audit": lambda **kwargs: run_audit_command(request=AuditCommandRequest(**kwargs)), + "build": lambda **kwargs: run_build_command(request=BuildCommandRequest(**kwargs)), } exit_code: int = command_runners[test_case.command]( project_dir=tmp_path, @@ -1483,8 +1485,8 @@ def test_given_build_or_run_load_flags_when_running_then_applies_loader_control( finally: setup_connection.close() - exit_code: int = run_build( - BuildCommandRequest( + exit_code: int = run_build_command( + request=BuildCommandRequest( project_dir=tmp_path, no_color=True, select=(test_case.args[test_case.args.index("--select") + 1],), @@ -1567,8 +1569,8 @@ def test_given_downstream_selection_when_running_build_then_loads_only_direct_se finally: setup_connection.close() - exit_code: int = run_build( - BuildCommandRequest( + exit_code: int = run_build_command( + request=BuildCommandRequest( project_dir=tmp_path, no_color=True, select=(test_case.args[test_case.args.index("--select") + 1],), @@ -1614,8 +1616,8 @@ def test_given_source_loader_fails_when_running_build_then_downstream_model_is_n ) -> None: write_repo_files(tmp_path, test_case.project_files) - exit_code: int = run_build( - BuildCommandRequest(project_dir=tmp_path, no_color=True, select=("stg_orders",)) + exit_code: int = run_build_command( + request=BuildCommandRequest(project_dir=tmp_path, no_color=True, select=("stg_orders",)) ) captured: CaptureResult[str] = capsys.readouterr() @@ -1662,8 +1664,8 @@ def test_given_build_auto_loads_source_when_json_output_then_includes_source_ass write_repo_files(tmp_path, test_case.project_files) json_output_path: Path = tmp_path / "target" / "build.json" - exit_code: int = run_build( - BuildCommandRequest( + exit_code: int = run_build_command( + request=BuildCommandRequest( project_dir=tmp_path, no_color=True, select=("stg_orders",), @@ -1842,8 +1844,8 @@ def test_given_build_skips_loader_when_manifest_is_written_then_marks_source_aut finally: setup_connection.close() - exit_code: int = run_build( - BuildCommandRequest( + exit_code: int = run_build_command( + request=BuildCommandRequest( project_dir=tmp_path, no_color=True, select=("stg_orders",), diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 68aecdd422..139135b6d9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -937,22 +937,15 @@ def test_given_cold_non_model_sql_finding_when_compiling_then_rules_reuse_early_ b"SELECT 1\r\n" ) collections: list[frozenset[Path] | None] = [] - original_collect: Callable[..., dict[Path, str]] = rules_module.collect_project_files + original_collect: Callable[..., dict[Path, str]] = rules_module.collect_lint_project_files def counting_collect( - *, - project_dir: Path, - selected_paths: frozenset[Path] | None, - discovered_inputs: DiscoveredProjectInputs | None = None, + *, project_dir: Path, selected_paths: frozenset[Path] | None ) -> dict[Path, str]: collections.append(selected_paths) - return original_collect( - project_dir=project_dir, - selected_paths=selected_paths, - discovered_inputs=discovered_inputs, - ) + return original_collect(project_dir=project_dir, selected_paths=selected_paths) - monkeypatch.setattr(rules_module, "collect_project_files", counting_collect) + monkeypatch.setattr(rules_module, "collect_lint_project_files", counting_collect) cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) cold_collections: int = len(collections) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/__init__.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/_test_types.py deleted file mode 100644 index 0ff9ee1674..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/_test_types.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.cli.entry.models import ( - SelectorFileSummary, -) - - -@dataclass(frozen=True) -class BuildRunContextTestCase: - description: str - connection_config: dict[str, object] - effective_vars: dict[str, object] - selector_files: tuple[SelectorFileSummary, ...] - full_refresh: bool - expected_fragments: tuple[str, ...] - expected_absent_fragments: tuple[str, ...] - selected_source_count: int = 0 - selected_python_count: int = 0 - total_source_count: int = 0 - total_task_count: int = 0 - - -@dataclass(frozen=True) -class BuildPhaseTimingsTestCase: - description: str - expected_output: str - - -@dataclass(frozen=True) -class CostFailureTimingTestCase: - description: str - clock_values: tuple[float, ...] - expected_cost_seconds: float - expected_error_message: str - - -@dataclass(frozen=True) -class ExecutionProjectionFailureTestCase: - description: str - build_status: str - expected_exit_code: int - expected_document_status: str - expected_completion_message: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_cost_failure_timings.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_cost_failure_timings.py deleted file mode 100644 index 9605e25ae6..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_cost_failure_timings.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Direct cost failure timing retention tests.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from unittest.mock import Mock - -import pytest - -from sqlbuild.cli.commands._helpers.build_execution import completion, phase_timings -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker -from tests.unit.src.sqlbuild.cli.commands._helpers.build_execution._test_types import ( - CostFailureTimingTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - CostFailureTimingTestCase( - description="completed work retains cost elapsed when finalization fails", - clock_values=(1.0, 4.0), - expected_cost_seconds=3.0, - expected_error_message="completed cost failure", - ) - ], - ids=lambda case: case.description, -) -def test_given_completed_work_cost_failure_when_finalizing_then_elapsed_is_retained( - test_case: CostFailureTimingTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monotonic: Mock = Mock(side_effect=test_case.clock_values) - tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker(monotonic=lambda: 0.0) - monkeypatch.setattr(completion.time, "monotonic", monotonic) - monkeypatch.setattr(phase_timings.time, "monotonic", monotonic) - monkeypatch.setattr(completion, "resolve_build_exit_code", Mock(return_value=0)) - monkeypatch.setattr(completion, "write_build_runtime_targets", Mock()) - monkeypatch.setattr( - completion, - "finalize_build_cost", - Mock(side_effect=RuntimeError(test_case.expected_error_message)), - ) - - with tracker.scope(), pytest.raises(RuntimeError, match=test_case.expected_error_message): - completion.complete_direct_build( - request=Mock(), - invocation=Mock(), - pipeline_result=Mock(), - preparation=Mock(), - outcome=Mock(), - check_results=(), - build_started_at=datetime.now(UTC), - command_started_at=0.0, - ) - - assert tracker.cost_collection_seconds == test_case.expected_cost_seconds - - -@pytest.mark.parametrize( - "test_case", - [ - CostFailureTimingTestCase( - description="no-work build retains cost elapsed when finalization fails", - clock_values=(2.0, 7.0), - expected_cost_seconds=5.0, - expected_error_message="no-work cost failure", - ) - ], - ids=lambda case: case.description, -) -def test_given_no_work_cost_failure_when_finalizing_then_elapsed_is_retained( - test_case: CostFailureTimingTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monotonic: Mock = Mock(side_effect=test_case.clock_values) - tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker(monotonic=lambda: 0.0) - monkeypatch.setattr(phase_timings.time, "monotonic", monotonic) - monkeypatch.setattr( - phase_timings, - "finalize_no_work_build_if_needed", - Mock(side_effect=RuntimeError(test_case.expected_error_message)), - ) - - with tracker.scope(), pytest.raises(RuntimeError, match=test_case.expected_error_message): - phase_timings.finalize_no_work_with_timings( - request=Mock(), - invocation=Mock(), - pipeline_result=Mock(), - command_started_at=0.0, - ) - - assert tracker.cost_collection_seconds == test_case.expected_cost_seconds diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_output_degradation.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_output_degradation.py deleted file mode 100644 index 337ce0949e..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_output_degradation.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Post-execution output degradation boundary tests.""" - -from __future__ import annotations - -import json -from io import StringIO -from pathlib import Path -from types import SimpleNamespace -from typing import cast -from unittest.mock import Mock - -import pytest - -from sqlbuild.cli.commands._helpers.build_execution import outputs -from sqlbuild.cli.commands.models import ( - BuildCommandRequest, - BuildExecutionPreparation, - BuildInvocation, - BuildRunOutcome, -) -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.cost.models import CostRunRecord -from sqlbuild.executor.build.models import BuildExecutionResult -from sqlbuild.executor.build.types import BuildStatus -from tests.unit.src.sqlbuild.cli.commands._helpers.build_execution._test_types import ( - ExecutionProjectionFailureTestCase, -) -from tests.unit.src.sqlbuild.cli.output.main.plan.helpers import build_plan_output - - -@pytest.mark.parametrize( - "test_case", - ( - ExecutionProjectionFailureTestCase( - description="successful execution projection failure", - build_status="success", - expected_exit_code=0, - expected_document_status="success", - expected_completion_message="warehouse execution completed successfully", - ), - ExecutionProjectionFailureTestCase( - description="failed execution projection failure", - build_status="failed", - expected_exit_code=1, - expected_document_status="failed", - expected_completion_message="execution result remains authoritative", - ), - ), - ids=lambda case: case.description, -) -def test_given_completed_execution_when_json_projection_fails_then_exit_status_remains_authoritative( - test_case: ExecutionProjectionFailureTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - output_path: Path = tmp_path / "execution.json" - outcome: BuildRunOutcome = BuildRunOutcome( - result=BuildExecutionResult(status=BuildStatus(test_case.build_status)), - python_results=(), - ) - monkeypatch.setattr(outputs, "format_build_footer", Mock(return_value="completed")) - monkeypatch.setattr( - outputs, "cost_output_payload", Mock(return_value={"unserializable": object()}) - ) - - outputs.write_build_completion_output( - request=cast( - BuildCommandRequest, - SimpleNamespace(json_output=False, json_output_path=output_path), - ), - invocation=cast( - BuildInvocation, - SimpleNamespace(use_color=False, progress_stream=StringIO()), - ), - pipeline_result=cast( - CompilePipelineResult, - SimpleNamespace(plan_output=Mock(), project=Mock()), - ), - preparation=cast( - BuildExecutionPreparation, - SimpleNamespace(callbacks=SimpleNamespace(elapsed=1.0)), - ), - outcome=outcome, - check_results=(), - cost_record=cast(CostRunRecord, Mock()), - ) - - payload: dict[str, object] = json.loads(output_path.read_text(encoding="utf-8")) - assert outputs.resolve_build_exit_code(outcome=outcome, check_results=()) == ( - test_case.expected_exit_code - ) - assert payload["status"] == test_case.expected_document_status - assert payload["projection_degraded"] is True - assert test_case.expected_completion_message in capsys.readouterr().err - - -@pytest.mark.parametrize( - "test_case", - [ - ExecutionProjectionFailureTestCase( - description="audit result persistence failure", - build_status="success", - expected_exit_code=0, - expected_document_status="success", - expected_completion_message="unused", - ) - ], - ids=lambda case: case.description, -) -def test_given_audit_history_write_failure_when_formatting_json_then_reports_bounded_degradation( - test_case: ExecutionProjectionFailureTestCase, -) -> None: - result: BuildExecutionResult = BuildExecutionResult( - status=BuildStatus(test_case.build_status), - audit_result_projection_attempted_count=3, - audit_result_projection_written_count=0, - audit_result_projection_failed_count=3, - ) - payload: dict[str, object] = json.loads( - outputs.execution_json_payload_with_degradation( - result=result, - plan=build_plan_output(), - python_node_results=(), - python_check_results=(), - command="build", - run_id="run", - cost_record=None, - execution_succeeded=True, - ) - ) - - assert payload["status"] == test_case.expected_document_status - assert payload["projection_degraded"] is True - assert payload["audit_result_projection"] == { - "attempted_count": 3, - "written_count": 0, - "failed_count": 3, - } - assert payload["projection_degradation_reasons"] == [ - { - "reason": "audit_result_projection_failure", - "attempted_count": 3, - "written_count": 0, - "failed_count": 3, - } - ] - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_phase_timings.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_phase_timings.py deleted file mode 100644 index 2d6a73ce6c..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_phase_timings.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Tests for final verbose build phase timing output.""" - -from __future__ import annotations - -from io import StringIO - -import pytest - -from sqlbuild.cli.commands._helpers.build_execution.phase_timings import ( - write_build_phase_timings, -) -from sqlbuild.cli.commands.models import BuildPhaseTimings -from tests.unit.src.sqlbuild.cli.commands._helpers.build_execution._test_types import ( - BuildPhaseTimingsTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - BuildPhaseTimingsTestCase( - description="renders every available monotonic phase duration", - expected_output=( - "\nPhase timings\n" - " compile 1.25s\n" - " planning 2.50s\n" - " connection preparation 3.75s\n" - " schema preparation 0.50s\n" - " execution 4.25s\n" - " cost collection 1.00s\n" - " total 13.25s\n" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_typed_phase_timings_when_writing_then_renders_available_durations( - test_case: BuildPhaseTimingsTestCase, -) -> None: - stream: StringIO = StringIO() - - write_build_phase_timings( - stream=stream, - timings=BuildPhaseTimings( - compile_seconds=1.25, - planning_seconds=2.5, - connection_preparation_seconds=3.75, - schema_preparation_seconds=0.5, - execution_seconds=4.25, - cost_collection_seconds=1.0, - total_seconds=13.25, - ), - use_color=False, - ) - - assert stream.getvalue() == test_case.expected_output diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_run_context.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_run_context.py deleted file mode 100644 index 13b9255c98..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_execution/test_run_context.py +++ /dev/null @@ -1,176 +0,0 @@ -"""Tests for resolved verbose build-start context output.""" - -from __future__ import annotations - -from io import StringIO -from pathlib import Path -from types import SimpleNamespace -from typing import cast - -import pytest - -from sqlbuild.cli.commands._helpers.build_execution.run_context import ( - write_build_run_context, -) -from sqlbuild.cli.commands.models import ( - BuildRunContext, -) -from sqlbuild.cli.entry.models import ( - SelectorFileSummary, -) -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledSourceView, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredTaskFunction -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.models import ( - ModelPlanEntry, - PlanOutput, - SeedPlanEntry, - SourceLoadPlanEntry, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig -from tests.unit.src.sqlbuild.cli.commands._helpers.build_execution._test_types import ( - BuildRunContextTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - BuildRunContextTestCase( - description="resolved placement dates and selector summary", - connection_config={ - "warehouse": "PROD_WH__LARGE", - "password": "never-print-password", - "token": "never-print-token", - "authenticator": "never-print-authenticator", - }, - effective_vars={ - "start_date": "1970-01-01", - "end_date": "2030-12-31", - "api_secret": "never-print-var", - }, - selector_files=( - SelectorFileSummary( - path=Path("/tmp/sqlbuild-dagster-select.txt"), selector_count=386 - ), - ), - full_refresh=True, - expected_fragments=( - "Execution", - "command sqb build", - "run_id 20260830T175430Z_fb5fb40a8b81", - "target test", - "database ANALYTICS", - "schema DEV_DAGSTER_DEV", - "warehouse PROD_WH__LARGE", - "concurrency 5 configured limit", - "full_refresh true", - "selected 1 of 2 build resources", - "date vars 1970-01-01 to 2030-12-31", - "Selection files", - "selector_file /tmp/sqlbuild-dagster-select.txt (386 selectors)", - ), - expected_absent_fragments=( - "never-print-password", - "never-print-token", - "never-print-authenticator", - "never-print-var", - ), - ), - BuildRunContextTestCase( - description="missing optional placement and unsafe date values", - connection_config={"warehouse": "safe\nforged"}, - effective_vars={"start_date": "never-print-date-secret", "end_date": "2030-12-31"}, - selector_files=(SelectorFileSummary(path=Path("unsafe\npath"), selector_count=1),), - full_refresh=False, - expected_fragments=( - "warehouse safe\\x0aforged", - "full_refresh false", - "date vars end_date=2030-12-31", - "unsafe\\x0apath", - ), - expected_absent_fragments=("never-print-date-secret", "safe\nforged", "unsafe\npath"), - ), - BuildRunContextTestCase( - description="python and source resources use the full build graph count", - connection_config={}, - effective_vars={}, - selector_files=(), - full_refresh=False, - expected_fragments=("selected 3 of 4 build resources",), - expected_absent_fragments=(), - selected_source_count=1, - selected_python_count=1, - total_source_count=1, - total_task_count=1, - ), - ], - ids=lambda case: case.description, -) -def test_given_resolved_build_context_when_writing_then_renders_safe_verbose_summary( - test_case: BuildRunContextTestCase, -) -> None: - project: CompiledProjectView = CompiledProject( - run_id="20260830T175430Z_fb5fb40a8b81", - effective_target_name="test", - effective_connection={}, - effective_vars=test_case.effective_vars, - effective_target_database="ANALYTICS", - effective_target_schema="DEV_DAGSTER_DEV", - models=cast(tuple[CompiledModelView, ...], (object(), object())), - sources=cast( - tuple[CompiledSourceView, ...], - tuple( - SimpleNamespace(source_entry=SimpleNamespace(loader="load_orders")) - for _ in range(test_case.total_source_count) - ), - ), - ) - plan: PlanOutput = PlanOutput( - model_entries=cast(tuple[ModelPlanEntry, ...], (object(),)), - seed_entries=cast(tuple[SeedPlanEntry, ...], ()), - source_load_entries=cast( - tuple[SourceLoadPlanEntry, ...], - tuple(object() for _ in range(test_case.selected_source_count)), - ), - ) - stream: StringIO = StringIO() - - write_build_run_context( - stream=stream, - context=BuildRunContext( - command="sqb build", - project=project, - plan=plan, - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig(name="test", adapter="duckdb"), - local_config=LocalConfig(), - task_functions=cast( - tuple[DiscoveredTaskFunction, ...], - tuple(object() for _ in range(test_case.total_task_count)), - ), - ), - python_plan_entries=cast( - tuple[PythonPlanEntry, ...], - tuple(object() for _ in range(test_case.selected_python_count)), - ), - connection_config=test_case.connection_config, - concurrency=5, - full_refresh=test_case.full_refresh, - selector_files=test_case.selector_files, - ), - use_color=False, - ) - - output: str = stream.getvalue() - expected_fragment: str - for expected_fragment in test_case.expected_fragments: - assert expected_fragment in output - absent_fragment: str - for absent_fragment in test_case.expected_absent_fragments: - assert absent_fragment not in output diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py index 52a57aba83..19d4a9294c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/_test_types.py @@ -38,15 +38,6 @@ class ModelExecutionLimitTestCase: expected_code: str | None = None -@dataclass(frozen=True) -class UnsupportedDurationLimitTestCase: - description: str - max_duration: str - remediation: str - expected_code: str - expected_error_fragment: str - - @dataclass(frozen=True) class RetentionDecreasePolicyTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py index a56e607ba3..aec230981d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/helpers.py @@ -4,6 +4,14 @@ from sqlbuild.adapter.contract.models import RetentionRequest from sqlbuild.adapter.contract.types import RetentionScope +from sqlbuild.cli.commands.constants import ( + EXECUTION_LIMIT_BUILD_NOTE, + MISSING_ORIGIN_BUILD_HELP, + RETENTION_DECREASE_BUILD_HELP, + SNAPSHOT_FULL_REFRESH_BUILD_HELP, + TABLE_TYPE_DOWNGRADE_BUILD_HELP, +) +from sqlbuild.cli.commands.models import PlanSafetyGates from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner.models import ( @@ -96,3 +104,22 @@ def build_retention_entry( statements=("SELECT 1",), decrease_policy=policy, ) + + +def build_policy_gates( + *, + allow_snapshot_full_refresh: bool = False, + allow_table_type_downgrade: bool = False, + allow_retention_decrease: bool = False, +) -> PlanSafetyGates: + return PlanSafetyGates( + allow_missing_migration_origin=False, + allow_snapshot_full_refresh=allow_snapshot_full_refresh, + allow_table_type_downgrade=allow_table_type_downgrade, + allow_retention_decrease=allow_retention_decrease, + missing_origin_help=MISSING_ORIGIN_BUILD_HELP, + snapshot_full_refresh_help=SNAPSHOT_FULL_REFRESH_BUILD_HELP, + table_type_downgrade_help=TABLE_TYPE_DOWNGRADE_BUILD_HELP, + retention_decrease_help=RETENTION_DECREASE_BUILD_HELP, + execution_limit_note=EXECUTION_LIMIT_BUILD_NOTE, + ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py index 12f300399f..9084ca978e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_execution_limits.py @@ -1,18 +1,11 @@ -from pathlib import Path - import pytest -from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( - enforce_model_execution_limit, -) -from sqlbuild.cli.commands._helpers.build_planning.invocation import resolve_build_invocation +from sqlbuild._native import enforce_model_execution_limit from sqlbuild.cli.commands.constants import EXECUTION_LIMIT_BUILD_NOTE from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import BuildCommandRequest from sqlbuild.spec.contracts.models import ExecutionLimitsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( ModelExecutionLimitTestCase, - UnsupportedDurationLimitTestCase, ) @@ -75,83 +68,3 @@ def test_given_model_count_above_limit_when_enforcing_then_error_has_custom_reme assert "Selected models: 26" in raised.value.message assert "Maximum models: 25" in raised.value.message assert "No warehouse changes were made." in raised.value.message - - -@pytest.mark.parametrize( - "test_case", - [ - UnsupportedDurationLimitTestCase( - description="DuckDB target cannot guarantee statement cancellation", - max_duration="30m", - remediation="Use an adapter with safe statement cancellation.", - expected_code="C415", - expected_error_fragment="cannot cancel active statements safely", - ) - ], - ids=lambda case: case.description, -) -def test_given_duration_limit_for_unsupported_adapter_when_resolving_then_configuration_is_rejected( - tmp_path: Path, - test_case: UnsupportedDurationLimitTestCase, -) -> None: - (tmp_path / "sqlbuild_project.toml").write_text( - """ -name = "shop" -adapter = "duckdb" -default_target = "dev" - -[targets.dev.execution_limits] -max_duration = "{test_case.max_duration}" -remediation = "{test_case.remediation}" -""".strip().format(test_case=test_case), - encoding="utf-8", - ) - - with pytest.raises(CliUserError) as raised: - resolve_build_invocation(request=BuildCommandRequest(project_dir=tmp_path)) - - assert raised.value.code == test_case.expected_code - assert test_case.expected_error_fragment in raised.value.message - assert raised.value.help == test_case.remediation - - -@pytest.mark.parametrize( - "test_case", - [ - UnsupportedDurationLimitTestCase( - description="Snowflake target rejects duration beyond statement timeout range", - max_duration="8d", - remediation="Choose a duration supported by the configured warehouse.", - expected_code="C416", - expected_error_fragment="max_duration exceeds the largest duration supported", - ) - ], - ids=lambda case: case.description, -) -def test_given_duration_above_adapter_maximum_when_resolving_then_configuration_is_rejected( - tmp_path: Path, - test_case: UnsupportedDurationLimitTestCase, -) -> None: - (tmp_path / "sqlbuild_project.toml").write_text( - """ -name = "shop" -adapter = "snowflake" -default_target = "dev" - -[targets.dev] -database = "analytics" -schema = "dev" - -[targets.dev.execution_limits] -max_duration = "{test_case.max_duration}" -remediation = "{test_case.remediation}" -""".strip().format(test_case=test_case), - encoding="utf-8", - ) - - with pytest.raises(CliUserError) as raised: - resolve_build_invocation(request=BuildCommandRequest(project_dir=tmp_path)) - - assert raised.value.code == test_case.expected_code - assert test_case.expected_error_fragment in raised.value.message - assert raised.value.help == test_case.remediation diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py index 4fb59e14a1..2d40c76480 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_full_refresh.py @@ -6,17 +6,15 @@ import pytest -from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( - enforce_snapshot_full_refresh_policy, -) -from sqlbuild.cli.commands.constants import SNAPSHOT_FULL_REFRESH_BUILD_HELP +from sqlbuild._native import enforce_plan_safety_policies from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.spec.contracts.models import SnapshotsConfig +from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( SnapshotFullRefreshPolicyTestCase, ) from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning.helpers import ( + build_policy_gates, build_snapshot_full_refresh_entry, ) @@ -79,13 +77,18 @@ def test_given_unsafe_snapshot_full_refresh_when_enforcing_policy_then_raises_us test_case: SnapshotFullRefreshPolicyTestCase, ) -> None: with pytest.raises(CliUserError) as exc_info: - enforce_snapshot_full_refresh_policy( + enforce_plan_safety_policies( plan=test_case.plan_output, + gates=build_policy_gates( + allow_snapshot_full_refresh=test_case.allow_snapshot_full_refresh + ), snapshots_config=test_case.snapshots_config, - allow_snapshot_full_refresh=test_case.allow_snapshot_full_refresh, - non_interactive_help=SNAPSHOT_FULL_REFRESH_BUILD_HELP, - input_stream=_InputStream(test_case.input_text, is_tty=test_case.input_is_tty), - output_stream=StringIO(), + target_name="dev", + execution_limits=ExecutionLimitsConfig(), + streams=( + _InputStream(test_case.input_text, is_tty=test_case.input_is_tty), + StringIO(), + ), ) error: CliUserError = exc_info.value @@ -138,13 +141,16 @@ def test_given_safe_or_confirmed_snapshot_full_refresh_when_enforcing_policy_the ) -> None: output_stream: StringIO = StringIO() - enforce_snapshot_full_refresh_policy( + enforce_plan_safety_policies( plan=test_case.plan_output, + gates=build_policy_gates(allow_snapshot_full_refresh=test_case.allow_snapshot_full_refresh), snapshots_config=test_case.snapshots_config, - allow_snapshot_full_refresh=test_case.allow_snapshot_full_refresh, - non_interactive_help=SNAPSHOT_FULL_REFRESH_BUILD_HELP, - input_stream=_InputStream(test_case.input_text, is_tty=test_case.input_is_tty), - output_stream=output_stream, + target_name="dev", + execution_limits=ExecutionLimitsConfig(), + streams=( + _InputStream(test_case.input_text, is_tty=test_case.input_is_tty), + output_stream, + ), ) assert output_stream.getvalue() == test_case.expected_output diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py index 406e6f8b0d..6580f51a52 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_plan_policies.py @@ -4,10 +4,8 @@ import pytest -from sqlbuild.cli.commands._helpers.build_planning.plan_policies import ( - dbt_interop_plan_safety_gates, - enforce_plan_safety_policies, -) +from sqlbuild._native import enforce_plan_safety_policies +from sqlbuild.cli.commands._helpers.dbt.sqlbuild_work import dbt_interop_plan_safety_gates from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py index 1570d995eb..c97cf962e7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_retention_decrease.py @@ -4,17 +4,16 @@ import pytest -from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( - enforce_retention_decrease_policy, -) -from sqlbuild.cli.commands.constants import RETENTION_DECREASE_BUILD_HELP +from sqlbuild._native import enforce_plan_safety_policies from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.compiler.planner.types import RetentionDirection +from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( RetentionDecreasePolicyTestCase, ) from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning.helpers import ( + build_policy_gates, build_retention_entry, ) @@ -78,12 +77,16 @@ def test_given_unsafe_retention_decrease_when_enforcing_then_raises_user_error( test_case: RetentionDecreasePolicyTestCase, ) -> None: with pytest.raises(CliUserError) as exc_info: - enforce_retention_decrease_policy( + enforce_plan_safety_policies( plan=test_case.plan_output, - allow_retention_decrease=test_case.allow_retention_decrease, - non_interactive_help=RETENTION_DECREASE_BUILD_HELP, - input_stream=_InputStream(test_case.input_text, is_tty=test_case.input_is_tty), - output_stream=StringIO(), + gates=build_policy_gates(allow_retention_decrease=test_case.allow_retention_decrease), + snapshots_config=SnapshotsConfig(), + target_name="dev", + execution_limits=ExecutionLimitsConfig(), + streams=( + _InputStream(test_case.input_text, is_tty=test_case.input_is_tty), + StringIO(), + ), ) error: CliUserError = exc_info.value @@ -136,12 +139,16 @@ def test_given_safe_or_confirmed_retention_change_when_enforcing_then_allows_exe ) -> None: output_stream: StringIO = StringIO() - enforce_retention_decrease_policy( + enforce_plan_safety_policies( plan=test_case.plan_output, - allow_retention_decrease=test_case.allow_retention_decrease, - non_interactive_help=RETENTION_DECREASE_BUILD_HELP, - input_stream=_InputStream(test_case.input_text, is_tty=test_case.input_is_tty), - output_stream=output_stream, + gates=build_policy_gates(allow_retention_decrease=test_case.allow_retention_decrease), + snapshots_config=SnapshotsConfig(), + target_name="dev", + execution_limits=ExecutionLimitsConfig(), + streams=( + _InputStream(test_case.input_text, is_tty=test_case.input_is_tty), + output_stream, + ), ) assert output_stream.getvalue() == test_case.expected_output diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py index 50f57acdae..d4f97d6e22 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_planning/test_table_type.py @@ -4,16 +4,15 @@ import pytest -from sqlbuild.cli.commands._helpers.build_planning.table_type import ( - enforce_table_type_downgrade_policy, -) -from sqlbuild.cli.commands.constants import TABLE_TYPE_DOWNGRADE_BUILD_HELP +from sqlbuild._native import enforce_plan_safety_policies from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.compiler.planner.models import PlanOutput +from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, SnapshotsConfig from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning._test_types import ( TableTypeDowngradePolicyTestCase, ) from tests.unit.src.sqlbuild.cli.commands._helpers.build_planning.helpers import ( + build_policy_gates, build_table_type_entry, ) @@ -63,12 +62,18 @@ def test_given_unsafe_table_type_downgrade_when_enforcing_then_raises_user_error test_case: TableTypeDowngradePolicyTestCase, ) -> None: with pytest.raises(CliUserError) as exc_info: - enforce_table_type_downgrade_policy( + enforce_plan_safety_policies( plan=test_case.plan_output, - allow_table_type_downgrade=test_case.allow_table_type_downgrade, - non_interactive_help=TABLE_TYPE_DOWNGRADE_BUILD_HELP, - input_stream=_InputStream(test_case.input_text, is_tty=test_case.input_is_tty), - output_stream=StringIO(), + gates=build_policy_gates( + allow_table_type_downgrade=test_case.allow_table_type_downgrade + ), + snapshots_config=SnapshotsConfig(), + target_name="dev", + execution_limits=ExecutionLimitsConfig(), + streams=( + _InputStream(test_case.input_text, is_tty=test_case.input_is_tty), + StringIO(), + ), ) error: CliUserError = exc_info.value @@ -123,12 +128,16 @@ def test_given_safe_or_confirmed_table_type_change_when_enforcing_then_allows_ex ) -> None: output: StringIO = StringIO() - enforce_table_type_downgrade_policy( + enforce_plan_safety_policies( plan=test_case.plan_output, - allow_table_type_downgrade=test_case.allow_table_type_downgrade, - non_interactive_help=TABLE_TYPE_DOWNGRADE_BUILD_HELP, - input_stream=_InputStream(test_case.input_text, is_tty=test_case.input_is_tty), - output_stream=output, + gates=build_policy_gates(allow_table_type_downgrade=test_case.allow_table_type_downgrade), + snapshots_config=SnapshotsConfig(), + target_name="dev", + execution_limits=ExecutionLimitsConfig(), + streams=( + _InputStream(test_case.input_text, is_tty=test_case.input_is_tty), + output, + ), ) assert output.getvalue() == test_case.expected_output diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py index c193fdb396..a3207baee3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/cost/test_refresh.py @@ -9,8 +9,7 @@ import pytest -from sqlbuild._native import read_cost_run, write_cost_run -from sqlbuild.cli.commands._helpers.cost.refresh import refresh_pending_cost_run +from sqlbuild._native import read_cost_run, refresh_pending_cost_run, write_cost_run from sqlbuild.cli.commands.models import CostCommandRequest from sqlbuild.cost.constants import LEDGER_PERSISTENCE_FAILURE_LIMITATION_PREFIX from sqlbuild.cost.models import CostRunRecord, RunCostSummary @@ -84,23 +83,23 @@ def test_given_pending_run_when_showing_detail_then_persisted_summary_is_refresh record=replace(record, cost=replace(record.cost, status=test_case.initial_status)), ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.discover_project_inputs", + "sqlbuild._native.discover_project_inputs", lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_effective_adapter_name", + "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_adapter", + "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", lambda **kwargs: _RefreshingCostAdapter(), ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_project_connection_config", + "sqlbuild._native.resolve_project_connection_config", lambda **kwargs: {"database": "ANALYTICS"}, ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.build_effective_target_config", + "sqlbuild.compiler.compile.main._effective_target.build_effective_target_config", lambda **kwargs: SimpleNamespace(database=None), ) request: CostCommandRequest = CostCommandRequest( @@ -188,19 +187,19 @@ def test_given_useful_partial_when_refresh_degrades_then_persisted_summary_is_pr query_count=test_case.candidate_query_count, ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.discover_project_inputs", + "sqlbuild._native.discover_project_inputs", lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_effective_adapter_name", + "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_adapter", + "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", lambda **kwargs: _StaticCostAdapter(candidate), ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_project_connection_config", + "sqlbuild._native.resolve_project_connection_config", lambda **kwargs: {}, ) request: CostCommandRequest = CostCommandRequest( @@ -271,19 +270,19 @@ def test_given_persisted_ledger_failure_when_refresh_has_no_coverage_then_failur query_count=test_case.candidate_query_count, ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.discover_project_inputs", + "sqlbuild._native.discover_project_inputs", lambda **kwargs: SimpleNamespace(project_config=object(), local_config=object()), ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_effective_adapter_name", + "sqlbuild.spec.contracts.main.resolve_effective_adapter_name.resolve_effective_adapter_name", lambda **kwargs: "snowflake", ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_adapter", + "sqlbuild.cli.commands._helpers.runtime.adapters.resolve_adapter", lambda **kwargs: _StaticCostAdapter(candidate), ) monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.cost.refresh.resolve_project_connection_config", + "sqlbuild._native.resolve_project_connection_config", lambda **kwargs: {}, ) request: CostCommandRequest = CostCommandRequest( diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py index f305772503..74548abcd1 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py @@ -2,8 +2,8 @@ import pytest +from sqlbuild._native import render_query_result from sqlbuild.adapter.contract.models import QueryResult -from sqlbuild.cli.commands._helpers.query.output import render_query_result from tests.unit.src.sqlbuild.cli.commands._helpers.query._test_types import ( QueryOutputTestCase, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/_test_types.py deleted file mode 100644 index 0108a2e288..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/_test_types.py +++ /dev/null @@ -1,53 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class NamespaceCase: - description: str - cli: str | None = None - environment: tuple[tuple[str, str], ...] = () - local: str | None = None - project: str | None = None - expected_value: str | None = None - expected_source: str = "unset" - - -@dataclass(frozen=True) -class InvalidNamespaceCase: - description: str - value: object - expected_error: str = "1–128" - - -@dataclass(frozen=True) -class InvalidNamespaceConfigCase: - description: str - filename: str - value_toml: str - expected_error: str = "scenario.run_namespace must be 1–128" - - -@dataclass(frozen=True) -class SelectScenariosTestCase: - description: str - selectors: tuple[str, ...] - exclude: tuple[str, ...] - expected_scenario_names: tuple[str, ...] - - -@dataclass(frozen=True) -class SelectScenariosErrorTestCase: - description: str - selectors: tuple[str, ...] - exclude: tuple[str, ...] - expected_error_fragment: str - expected_error_code: str - - -@dataclass(frozen=True) -class ScenarioCompilePresentationTestCase: - description: str - tty: bool - expected_fragment: str - unexpected_fragment: str - expected_terminal: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py deleted file mode 100644 index 8cbbdcb454..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py +++ /dev/null @@ -1,76 +0,0 @@ -from io import StringIO -from pathlib import Path -from typing import Any -from unittest.mock import Mock - -import pytest - -from sqlbuild.cli.commands._helpers.scenario_execution import runner -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledSqlScenario, -) -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -class ScenarioProgressStream(StringIO): - def __init__(self, *, tty: bool) -> None: - super().__init__() - self._tty: bool = tty - - def isatty(self) -> bool: - return self._tty - - -def configure_scenario_runner( - *, - monkeypatch: pytest.MonkeyPatch, - stream: ScenarioProgressStream, - compile_project: Any, -) -> None: - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig(name="project", adapter="duckdb"), - local_config=LocalConfig(), - ) - monkeypatch.setattr(runner.sys, "stdout", stream) - monkeypatch.setattr(runner, "supports_color", lambda: False) - monkeypatch.setattr(runner, "discover_project_inputs", lambda **_: discovered_inputs) - monkeypatch.setattr(runner, "resolve_effective_adapter_name", lambda **_: "duckdb") - monkeypatch.setattr(runner, "resolve_adapter", lambda **_: Mock()) - monkeypatch.setattr(runner, "resolve_project_connection_config", lambda **_: {}) - monkeypatch.setattr(runner, "resolve_external_sql_reference_resolver", lambda **_: None) - monkeypatch.setattr(runner, "run_compile_only_pipeline", compile_project) - monkeypatch.setattr(runner, "select_scenarios", lambda **_: ()) - monkeypatch.setattr(runner, "run_warehouse_scenarios", lambda **_: 0) - - -def build_project_with_scenarios(project_dir: Path) -> CompiledProjectView: - scenarios: tuple[CompiledSqlScenario, ...] = tuple( - CompiledSqlScenario( - key=CompiledObjectKey(CompiledResourceType.SQL_SCENARIO, scenario_name), - name=scenario_name, - scenario_file=DiscoveredSqlScenarioFile( - file_path=project_dir / relative_path, - relative_path=Path(relative_path), - contents="SCENARIO(description 'Test scenario.');\n\nSELECT 1\n", - header_values={}, - sql_body="SELECT 1", - name=scenario_name, - ), - sql_body="SELECT 1", - ) - for scenario_name, relative_path in ( - ("orders_paid", "tests/scenarios/orders/paid.sql"), - ("orders_refund", "tests/scenarios/orders/refund.sql"), - ) - ) - return CompiledProject( - run_id="test_run", - effective_target_name=None, - effective_connection={}, - effective_vars={}, - sql_scenarios=scenarios, - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py deleted file mode 100644 index af05cab4c6..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Namespace precedence and rejection before execution.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild._native import discover_project_inputs -from sqlbuild.cli.commands._helpers.scenario_execution.namespace import resolve_scenario_namespace -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.discovery.exceptions import ProjectConfigError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.main.parse_scenario_run_namespace import parse_scenario_run_namespace -from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( - resolve_effective_scenario_config, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, ScenarioConfig -from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_execution._test_types import ( - InvalidNamespaceCase, - InvalidNamespaceConfigCase, - NamespaceCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - NamespaceCase( - "CLI wins", - cli="job-1", - environment=(("SQLBUILD_SCENARIO_NAMESPACE", "job-2"),), - local="local", - project="project", - expected_value="job-1", - expected_source="cli", - ), - NamespaceCase( - "environment wins", - environment=(("SQLBUILD_SCENARIO_NAMESPACE", "job-2"),), - local="local", - project="project", - expected_value="job-2", - expected_source="env", - ), - NamespaceCase( - "local wins", - local="local", - project="project", - expected_value="local", - expected_source="local config", - ), - NamespaceCase( - "project fallback", - project="project", - expected_value="project", - expected_source="project config", - ), - NamespaceCase("unset preserves default"), - NamespaceCase( - "CLI overrides invalid environment", - cli="job-1", - environment=(("SQLBUILD_SCENARIO_NAMESPACE", ""),), - expected_value="job-1", - expected_source="cli", - ), - ), - ids=lambda case: case.description, -) -def test_given_namespace_sources_when_resolving_then_highest_precedence_wins( - test_case: NamespaceCase, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.delenv("SQLBUILD_SCENARIO_NAMESPACE", raising=False) - for key, value in test_case.environment: - monkeypatch.setenv(key, value) - inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig( - name="orders", - adapter="duckdb", - scenario=ScenarioConfig(run_namespace=test_case.project), - ), - local_config=LocalConfig(scenario=ScenarioConfig(run_namespace=test_case.local)), - ) - resolved, namespace = resolve_scenario_namespace(inputs=inputs, cli_value=test_case.cli) - assert namespace.value == test_case.expected_value - assert namespace.source == test_case.expected_source - assert ( - resolve_effective_scenario_config( - project_config=resolved.project_config, local_config=resolved.local_config - ).run_namespace - == test_case.expected_value - ) - - -@pytest.mark.parametrize( - "test_case", - ( - InvalidNamespaceCase("empty", ""), - InvalidNamespaceCase("whitespace", " "), - InvalidNamespaceCase("slash", "job/1"), - InvalidNamespaceCase("colon", "job:1"), - InvalidNamespaceCase("newline", "job\n"), - InvalidNamespaceCase("non ASCII", "é"), - InvalidNamespaceCase("too long", "x" * 129), - InvalidNamespaceCase("numeric", 42), - InvalidNamespaceCase("explicit null", None), - ), - ids=lambda case: case.description, -) -def test_given_invalid_namespace_when_parsing_then_rejects(test_case: InvalidNamespaceCase) -> None: - with pytest.raises(SpecConfigError, match=test_case.expected_error): - parse_scenario_run_namespace(test_case.value) - - -@pytest.mark.parametrize( - "test_case", - ( - NamespaceCase("all allowed characters", cli="AZaz09-_.", expected_value="AZaz09-_."), - NamespaceCase("maximum length", cli="x" * 128, expected_value="x" * 128), - ), - ids=lambda case: case.description, -) -def test_given_valid_namespace_when_parsing_then_preserves_value(test_case: NamespaceCase) -> None: - assert parse_scenario_run_namespace(test_case.cli) == test_case.expected_value - - -@pytest.mark.parametrize( - "test_case", - (InvalidNamespaceCase("empty environment does not fall back", ""),), - ids=lambda case: case.description, -) -def test_given_empty_environment_when_resolving_then_does_not_fall_back( - test_case: InvalidNamespaceCase, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setenv("SQLBUILD_SCENARIO_NAMESPACE", str(test_case.value)) - inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig( - name="orders", adapter="duckdb", scenario=ScenarioConfig(run_namespace="project") - ), - local_config=LocalConfig(), - ) - with pytest.raises(CliUserError, match=test_case.expected_error): - resolve_scenario_namespace(inputs=inputs, cli_value=None) - - -@pytest.mark.parametrize( - "test_case", - (NamespaceCase("both config files", project="project", local="local", expected_value="local"),), - ids=lambda case: case.description, -) -def test_given_project_and_local_config_when_discovering_then_loads_namespace( - test_case: NamespaceCase, tmp_path: Path -) -> None: - (tmp_path / "sqlbuild_project.toml").write_text( - f'name = "orders"\nadapter = "duckdb"\n[scenario]\nrun_namespace = "{test_case.project}"\n' - ) - (tmp_path / "sqlbuild_local.toml").write_text( - f'[scenario]\nrun_namespace = "{test_case.local}"\n' - ) - inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - assert ( - resolve_effective_scenario_config( - project_config=inputs.project_config, local_config=inputs.local_config - ).run_namespace - == test_case.expected_value - ) - - -@pytest.mark.parametrize( - "test_case", - ( - InvalidNamespaceConfigCase("project empty", "sqlbuild_project.toml", '""'), - InvalidNamespaceConfigCase("local invalid characters", "sqlbuild_local.toml", '"job/1"'), - InvalidNamespaceConfigCase("project numeric", "sqlbuild_project.toml", "42"), - InvalidNamespaceConfigCase("local boolean", "sqlbuild_local.toml", "true"), - ), - ids=lambda case: case.description, -) -def test_given_invalid_config_namespace_when_discovering_then_schema_validation_rejects( - test_case: InvalidNamespaceConfigCase, tmp_path: Path -) -> None: - (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') - with (tmp_path / test_case.filename).open("a") as config: - config.write(f"\n[scenario]\nrun_namespace = {test_case.value_toml}\n") - with pytest.raises(ProjectConfigError, match=test_case.expected_error): - discover_project_inputs(project_dir=tmp_path) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_selection.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_selection.py deleted file mode 100644 index 20d39fed5b..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_selection.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.cli.commands._helpers.scenario_execution.selection import select_scenarios -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.compile.models import CompiledSqlScenario -from sqlbuild.compiler.compile.types import CompiledProjectView -from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_execution._test_types import ( - SelectScenariosErrorTestCase, - SelectScenariosTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_execution.helpers import ( - build_project_with_scenarios, -) - - -@pytest.mark.parametrize( - "test_case", - [ - SelectScenariosTestCase( - description="excludes scenarios from explicit selection", - selectors=("tests/scenarios",), - exclude=("orders_refund",), - expected_scenario_names=("orders_paid",), - ) - ], - ids=lambda case: case.description, -) -def test_given_scenario_selectors_when_excluding_then_returns_remaining_scenarios( - test_case: SelectScenariosTestCase, - tmp_path: Path, -) -> None: - project_dir: Path = tmp_path - project: CompiledProjectView = build_project_with_scenarios(project_dir) - - selected: tuple[CompiledSqlScenario, ...] = select_scenarios( - project=project, - selectors=test_case.selectors, - exclude=test_case.exclude, - project_dir=project_dir, - ) - - assert tuple(scenario.name for scenario in selected) == test_case.expected_scenario_names - - -@pytest.mark.parametrize( - "test_case", - [ - SelectScenariosErrorTestCase( - description="rejects upstream graph selector", - selectors=("+orders_paid",), - exclude=(), - expected_error_fragment="uses graph operators, which are not supported", - expected_error_code="C457", - ), - SelectScenariosErrorTestCase( - description="rejects path graph selector", - selectors=("orders_paid~orders_refund",), - exclude=(), - expected_error_fragment="uses graph operators, which are not supported", - expected_error_code="C457", - ), - ], - ids=lambda case: case.description, -) -def test_given_graph_scenario_selector_when_selecting_then_raises_clear_error( - test_case: SelectScenariosErrorTestCase, - tmp_path: Path, -) -> None: - with pytest.raises(CliUserError) as exc_info: - select_scenarios( - project=build_project_with_scenarios(tmp_path), - selectors=test_case.selectors, - exclude=test_case.exclude, - project_dir=tmp_path, - ) - - assert test_case.expected_error_fragment in str(exc_info.value) - assert exc_info.value.code == test_case.expected_error_code diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_output/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_output/_test_types.py deleted file mode 100644 index 320c95eed7..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_output/_test_types.py +++ /dev/null @@ -1,15 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ScenarioRunOutputTestCase: - description: str - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class ScenarioCaptureOutputTestCase: - description: str - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_output/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_output/test_output.py deleted file mode 100644 index 876799e793..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_output/test_output.py +++ /dev/null @@ -1,129 +0,0 @@ -from __future__ import annotations - -from io import StringIO -from pathlib import Path - -import pytest - -from sqlbuild.cli.commands._helpers.scenario_capture.capture_run import _write_capture_result -from sqlbuild.cli.commands._helpers.scenario_output.result_output import write_scenario_result -from sqlbuild.compiler.planner.types import ScenarioArtifactKind -from sqlbuild.executor.scenario.models import ( - ScenarioExpectedExpectationExecutionResult, - ScenarioRunResult, - ScenarioSnapshotCaptureRelationResult, - ScenarioSnapshotCaptureResult, - ScenarioSnapshotCaptureRunResult, -) -from sqlbuild.executor.scheduling.types import ExecutionStatus -from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_output._test_types import ( - ScenarioCaptureOutputTestCase, - ScenarioRunOutputTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - ScenarioRunOutputTestCase( - description="styles scenario and check statuses semantically", - expected_fragments=( - "orders_paid", - "\033[32mPASS\033[0m", - "expected fact_orders", - "\033[38;5;167mFAIL\033[0m 2 mismatched", - ), - ), - ScenarioRunOutputTestCase( - description="leaves no-color scenario output plain", - expected_fragments=("orders_paid", "PASS", "FAIL 2 mismatched"), - unexpected_fragments=("\033[",), - ), - ), - ids=lambda case: case.description, -) -def test_given_scenario_result_when_writing_output_then_styles_statuses( - test_case: ScenarioRunOutputTestCase, -) -> None: - stream: StringIO = StringIO() - write_scenario_result( - result=ScenarioRunResult( - scenario_name="orders_paid", - status=ExecutionStatus.SUCCESS, - expected_results=( - ScenarioExpectedExpectationExecutionResult( - scenario_name="orders_paid", - model_name="fact_orders", - status=ExecutionStatus.FAILED, - mismatched_row_count=2, - ), - ), - ), - stream=stream, - use_color="\033[" not in test_case.unexpected_fragments, - ) - - output: str = stream.getvalue() - for fragment in test_case.expected_fragments: - assert fragment in output - for fragment in test_case.unexpected_fragments: - assert fragment not in output - - -@pytest.mark.parametrize( - "test_case", - ( - ScenarioCaptureOutputTestCase( - description="styles capture result and relation statuses semantically", - expected_fragments=( - "orders_paid", - "\033[32mPASS\033[0m", - "source raw__orders", - "\033[38;5;167mFAIL\033[0m 4 rows, 128 B", - ), - ), - ScenarioCaptureOutputTestCase( - description="leaves no-color capture output plain", - expected_fragments=("orders_paid", "PASS", "source raw__orders", "FAIL 4 rows"), - unexpected_fragments=("\033[",), - ), - ), - ids=lambda case: case.description, -) -def test_given_scenario_capture_result_when_writing_output_then_styles_statuses( - test_case: ScenarioCaptureOutputTestCase, -) -> None: - stream: StringIO = StringIO() - output_path: Path = Path(__file__) - _write_capture_result( - result=ScenarioSnapshotCaptureRunResult( - scenario_name="orders_paid", - status=ExecutionStatus.SUCCESS, - capture_result=ScenarioSnapshotCaptureResult( - scenario_name="orders_paid", - status=ExecutionStatus.SUCCESS, - manifest_path=output_path, - relation_results=( - ScenarioSnapshotCaptureRelationResult( - kind=ScenarioArtifactKind.SOURCE, - logical_name="raw__orders", - source_relation="raw.orders", - file_path=output_path, - status=ExecutionStatus.FAILED, - row_count=4, - byte_count=128, - ), - ), - ), - ), - scenario_plan=None, - project_dir=output_path.parent, - stream=stream, - use_color="\033[" not in test_case.unexpected_fragments, - ) - - output: str = stream.getvalue() - for fragment in test_case.expected_fragments: - assert fragment in output - for fragment in test_case.unexpected_fragments: - assert fragment not in output diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py index a062c17353..6ee8bb077c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py @@ -5,8 +5,8 @@ import pytest -from sqlbuild.cli.commands.main.project import _dag as dag_command -from sqlbuild.cli.commands.main.project._dag import run_dag +from sqlbuild._native import run_dag_command +from sqlbuild.cli.commands._helpers.runtime import adapters as adapter_resolution from tests.unit.src.sqlbuild.cli.commands.main.dag._test_types import ( DagCommandTestCase, PythonDagCommandTestCase, @@ -40,12 +40,12 @@ def test_given_advisory_over_broad_global_when_running_dag_then_outputs_static_g ) -> None: project_dir: Path = prepare_advisory_placement_dag_project(tmp_path) monkeypatch.setattr( - dag_command, + adapter_resolution, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) - exit_code: int = run_dag( + exit_code: int = run_dag_command( project_dir=project_dir, no_sql_validation=True, json_output=True, @@ -80,12 +80,12 @@ def test_given_local_project_when_running_dag_json_then_outputs_static_graph_wit ) -> None: project_dir: Path = prepare_static_dag_project(tmp_path) monkeypatch.setattr( - dag_command, + adapter_resolution, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) - exit_code: int = run_dag( + exit_code: int = run_dag_command( project_dir=project_dir, no_sql_validation=True, json_output=True, @@ -171,12 +171,12 @@ def test_given_python_nodes_when_running_dag_json_then_outputs_artifact_metadata ) -> None: project_dir: Path = prepare_python_dag_project(tmp_path) monkeypatch.setattr( - dag_command, + adapter_resolution, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter(), ) - exit_code: int = run_dag( + exit_code: int = run_dag_command( project_dir=project_dir, no_sql_validation=True, json_output=True, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py index c053d7e024..dbcc9a35a7 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py @@ -6,17 +6,25 @@ @dataclass(frozen=True) class BuildProcessReportTestCase: description: str - expected_exit_code: int | None = None - expected_error_type: type[BaseException] | None = None + error_type: type[BaseException] + verbose: bool + expected_fragments: tuple[str, ...] + expected_absent_fragments: tuple[str, ...] @dataclass(frozen=True) -class BuildPartialTimingOutputTestCase: +class BuildDurationLimitTestCase: description: str - error_type: type[BaseException] - compile_seconds: float | None - planning_seconds: float | None - connection_seconds: float | None - execution_seconds: float | None - expected_fragments: tuple[str, ...] - expected_absent_fragments: tuple[str, ...] + project_toml: str + remediation: str + expected_code: str + expected_error_fragment: str + + +@dataclass(frozen=True) +class ScenarioCompilePresentationTestCase: + description: str + tty: bool + expected_fragment: str + unexpected_fragment: str + expected_terminal: str diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py index 77c9a121eb..6da7fb9048 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/helpers.py @@ -1,20 +1,91 @@ """Test helpers for build execution command boundaries.""" -from types import ModuleType +import sys +from contextlib import nullcontext +from io import StringIO +from pathlib import Path +from typing import Any from unittest.mock import Mock import pytest -from sqlbuild.cli.commands.models import BuildInvocation -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig +from sqlbuild import _native as native +from sqlbuild.cli.commands._helpers.planning import external_refs as external_refs_module +from sqlbuild.cli.commands._helpers.runtime import adapters as adapters_module +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSqlScenario +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile +from sqlbuild.diagnostics.main import process_resource_reporting as resource_reporting_module +from sqlbuild.executor.pipeline.main import run as executor_run_module +from sqlbuild.presentation.main import supports_color as supports_color_module +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig -def stub_resolved_build_invocation( - *, monkeypatch: pytest.MonkeyPatch, build_module: ModuleType +def fail_build_discovery(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) -> Mock: + """Make project discovery raise and record the debug resource reporter calls.""" + + def discover(**_kwargs: object) -> DiscoveredProjectInputs: + raise error + + reporter: Mock = Mock(return_value=nullcontext()) + monkeypatch.setattr(native, "discover_project_inputs", discover) + monkeypatch.setattr(resource_reporting_module, "process_resource_reporting", reporter) + return reporter + + +class ScenarioProgressStream(StringIO): + """A progress stream that reports whether it is a terminal.""" + + def __init__(self, *, tty: bool) -> None: + super().__init__() + self._tty: bool = tty + + def isatty(self) -> bool: + return self._tty + + +def configure_native_scenario_command( + *, + monkeypatch: pytest.MonkeyPatch, + stream: ScenarioProgressStream, + compile_project: Any, ) -> None: - """Keep command-wrapper tests isolated from project discovery.""" + """Isolate the native scenario command from discovery, adapters and the warehouse.""" + + discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( + project_config=ProjectConfig(name="project", adapter="duckdb"), + local_config=LocalConfig(), + ) + monkeypatch.setattr(sys, "stdout", stream) + monkeypatch.setattr(supports_color_module, "supports_color", lambda: False) + monkeypatch.setattr(native, "discover_project_inputs", lambda **_: discovered_inputs) + monkeypatch.setattr(native, "resolve_project_connection_config", lambda **_: {}) + monkeypatch.setattr(native, "run_compile_only_pipeline", compile_project) + monkeypatch.setattr(adapters_module, "resolve_adapter", lambda **_: Mock()) + monkeypatch.setattr( + external_refs_module, "resolve_external_sql_reference_resolver", lambda **_: None + ) + monkeypatch.setattr(executor_run_module, "run_scenario_test_pipeline", lambda **_: ()) + + +def build_compiled_scenario_project(project_dir: Path) -> Mock: + """A compile result with one scenario under tests/scenarios.""" - invocation: Mock = Mock(spec=BuildInvocation) - invocation.execution_limits = ExecutionLimitsConfig() - invocation.effective_target_name = None - monkeypatch.setattr(build_module, "resolve_build_invocation", lambda **_kwargs: invocation) + scenario_file: DiscoveredSqlScenarioFile = DiscoveredSqlScenarioFile( + file_path=project_dir / "tests/scenarios/orders_paid.sql", + relative_path=Path("tests/scenarios/orders_paid.sql"), + contents="SCENARIO(description 'Test scenario.');\n\nSELECT 1\n", + header_values={}, + sql_body="SELECT 1", + name="orders_paid", + ) + pipeline_result: Mock = Mock() + pipeline_result.project.sql_scenarios = ( + CompiledSqlScenario( + key=CompiledObjectKey(CompiledResourceType.SQL_SCENARIO, "orders_paid"), + name="orders_paid", + scenario_file=scenario_file, + sql_body="SELECT 1", + ), + ) + return pipeline_result diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py index 19ada1111b..a72e7bdbb0 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_build_process_resources.py @@ -1,243 +1,106 @@ -"""Tests for debug process reporting across build outcomes.""" +"""Tests for debug process reporting and partial timings across failed native builds.""" from __future__ import annotations -from contextlib import nullcontext -from typing import Any +from pathlib import Path from unittest.mock import Mock import pytest -from sqlbuild.cli.commands.main.execution import _build as build_module +from sqlbuild._native import run_build_command +from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import BuildCommandRequest -from sqlbuild.diagnostics.models import PartialBuildPhaseTimings from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( - BuildPartialTimingOutputTestCase, + BuildDurationLimitTestCase, BuildProcessReportTestCase, ) -from tests.unit.src.sqlbuild.cli.commands.main.execution.helpers import ( - stub_resolved_build_invocation, -) +from tests.unit.src.sqlbuild.cli.commands.main.execution.helpers import fail_build_discovery @pytest.mark.parametrize( "test_case", [ BuildProcessReportTestCase( - description="successful debug build reports resources", - expected_exit_code=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_successful_debug_build_when_finishing_then_reports_process_resources( - test_case: BuildProcessReportTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - stub_resolved_build_invocation(monkeypatch=monkeypatch, build_module=build_module) - reporter: Mock = Mock() - monkeypatch.setattr(build_module, "process_resource_reporting", reporter) - reporter.return_value = nullcontext() - monkeypatch.setattr(build_module, "_run_build", lambda **_kwargs: test_case.expected_exit_code) - - exit_code: int = build_module.run_build(BuildCommandRequest(debug=True)) - - assert exit_code == test_case.expected_exit_code - reporter.assert_called_once_with(enabled=True) - - -@pytest.mark.parametrize( - "test_case", - [ + description="failed debug build reports resources and partial timings", + error_type=RuntimeError, + verbose=False, + expected_fragments=("Phase timings", "total"), + expected_absent_fragments=("compile", "planning", "execution"), + ), BuildProcessReportTestCase( - description="failed debug build reports resources", - expected_error_type=RuntimeError, - ) + description="interrupted debug build reports resources and partial timings", + error_type=KeyboardInterrupt, + verbose=True, + expected_fragments=("Phase timings", "total"), + expected_absent_fragments=("connection preparation",), + ), ], ids=lambda case: case.description, ) -def test_given_failed_debug_build_when_unwinding_then_reports_process_resources( +def test_given_failed_debug_build_when_unwinding_then_reports_resources_and_partial_timings( test_case: BuildProcessReportTestCase, monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + tmp_path: Path, ) -> None: - stub_resolved_build_invocation(monkeypatch=monkeypatch, build_module=build_module) - reporter: Mock = Mock() - - def fail_build(**_kwargs: object) -> int: - raise RuntimeError("build failed") - - monkeypatch.setattr(build_module, "process_resource_reporting", reporter) - reporter.return_value = nullcontext() - monkeypatch.setattr(build_module, "_run_build", fail_build) - - with pytest.raises(test_case.expected_error_type or RuntimeError): - build_module.run_build(BuildCommandRequest(debug=True)) - - reporter.assert_called_once_with(enabled=True) + reporter: Mock = fail_build_discovery(monkeypatch=monkeypatch, error=test_case.error_type()) - -@pytest.mark.parametrize( - "test_case", - [ - BuildProcessReportTestCase( - description="interrupted debug build reports resources", - expected_error_type=KeyboardInterrupt, + with pytest.raises(test_case.error_type): + run_build_command( + request=BuildCommandRequest(project_dir=tmp_path, debug=True, verbose=test_case.verbose) ) - ], - ids=lambda case: case.description, -) -def test_given_interrupted_debug_build_when_unwinding_then_reports_process_resources( - test_case: BuildProcessReportTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - stub_resolved_build_invocation(monkeypatch=monkeypatch, build_module=build_module) - reporter: Mock = Mock() - - def interrupt_build(**_kwargs: object) -> int: - raise KeyboardInterrupt - - monkeypatch.setattr(build_module, "process_resource_reporting", reporter) - reporter.return_value = nullcontext() - monkeypatch.setattr(build_module, "_run_build", interrupt_build) - - with pytest.raises(test_case.expected_error_type or KeyboardInterrupt): - build_module.run_build(BuildCommandRequest(debug=True)) reporter.assert_called_once_with(enabled=True) + errors: str = capsys.readouterr().err + fragment: str + for fragment in test_case.expected_fragments: + assert fragment in errors + for fragment in test_case.expected_absent_fragments: + assert fragment not in errors @pytest.mark.parametrize( "test_case", [ - BuildPartialTimingOutputTestCase( - description="direct compile failure reports compile and total", - error_type=RuntimeError, - compile_seconds=1.0, - planning_seconds=None, - connection_seconds=None, - execution_seconds=None, - expected_fragments=("Phase timings", "compile", "total"), - expected_absent_fragments=("planning", "connection preparation", "execution"), - ), - BuildPartialTimingOutputTestCase( - description="virtual planning connection failure reports compile planning and total", - error_type=RuntimeError, - compile_seconds=1.0, - planning_seconds=4.0, - connection_seconds=None, - execution_seconds=None, - expected_fragments=("compile", "planning", "total"), - expected_absent_fragments=("connection preparation", "execution"), - ), - BuildPartialTimingOutputTestCase( - description="direct connection interruption reports preparation and total", - error_type=KeyboardInterrupt, - compile_seconds=1.0, - planning_seconds=2.0, - connection_seconds=3.0, - execution_seconds=None, - expected_fragments=("compile", "planning", "connection preparation", "total"), - expected_absent_fragments=("execution",), + BuildDurationLimitTestCase( + description="DuckDB target cannot guarantee statement cancellation", + project_toml=( + 'name = "shop"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' + '[targets.dev.execution_limits]\nmax_duration = "30m"\n' + ), + remediation="Use an adapter with safe statement cancellation.", + expected_code="C415", + expected_error_fragment="cannot cancel active statements safely", ), - BuildPartialTimingOutputTestCase( - description="virtual execution interruption reports all available phases", - error_type=KeyboardInterrupt, - compile_seconds=1.0, - planning_seconds=2.0, - connection_seconds=3.0, - execution_seconds=4.0, - expected_fragments=( - "compile", - "planning", - "connection preparation", - "execution", - "total", + BuildDurationLimitTestCase( + description="Snowflake target rejects duration beyond statement timeout range", + project_toml=( + 'name = "shop"\nadapter = "snowflake"\ndefault_target = "dev"\n\n' + '[targets.dev]\ndatabase = "analytics"\nschema = "dev"\n\n' + '[targets.dev.execution_limits]\nmax_duration = "8d"\n' ), - expected_absent_fragments=(), + remediation="Choose a duration supported by the configured warehouse.", + expected_code="C416", + expected_error_fragment="max_duration exceeds the largest duration supported", ), ], ids=lambda case: case.description, ) -def test_given_phase_failure_when_building_then_available_partial_timings_are_written( - test_case: BuildPartialTimingOutputTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - stub_resolved_build_invocation(monkeypatch=monkeypatch, build_module=build_module) - timing_tracker: Mock = Mock() - timing_tracker.snapshot.return_value = PartialBuildPhaseTimings( - compile_seconds=test_case.compile_seconds, - planning_seconds=test_case.planning_seconds, - connection_preparation_seconds=test_case.connection_seconds, - execution_seconds=test_case.execution_seconds, - total_seconds=10.0, - ) - timing_tracker.scope.return_value = nullcontext() - - def fail_build(**_kwargs: object) -> int: - raise test_case.error_type("phase failed") - - monkeypatch.setattr(build_module, "BuildPhaseTimingTracker", lambda: timing_tracker) - monkeypatch.setattr(build_module, "process_resource_reporting", lambda **_kwargs: nullcontext()) - monkeypatch.setattr(build_module, "_run_build", fail_build) - - with pytest.raises(test_case.error_type): - build_module.run_build(BuildCommandRequest(verbose=True)) - - output: str = capsys.readouterr().err - expected_fragment: str - for expected_fragment in test_case.expected_fragments: - assert expected_fragment in output - absent_fragment: str - for absent_fragment in test_case.expected_absent_fragments: - assert absent_fragment not in output - - -@pytest.mark.parametrize( - "test_case", - [ - BuildPartialTimingOutputTestCase( - description="JSON cost failure keeps stdout clean and writes partial timing to stderr", - error_type=RuntimeError, - compile_seconds=1.0, - planning_seconds=2.0, - connection_seconds=3.0, - execution_seconds=4.0, - expected_fragments=("Phase timings", "cost collection", "5.00s"), - expected_absent_fragments=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_json_cost_failure_when_building_then_partial_timing_only_uses_stderr( - test_case: BuildPartialTimingOutputTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], +def test_given_unenforceable_duration_limit_when_building_then_configuration_is_rejected( + test_case: BuildDurationLimitTestCase, + tmp_path: Path, ) -> None: - stub_resolved_build_invocation(monkeypatch=monkeypatch, build_module=build_module) - timing_tracker: Mock = Mock() - timing_tracker.snapshot.return_value = PartialBuildPhaseTimings( - compile_seconds=test_case.compile_seconds, - planning_seconds=test_case.planning_seconds, - connection_preparation_seconds=test_case.connection_seconds, - execution_seconds=test_case.execution_seconds, - cost_collection_seconds=5.0, - total_seconds=15.0, + (tmp_path / "sqlbuild_project.toml").write_text( + f'{test_case.project_toml}remediation = "{test_case.remediation}"\n', encoding="utf-8" ) - timing_tracker.scope.return_value = nullcontext() - def fail_build(**_kwargs: object) -> int: - raise test_case.error_type("cost finalization failed") + with pytest.raises(CliUserError) as raised: + run_build_command(request=BuildCommandRequest(project_dir=tmp_path)) - monkeypatch.setattr(build_module, "BuildPhaseTimingTracker", lambda: timing_tracker) - monkeypatch.setattr(build_module, "process_resource_reporting", lambda **_kwargs: nullcontext()) - monkeypatch.setattr(build_module, "_run_build", fail_build) + assert raised.value.code == test_case.expected_code + assert test_case.expected_error_fragment in raised.value.message + assert raised.value.help == test_case.remediation - with pytest.raises(test_case.error_type, match="cost finalization failed"): - build_module.run_build(BuildCommandRequest(verbose=True, json_output=True)) - captured: Any = capsys.readouterr() - assert captured.out == "" - expected_fragment: str - for expected_fragment in test_case.expected_fragments: - assert expected_fragment in captured.err +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_runner.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py similarity index 80% rename from tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_runner.py rename to tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py index 68c3a5ab28..f20b93c541 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_runner.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_scenario_command.py @@ -1,22 +1,23 @@ from __future__ import annotations from contextvars import Token -from typing import Any, cast +from pathlib import Path +from typing import Any from unittest.mock import Mock import pytest -from sqlbuild._native import NativeProgressProjector -from sqlbuild.cli.commands._helpers.scenario_execution import runner +from sqlbuild._native import NativeProgressProjector, run_scenario_test_command from sqlbuild.cli.commands.models import ScenarioTestCommandRequest from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope -from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_execution._test_types import ( +from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( ScenarioCompilePresentationTestCase, ) -from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_execution.helpers import ( +from tests.unit.src.sqlbuild.cli.commands.main.execution.helpers import ( ScenarioProgressStream, - configure_scenario_runner, + build_compiled_scenario_project, + configure_native_scenario_command, ) @@ -42,19 +43,19 @@ ) def test_given_scenario_compile_when_running_then_presentation_has_one_owner( monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, test_case: ScenarioCompilePresentationTestCase, ) -> None: stream: ScenarioProgressStream = ScenarioProgressStream(tty=test_case.tty) - pipeline_result: Mock = Mock() - pipeline_result.project.sql_scenarios = () + pipeline_result: Mock = build_compiled_scenario_project(tmp_path) def compile_project(**kwargs: Any) -> CompilePipelineResult: on_progress: Any = kwargs["on_progress"] on_progress("Compiling project...") on_progress("Compiled project.") - return cast(CompilePipelineResult, pipeline_result) + return pipeline_result - configure_scenario_runner( + configure_native_scenario_command( monkeypatch=monkeypatch, stream=stream, compile_project=compile_project ) events: list[LifecycleEvent] = [] @@ -66,13 +67,16 @@ def compile_project(**kwargs: Any) -> CompilePipelineResult: try: with dispatcher_scope(dispatcher): - exit_code: int = runner.run_scenario(ScenarioTestCommandRequest()) + exit_code: int = run_scenario_test_command( + request=ScenarioTestCommandRequest(project_dir=tmp_path) + ) finally: projector.restore(token) assert exit_code == 0 assert test_case.expected_fragment in stream.getvalue() assert test_case.unexpected_fragment not in stream.getvalue() + assert "PASS=0 FAIL=0 TOTAL=0" in stream.getvalue() assert tuple(event.event_type for event in events[-2:]) == ( "operation_started", test_case.expected_terminal, @@ -101,6 +105,7 @@ def compile_project(**kwargs: Any) -> CompilePipelineResult: ) def test_given_scenario_compile_failure_when_running_then_lifecycle_fails_and_progress_closes( monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, test_case: ScenarioCompilePresentationTestCase, ) -> None: stream: ScenarioProgressStream = ScenarioProgressStream(tty=test_case.tty) @@ -110,7 +115,9 @@ def fail_compile(**kwargs: Any) -> CompilePipelineResult: on_progress("Compiling project...") raise RuntimeError("compile failed") - configure_scenario_runner(monkeypatch=monkeypatch, stream=stream, compile_project=fail_compile) + configure_native_scenario_command( + monkeypatch=monkeypatch, stream=stream, compile_project=fail_compile + ) events: list[LifecycleEvent] = [] dispatcher: EventDispatcher = EventDispatcher() projector: NativeProgressProjector = NativeProgressProjector(stream=stream, use_color=False) @@ -121,7 +128,7 @@ def fail_compile(**kwargs: Any) -> CompilePipelineResult: try: with dispatcher_scope(dispatcher): with pytest.raises(RuntimeError, match="compile failed"): - runner.run_scenario(ScenarioTestCommandRequest()) + run_scenario_test_command(request=ScenarioTestCommandRequest(project_dir=tmp_path)) finally: projector.restore(token) @@ -131,3 +138,7 @@ def fail_compile(**kwargs: Any) -> CompilePipelineResult: "operation_started", test_case.expected_terminal, ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py index 43e3916b90..f2b1e72ffe 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py @@ -5,7 +5,9 @@ import pytest from _pytest.capture import CaptureResult -import sqlbuild.cli.commands.main.workspace._init as init_module +import sqlbuild._native as native_module +import sqlbuild.presentation.main.supports_color as color_module +from sqlbuild._native import run_init_command from tests.unit.src.sqlbuild.cli.commands.main.init._test_types import InitScaffoldTestCase @@ -84,10 +86,10 @@ def test_given_empty_directory_when_running_init_then_scaffolds_project_director ) -> None: project_dir: Path = tmp_path / test_case.project_dir_name project_dir.mkdir() - monkeypatch.setattr(init_module, "update_sqlbuild_skills", lambda *, project_dir: None) - monkeypatch.setattr(init_module, "supports_color", lambda: False) + monkeypatch.setattr(native_module, "update_sqlbuild_skills", lambda *, project_dir: None) + monkeypatch.setattr(color_module, "supports_color", lambda: False) - result: int = init_module.run_init(project_dir) + result: int = run_init_command(project_dir) captured: CaptureResult[str] = capsys.readouterr() assert result == 0 @@ -128,10 +130,10 @@ def test_given_color_terminal_when_running_init_then_it_styles_heading( ) -> None: project_dir: Path = tmp_path / test_case.project_dir_name project_dir.mkdir() - monkeypatch.setattr(init_module, "update_sqlbuild_skills", lambda *, project_dir: None) - monkeypatch.setattr(init_module, "supports_color", lambda: True) + monkeypatch.setattr(native_module, "update_sqlbuild_skills", lambda *, project_dir: None) + monkeypatch.setattr(color_module, "supports_color", lambda: True) - result: int = init_module.run_init(project_dir) + result: int = run_init_command(project_dir) captured: CaptureResult[str] = capsys.readouterr() assert result == 0 diff --git a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py index 0a32625d93..b4c4ec7dbc 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/inspection/cost/test_cost.py @@ -8,8 +8,7 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild._native import write_cost_run -from sqlbuild.cli.commands._helpers.cost.command import _parse_since_bound, run_cost_command +from sqlbuild._native import parse_cost_since_bound, run_cost_command, write_cost_run from sqlbuild.cli.commands.exceptions import CliUserError from tests.unit.src.sqlbuild.cli.commands.main.inspection.cost._test_types import ( CostCommandTestCase, @@ -237,7 +236,7 @@ def test_given_supported_relative_since_duration_when_parsing_then_returns_expec test_case: CostRelativeSinceTestCase, ) -> None: before: datetime = datetime.now(UTC) - result: datetime | None = _parse_since_bound(value=test_case.value) + result: datetime | None = parse_cost_since_bound(value=test_case.value) after: datetime = datetime.now(UTC) assert result is not None @@ -267,7 +266,7 @@ def test_given_unsupported_relative_since_duration_when_parsing_then_raises_clea ) -> None: assert test_case.expected_error_fragment is not None with pytest.raises(CliUserError, match=test_case.expected_error_fragment): - _parse_since_bound(value=test_case.value) + parse_cost_since_bound(value=test_case.value) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/inspection/query/test_query.py b/tests/unit/src/sqlbuild/cli/commands/main/inspection/query/test_query.py index 16e03d9725..3b1024e00d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/inspection/query/test_query.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/inspection/query/test_query.py @@ -5,8 +5,8 @@ import duckdb import pytest +from sqlbuild._native import run_query_command from sqlbuild.cli.commands.exceptions import QueryExecutionError -from sqlbuild.cli.commands.main.inspection._query import run_query from tests.unit.src.sqlbuild.cli.commands.main.inspection.query._test_types import ( QueryExecutionErrorTestCase, ) @@ -38,7 +38,7 @@ def test_given_warehouse_rejects_query_when_running_query_then_raises_cli_error( ) with pytest.raises(QueryExecutionError) as raised: - run_query(project_dir=tmp_path, sql=test_case.sql) + run_query_command(project_dir=tmp_path, sql=test_case.sql) assert raised.value.code == test_case.expected_code assert raised.value.exit_code == test_case.expected_exit_code diff --git a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/_test_types.py b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/_test_types.py index 53195f551d..4b115bcb5b 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/_test_types.py @@ -40,3 +40,11 @@ class ExecutionJsonCostTestCase: class ExecutionJsonMicrobatchLimitTestCase: description: str expected_microbatch: dict[str, object] + + +@dataclass(frozen=True) +class ExecutionJsonAuditDegradationTestCase: + description: str + result: BuildExecutionResult + expected_status: str + expected_projection: dict[str, int] diff --git a/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_degradable_build_json.py b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_degradable_build_json.py new file mode 100644 index 0000000000..a21a7b7d90 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/output/main/build_execution_json/test_degradable_build_json.py @@ -0,0 +1,60 @@ +"""Degradable build execution JSON boundary tests.""" + +from __future__ import annotations + +import json + +import pytest + +from sqlbuild._native import format_degradable_build_execution_json +from sqlbuild.executor.build.models import BuildExecutionResult +from sqlbuild.executor.build.types import BuildStatus +from tests.unit.src.sqlbuild.cli.output.main.build_execution_json._test_types import ( + ExecutionJsonAuditDegradationTestCase, +) +from tests.unit.src.sqlbuild.cli.output.main.plan.helpers import build_plan_output + + +@pytest.mark.parametrize( + "test_case", + [ + ExecutionJsonAuditDegradationTestCase( + description="audit result persistence failure", + result=BuildExecutionResult( + status=BuildStatus.SUCCESS, + audit_result_projection_attempted_count=3, + audit_result_projection_written_count=0, + audit_result_projection_failed_count=3, + ), + expected_status="success", + expected_projection={"attempted_count": 3, "written_count": 0, "failed_count": 3}, + ) + ], + ids=lambda case: case.description, +) +def test_given_audit_history_write_failure_when_formatting_json_then_reports_bounded_degradation( + test_case: ExecutionJsonAuditDegradationTestCase, +) -> None: + payload: dict[str, object] = json.loads( + format_degradable_build_execution_json( + result=test_case.result, + plan=build_plan_output(), + python_node_results=(), + python_check_results=(), + command="build", + run_id="run", + cost=None, + execution_succeeded=True, + ) + ) + + assert payload["status"] == test_case.expected_status + assert payload["projection_degraded"] is True + assert payload["audit_result_projection"] == test_case.expected_projection + assert payload["projection_degradation_reasons"] == [ + {"reason": "audit_result_projection_failure", **test_case.expected_projection} + ] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index c9bd338579..1107511c5a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -17,12 +17,12 @@ from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView from sqlbuild.compiler.discovery.models import DiscoveredSeedFile from sqlbuild.compiler.fingerprints.models import Fingerprint +from sqlbuild.compiler.planner.main.commands._audit import build_audit_command_plan from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context from sqlbuild.compiler.planner.main.commands._scope import resolve_static_command_scope from sqlbuild.compiler.planner.main.commands._seed import build_seed_command_plan from sqlbuild.compiler.planner.main.commands._seed_state import read_selected_seed_fingerprints from sqlbuild.compiler.planner.main.commands._sql_test import build_test_command_plan -from sqlbuild.compiler.planner.main.commands.audit import build_audit_command_plan from sqlbuild.compiler.planner.models import ( PlannerRelationsContext, PlannerScope, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index 14309ea470..43b1eff0ca 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -238,11 +238,11 @@ class BatchedPreparationFailureTestCase: @dataclass(frozen=True) class CollectProjectFilesTestCase: - """Test case for sharing discovered SQL text when collecting lint inputs.""" + """Test case for collecting exact authored SQL text for lint inputs.""" description: str model_bytes: bytes - expected_shared: bool + expected_contents: str @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index 760401f6ca..99430dead4 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -4,8 +4,20 @@ from pathlib import Path -from sqlbuild._native import scan_headers, sql_body_ranges -from sqlbuild.lint.models import HeaderSpan, LintBody +from sqlbuild._native import ( + lint_native_sql_bodies, + neutralize_lint_interpolation, + restore_lint_interpolation, + scan_headers, + sql_body_ranges, +) +from sqlbuild.lint.models import ( + HeaderSpan, + InterpolationSite, + LintBody, + LintConfig, + LintViolation, +) from sqlbuild.lint.types import NativeLintPreparationRequest, NativePreparedSql @@ -131,3 +143,29 @@ def files_resolving_to(*, files: dict[Path, str], selected: frozenset[Path]) -> """Return the files whose fully resolved path is one of the selected paths.""" return dict(filter(lambda item: item[0].resolve() in selected, files.items())) + + +def neutralize_interpolation( + *, body: str, dialect: str, for_formatting: bool = False +) -> tuple[str, tuple[InterpolationSite, ...]]: + """Neutralize interpolation natively and return the sites as lint models.""" + + neutralized, sites = neutralize_lint_interpolation(body, dialect, for_formatting=for_formatting) + return neutralized, tuple(InterpolationSite(*site) for site in sites) + + +def restore_interpolation(*, fixed: str, sites: tuple[InterpolationSite, ...]) -> tuple[bool, str]: + """Restore neutralized interpolation natively; returns whether it succeeded and the text.""" + + return restore_lint_interpolation(fixed, [tuple(vars(site).values()) for site in sites]) + + +def run_native_sql_lint( + *, bodies: tuple[LintBody, ...], contents_by_path: dict[Path, str], config: LintConfig +) -> dict[Path, tuple[LintViolation, ...]]: + """Lint bodies natively and group the violations by file.""" + + grouped: dict[Path, list[LintViolation]] = {} + for violation in lint_native_sql_bodies(list(bodies), contents_by_path, config): + grouped.setdefault(violation.file_path, []).append(violation) + return {path: tuple(violations) for path, violations in grouped.items()} diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py index 61b810260e..87595805d7 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_expansion.py @@ -7,8 +7,8 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import run_lint from sqlbuild.lint.exceptions import ProjectCompileError -from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult from tests.unit.src.sqlbuild.lint._helpers._test_types import ( BatchedPreparationFailureTestCase, diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py index 0b46b6c6b4..f38b7fff12 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py @@ -2,257 +2,25 @@ from __future__ import annotations -import json from pathlib import Path -from unittest.mock import Mock import pytest from sqlbuild.compiler.compile.models import ExpansionSpan -from sqlbuild.lint._helpers import native_sql from sqlbuild.lint.models import LintBody, LintConfig, LintViolation from tests.unit.src.sqlbuild.lint._helpers._test_types import ( - GeneratedRangeFallbackTestCase, - InvalidNativeSqlResponseTestCase, ManyViolationLocationTestCase, - NativeParseIsolationTestCase, - NativeSqlDependencyCacheTestCase, NativeSqlFindingTestCase, NativeSqlFixTestCase, - NativeSqlReuseTestCase, ReservedCteLintTestCase, ) from tests.unit.src.sqlbuild.lint._helpers.helpers import ( authored_positions, + run_native_sql_lint, unused_cte_orders_file, ) -@pytest.mark.parametrize( - "test_case", - [ - InvalidNativeSqlResponseTestCase( - description="response must be an object", - response="[]", - expected_message="non-object response", - ), - InvalidNativeSqlResponseTestCase( - description="response version must match", - response='{"version":2,"diagnostics":[]}', - expected_message="unsupported response version", - ), - InvalidNativeSqlResponseTestCase( - description="diagnostics must be a list", - response='{"version":1,"diagnostics":{}}', - expected_message="missing a diagnostics list", - ), - InvalidNativeSqlResponseTestCase( - description="source span must fit SQL text", - response=( - '{"version":1,"diagnostics":[' - '{"code":"SQBRSQL001","message":"bad","start":99,"end":100}]}' - ), - expected_message="invalid code, message, or source span", - ), - InvalidNativeSqlResponseTestCase( - description="remediation must be present", - response=( - '{"version":1,"diagnostics":[{"code":"SQBRSQL001","message":"bad","start":0,"end":1}]}' - ), - expected_message="invalid code, message, or source span", - ), - InvalidNativeSqlResponseTestCase( - description="fix edit must have a valid authored range", - response=( - '{"version":1,"diagnostics":[{"code":"SQBRSQL001","message":"bad",' - '"remediation":"repair","start":0,"end":1,' - '"fix":{"start":1,"end":1,"replacement":"IS"}}]}' - ), - expected_message="invalid fix edit", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_native_response_when_linting_then_boundary_fails_closed( - monkeypatch: pytest.MonkeyPatch, - test_case: InvalidNativeSqlResponseTestCase, - tmp_path: Path, -) -> None: - monkeypatch.setattr( - native_sql._native, - "lint_sql_batch_json", - lambda _request: json.dumps([{"Ok": json.loads(test_case.response)}]), - ) - target: Path = tmp_path / "model.sql" - body: LintBody = LintBody( - file_path=target, - body_start=0, - body_end=8, - lint_text="SELECT 1", - passes=(), - ) - - violations: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( - bodies=(body,), - contents_by_path={target: "SELECT 1"}, - config=LintConfig(dialect="duckdb"), - ) - (fault,) = violations[target] - assert fault.code == "L003" - assert fault.severity == "fault" - assert test_case.expected_message in fault.message - assert fault.fix is None - - -@pytest.mark.parametrize( - "test_case", - [ - NativeParseIsolationTestCase( - description="one unsupported body", - error_message="Parse error at line 2, column 8: unsupported GROUPING SETS", - expected_position=(2, 8), - ) - ], - ids=lambda case: case.description, -) -def test_given_one_unparseable_body_when_linting_then_other_files_still_complete( - test_case: NativeParseIsolationTestCase, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - lint_response: Mock = Mock( - return_value=json.dumps( - [{"Err": test_case.error_message}, {"Ok": {"version": 1, "diagnostics": []}}] - ) - ) - monkeypatch.setattr(native_sql._native, "lint_sql_batch_json", lint_response) - failed_path: Path = tmp_path / "failed.sql" - healthy_path: Path = tmp_path / "healthy.sql" - failed_text: str = "SELECT category\nGROUP BY GROUPING SETS ((category))" - healthy_text: str = "SELECT 1" - bodies: tuple[LintBody, ...] = ( - LintBody( - file_path=failed_path, - body_start=0, - body_end=len(failed_text), - lint_text=failed_text, - passes=(), - ), - LintBody( - file_path=healthy_path, - body_start=0, - body_end=len(healthy_text), - lint_text=healthy_text, - passes=(), - ), - ) - - result: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( - bodies=bodies, - contents_by_path={failed_path: failed_text, healthy_path: healthy_text}, - config=LintConfig(dialect="snowflake"), - ) - - assert tuple(result) == (failed_path,) - assert result[failed_path][0].code == "L003" - assert (result[failed_path][0].line, result[failed_path][0].column) == ( - test_case.expected_position - ) - assert result[failed_path][0].fix is None - - -@pytest.mark.parametrize( - "test_case", - [NativeSqlReuseTestCase(description="identical expanded SQL bodies", expected_call_count=1)], - ids=lambda case: case.description, -) -def test_given_identical_expanded_bodies_when_linting_then_native_analysis_is_reused( - test_case: NativeSqlReuseTestCase, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - _ = test_case - calls: list[str] = [] - - def lint_sql_json(request: str) -> str: - calls.extend(json.dumps(payload) for payload in json.loads(request)) - return json.dumps([{"Ok": {"version": 1, "diagnostics": []}} for _ in json.loads(request)]) - - monkeypatch.setattr(native_sql._native, "lint_sql_batch_json", lint_sql_json) - paths: tuple[Path, Path] = (tmp_path / "first.sql", tmp_path / "second.sql") - bodies: tuple[LintBody, ...] = tuple( - LintBody( - file_path=path, - body_start=0, - body_end=8, - lint_text="SELECT 1", - passes=(), - ) - for path in paths - ) - - result: dict[Path, tuple] = native_sql.run_native_sql_lint( - bodies=bodies, - contents_by_path={path: "SELECT 1" for path in paths}, - config=LintConfig(dialect="duckdb"), - ) - - assert result == {} - assert len(calls) == test_case.expected_call_count - - -@pytest.mark.parametrize( - "test_case", - [ - NativeSqlDependencyCacheTestCase( - description="same SQL with different dependency identities", - expected_call_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_different_dependency_identities_when_linting_then_native_cache_is_separate( - test_case: NativeSqlDependencyCacheTestCase, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - calls: list[str] = [] - - def lint_sql_json(request: str) -> str: - calls.extend(json.dumps(payload) for payload in json.loads(request)) - return json.dumps([{"Ok": {"version": 1, "diagnostics": []}} for _ in json.loads(request)]) - - monkeypatch.setattr(native_sql._native, "lint_sql_batch_json", lint_sql_json) - sql: str = "WITH imported_orders AS (SELECT * FROM __sqb_lint_0__) SELECT 1" - paths: tuple[Path, Path] = (tmp_path / "dependency.sql", tmp_path / "dynamic.sql") - bodies: tuple[LintBody, LintBody] = ( - LintBody( - file_path=paths[0], - body_start=0, - body_end=len(sql), - lint_text=sql, - passes=(), - dependency_identifiers=("__sqb_lint_0__",), - ), - LintBody( - file_path=paths[1], - body_start=0, - body_end=len(sql), - lint_text=sql, - passes=(), - ), - ) - - result: dict[Path, tuple] = native_sql.run_native_sql_lint( - bodies=bodies, - contents_by_path={path: sql for path in paths}, - config=LintConfig(dialect="duckdb"), - ) - - assert result == {} - assert len(calls) == test_case.expected_call_count - - @pytest.mark.parametrize( "test_case", [ @@ -277,7 +45,7 @@ def test_given_reserved_harness_cte_when_linting_then_framework_input_is_not_rep passes=(), ) - result: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( + result: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( bodies=(body,), contents_by_path={target: test_case.sql}, config=LintConfig(dialect="duckdb"), @@ -316,7 +84,7 @@ def test_given_multi_branch_boolean_case_when_linting_then_partial_fix_is_withhe passes=(), ) - result: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( + result: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( bodies=(body,), contents_by_path={target: test_case.sql}, config=LintConfig( @@ -363,7 +131,7 @@ def test_given_generated_interior_with_authored_boundaries_when_linting_then_fix ), ), ) - result: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( + result: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( bodies=(body,), contents_by_path={target: test_case.sql.replace("42", "__")}, config=LintConfig(dialect="duckdb", enabled_native_rules=(test_case.expected_code,)), @@ -418,7 +186,7 @@ def test_given_cross_join_used_downstream_when_linting_then_no_unused_join_is_re passes=(), ) - result: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( + result: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( bodies=(body,), contents_by_path={target: test_case.sql}, config=LintConfig( @@ -432,75 +200,6 @@ def test_given_cross_join_used_downstream_when_linting_then_no_unused_join_is_re ) -@pytest.mark.parametrize( - "test_case", - [ - GeneratedRangeFallbackTestCase( - description="diagnostic crossing one macro expansion", - authored="SELECT a@macro()b", - expanded="SELECT aLIMITb", - diagnostic_start=7, - diagnostic_end=14, - expected_position=(1, 8), - ) - ], - ids=lambda case: case.description, -) -def test_given_diagnostic_crossing_generated_sql_when_mapping_then_range_falls_back_to_point( - test_case: GeneratedRangeFallbackTestCase, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """Do not fabricate an authored range when a native span crosses an expansion.""" - - response: str = json.dumps( - { - "version": 1, - "diagnostics": [ - { - "code": "SQBRSQL004", - "message": "bad", - "remediation": "fix it", - "start": test_case.diagnostic_start, - "end": test_case.diagnostic_end, - } - ], - } - ) - monkeypatch.setattr( - native_sql._native, - "lint_sql_batch_json", - lambda _request: json.dumps([{"Ok": json.loads(response)}]), - ) - target: Path = tmp_path / "model.sql" - body: LintBody = LintBody( - file_path=target, - body_start=0, - body_end=len(test_case.authored), - lint_text=test_case.expanded, - passes=( - ( - ExpansionSpan( - source_start=8, - source_end=16, - output_start=8, - output_end=13, - ), - ), - ), - ) - - result: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( - bodies=(body,), - contents_by_path={target: test_case.authored}, - config=LintConfig(dialect="duckdb"), - ) - - violation: LintViolation = result[target][0] - assert (violation.line, violation.column) == test_case.expected_position - assert (violation.end_line, violation.end_column) == (None, None) - - @pytest.mark.parametrize( "test_case", [ @@ -533,7 +232,7 @@ def test_given_many_violations_per_file_when_linting_then_locations_match_author dialect="duckdb", enabled_native_rules=(test_case.expected_code,) ) - result: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( + result: dict[Path, tuple[LintViolation, ...]] = run_native_sql_lint( bodies=bodies, contents_by_path=contents_by_path, config=config ) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py index 09c7298277..25c07dd5c7 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -6,10 +6,8 @@ import pytest -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import collect_lint_project_files, discover_project_inputs, run_lint from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.lint._helpers.project_files import collect_project_files -from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult from tests.unit.src.sqlbuild.lint._helpers._test_types import ( CollectProjectFilesTestCase, @@ -28,12 +26,12 @@ CollectProjectFilesTestCase( description="identical text shares the discovered string", model_bytes=b"MODEL (materialized table);\nSELECT 1 AS order_id\n", - expected_shared=True, + expected_contents="MODEL (materialized table);\nSELECT 1 AS order_id\n", ), CollectProjectFilesTestCase( description="carriage returns keep the exact authored text", model_bytes=b"MODEL (materialized table);\r\nSELECT 1 AS order_id\r\n", - expected_shared=False, + expected_contents="MODEL (materialized table);\r\nSELECT 1 AS order_id\r\n", ), ], ids=lambda case: case.description, @@ -45,17 +43,14 @@ def test_given_discovered_project_when_collecting_files_then_returns_exact_autho project_dir=tmp_path, model_bytes=test_case.model_bytes ) inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - discovered: dict[Path, str] = {item.file_path: item.contents for item in inputs.model_files} - files: dict[Path, str] = collect_project_files(project_dir=tmp_path, discovered_inputs=inputs) + files: dict[Path, str] = collect_lint_project_files(project_dir=tmp_path) shared: LintRunResult = run_lint( project_dir=tmp_path, config=LintConfig(), discovered_inputs=inputs ) fresh: LintRunResult = run_lint(project_dir=tmp_path, config=LintConfig()) - assert files == collect_project_files(project_dir=tmp_path) - assert files[model_path] == test_case.model_bytes.decode("utf-8") - assert (files[model_path] is discovered[model_path]) is test_case.expected_shared + assert files[model_path] == test_case.expected_contents assert shared.source_texts == fresh.source_texts assert shared.violations == fresh.violations assert shared.files_checked == fresh.files_checked == 2 @@ -82,13 +77,18 @@ def test_given_selected_resolved_paths_when_collecting_files_then_matches_resolv ) -> None: write_linked_orders_project(project_dir=tmp_path) selected: frozenset[Path] = frozenset({(tmp_path / test_case.selected).resolve()}) - every_file: dict[Path, str] = collect_project_files(project_dir=tmp_path) + every_file: dict[Path, str] = collect_lint_project_files(project_dir=tmp_path) - collected: dict[Path, str] = collect_project_files( + collected: dict[Path, str] = collect_lint_project_files( project_dir=tmp_path, selected_paths=selected ) - from_sources: dict[Path, str] = collect_project_files( - project_dir=tmp_path, selected_paths=selected, source_files=every_file + from_sources: dict[Path, str] = dict( + run_lint( + project_dir=tmp_path, + config=LintConfig(native_enabled=False), + selected_paths=selected, + source_files=every_file, + ).source_texts ) relative: set[str] = {path.relative_to(tmp_path).as_posix() for path in collected} diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py index e4e7cb90fe..d214b5b1fb 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py @@ -5,11 +5,6 @@ import pytest import sqlbuild._native as _native -from sqlbuild.lint._helpers.sqlbuild_tokens import ( - neutralize_interpolation, - restore_interpolation, -) -from sqlbuild.lint.exceptions import InterpolationRestorationError from sqlbuild.lint.models import InterpolationSite from tests.unit.src.sqlbuild.lint._helpers._test_types import ( DialectNeutralizeInterpolationTestCase, @@ -19,6 +14,10 @@ RestoreInterpolationTestCase, UnicodePreparationTestCase, ) +from tests.unit.src.sqlbuild.lint._helpers.helpers import ( + neutralize_interpolation, + restore_interpolation, +) @pytest.mark.parametrize( @@ -50,7 +49,7 @@ def test_given_unicode_before_intrinsics_when_preparing_then_lint_preserves_sema == lint_sites[0].sentinel ) assert len(format_sites) == 4 - assert restore_interpolation(fixed=format_sql, sites=format_sites) == sql + assert restore_interpolation(fixed=format_sql, sites=format_sites) == (True, sql) assert all( sql[site.original_start : site.original_end] == site.original_text for site in format_sites ) @@ -366,7 +365,8 @@ def test_given_fixed_text_when_restoring_then_original_interpolation_returns( ) -> None: sites: tuple[InterpolationSite, ...] _neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") - restored: str = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) + succeeded, restored = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) + assert succeeded assert restored == test_case.expected_restored @@ -388,14 +388,14 @@ def test_given_fixed_text_when_restoring_then_original_interpolation_returns( ], ids=lambda case: case.description, ) -def test_given_mangled_sentinels_when_restoring_then_error_is_raised( +def test_given_mangled_sentinels_when_restoring_then_failure_is_reported( test_case: RestoreFailureTestCase, ) -> None: sites: tuple[InterpolationSite, ...] _neutralized, sites = neutralize_interpolation(body=test_case.body, dialect="generic") - with pytest.raises(InterpolationRestorationError) as error: - _ = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) - assert test_case.expected_message_fragment in str(error.value) + succeeded, message = restore_interpolation(fixed=test_case.fixed_neutralized, sites=sites) + assert not succeeded + assert test_case.expected_message_fragment in message @pytest.mark.parametrize( @@ -455,4 +455,4 @@ def test_given_dollar_quoted_text_when_neutralizing_then_python_and_native_sites assert native is not None assert native[0] == lint_sql assert tuple(InterpolationSite(*site) for site in native[1]) == lint_sites - assert restore_interpolation(fixed=format_sql, sites=format_sites) == sql + assert restore_interpolation(fixed=format_sql, sites=format_sites) == (True, sql) diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index e532fb69c7..0ce8cb5187 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -85,7 +85,7 @@ class FormatSelectionWorkTestCase: selected_file_count: int expected_fixture_parse_count: int expected_native_batch_count: int - expected_native_request_count: int + expected_native_file_count: int @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/lint/helpers.py b/tests/unit/src/sqlbuild/lint/helpers.py index d41a092202..70ab1cac2c 100644 --- a/tests/unit/src/sqlbuild/lint/helpers.py +++ b/tests/unit/src/sqlbuild/lint/helpers.py @@ -1,8 +1,14 @@ """Project writers shared by lint behavior tests.""" import itertools +import json import re from pathlib import Path +from typing import cast + +import pytest + +from sqlbuild._native import run_cli _PROJECT_TOML: str = 'name = "demo"\nadapter = "duckdb"\n' @@ -128,3 +134,25 @@ def reserved_keyword_pairs() -> tuple[tuple[str, str], ...]: for dialect in _DIALECTS: pairs.extend((dialect, keyword) for keyword in sorted(reserved_keywords(dialect))) return tuple(pairs) + + +def format_project_json( + *, project_dir: Path, capsys: pytest.CaptureFixture[str], arguments: tuple[str, ...] = () +) -> dict[str, object]: + """Run `sqb format --json` on a project and return its parsed report.""" + + _ = run_cli(["--project-dir", str(project_dir), "format", "--json", *arguments]) + report: dict[str, object] = json.loads(capsys.readouterr().out) + return report + + +def violation_reports(report: dict[str, object]) -> list[dict[str, object]]: + """Return the violation objects of a format report.""" + + return cast(list[dict[str, object]], report["violations"]) + + +def formatted_files(report: dict[str, object]) -> list[str]: + """Return the changed file paths of a format report.""" + + return cast(list[str], report["formatted_files"]) diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index 5607b79c00..0d452ce596 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from pathlib import Path from unittest.mock import Mock @@ -10,7 +9,6 @@ from _pytest.capture import CaptureResult import sqlbuild._native as native_module -import sqlbuild.cli.commands._helpers.lint.selection as lint_selection_module import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module from sqlbuild._native import run_cli as main from tests.unit.src.sqlbuild.lint._test_types import ( @@ -476,8 +474,8 @@ def test_given_fixture_only_mode_when_formatting_then_only_fixture_nulls_are_rew "__expected__rows AS (select 1 AS id, TRUE AS flag)", ) test_file.write_text(authored, encoding="utf-8") - native_format: Mock = Mock(wraps=native_module.format_sql_batch_json) - monkeypatch.setattr(native_module, "format_sql_batch_json", native_format) + native_format: Mock = Mock(wraps=native_module.format_lint_sql_bodies) + monkeypatch.setattr(native_module, "format_lint_sql_bodies", native_format) exit_code: int = main(["--project-dir", str(tmp_path), "format", *test_case.arguments]) @@ -496,7 +494,7 @@ def test_given_fixture_only_mode_when_formatting_then_only_fixture_nulls_are_rew selected_file_count=2, expected_fixture_parse_count=1, expected_native_batch_count=1, - expected_native_request_count=1, + expected_native_file_count=2, ) ], ids=lambda case: case.description, @@ -552,13 +550,13 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select unselected_file.parent.mkdir() unselected_file.write_text(fixture_sql, encoding="utf-8") fixture_parse: Mock = Mock(wraps=fixture_formatting_module.SqlTestCteExtractor.extract) - native_format: Mock = Mock(wraps=native_module.format_sql_batch_json) + native_format: Mock = Mock(wraps=native_module.format_lint_sql_bodies) full_discovery: Mock = Mock(side_effect=AssertionError("full discovery must not run")) project_graph: Mock = Mock(side_effect=AssertionError("project graph must not build")) monkeypatch.setattr(fixture_formatting_module.SqlTestCteExtractor, "extract", fixture_parse) - monkeypatch.setattr(native_module, "format_sql_batch_json", native_format) - monkeypatch.setattr(lint_selection_module, "discover_project_inputs", full_discovery) - monkeypatch.setattr(lint_selection_module, "build_project_graph", project_graph) + monkeypatch.setattr(native_module, "format_lint_sql_bodies", native_format) + monkeypatch.setattr(native_module, "discover_project_inputs", full_discovery) + monkeypatch.setattr(native_module, "build_project_graph", project_graph) exit_code: int = main( [ @@ -575,9 +573,7 @@ def test_given_test_path_selector_when_formatting_then_work_is_bounded_to_select assert f"{test_case.selected_file_count} files checked" in captured.err assert fixture_parse.call_count == test_case.expected_fixture_parse_count assert native_format.call_count == test_case.expected_native_batch_count - native_requests: object = json.loads(native_format.call_args.args[0]) - assert isinstance(native_requests, list) - assert len(native_requests) == test_case.expected_native_request_count + assert len(native_format.call_args.args[0]) == test_case.expected_native_file_count assert full_discovery.call_count == 0 assert project_graph.call_count == 0 assert all( diff --git a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py index 029bf538cd..411ca2e86f 100644 --- a/tests/unit/src/sqlbuild/lint/test_expanded_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_expanded_lint.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild._native import run_lint from sqlbuild.compiler.compile.main.expand_sql_with_spans import expand_sql_with_spans from sqlbuild.compiler.compile.main.map_expanded_offset import map_expanded_offset from sqlbuild.compiler.compile.models import MappedOffset, SqlExpansionContext @@ -15,7 +16,6 @@ from sqlbuild.lint._helpers.expansion import build_lint_expansion_context, prepare_lint_body from sqlbuild.lint._helpers.headers import lint_file_role from sqlbuild.lint.exceptions import ProjectCompileError -from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintBody, LintConfig, LintRunResult from tests.unit.src.sqlbuild.lint._test_types import ( ExpandedLintTestCase, diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index 5cd6a93000..4b08a93296 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -6,10 +6,8 @@ import pytest -from sqlbuild.lint._helpers.suppressions import apply_suppressions +from sqlbuild._native import apply_lint_suppressions, run_lint from sqlbuild.lint.constants import LINT_ENGINE_SQLBUILD, VIOLATION_SEVERITY_FAULT -from sqlbuild.lint.main.run_format import run_format -from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult, LintViolation from tests.unit.src.sqlbuild.lint._test_types import ( FixtureCeremonyFormatTestCase, @@ -19,6 +17,11 @@ LintBehaviorTestCase, LintProjectTestCase, ) +from tests.unit.src.sqlbuild.lint.helpers import ( + format_project_json, + formatted_files, + violation_reports, +) CLEAN_MODEL: str = 'MODEL (\n materialized table,\n description "ok"\n);\nSELECT 1 AS x FROM t\n' NO_DESCRIPTION_MODEL: str = "MODEL (\n materialized table\n);\nSELECT 1 AS x FROM t\n" @@ -136,7 +139,7 @@ def test_given_synthetic_project_when_linting_then_results_match_expected( ids=lambda case: case.description, ) def test_given_synthetic_project_when_formatting_then_results_match_expected( - test_case: FormatProjectTestCase, tmp_path: Path + test_case: FormatProjectTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") relative_path: str @@ -145,12 +148,14 @@ def test_given_synthetic_project_when_formatting_then_results_match_expected( target: Path = tmp_path / relative_path _ = target.parent.mkdir(parents=True, exist_ok=True) _ = target.write_text(contents, encoding="utf-8") - result: LintRunResult = run_format(project_dir=tmp_path, config=LintConfig()) - assert len(result.formatted_files) == test_case.expected_formatted_count - remaining: tuple = tuple(violation.code for violation in result.faults) + report: dict[str, object] = format_project_json(project_dir=tmp_path, capsys=capsys) + assert len(formatted_files(report)) == test_case.expected_formatted_count + remaining: tuple = tuple( + (violation["code"], violation["severity"]) for violation in violation_reports(report) + ) fault_code: str for fault_code in test_case.expected_fault_codes: - assert fault_code in remaining + assert (fault_code, "fault") in remaining relative_path: str fragment: str for relative_path, fragment in test_case.expected_written_fragments.items(): @@ -199,7 +204,9 @@ def test_given_synthetic_project_when_formatting_then_results_match_expected( ids=lambda case: case.description, ) def test_given_inherited_descriptions_when_formatting_then_faults_match_compile_resolution( - test_case: FormatDescriptionResolutionTestCase, tmp_path: Path + test_case: FormatDescriptionResolutionTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(test_case.project_toml, encoding="utf-8") relative_path: str @@ -209,12 +216,17 @@ def test_given_inherited_descriptions_when_formatting_then_faults_match_compile_ _ = target.parent.mkdir(parents=True, exist_ok=True) _ = target.write_text(contents, encoding="utf-8") - result: LintRunResult = run_format(project_dir=tmp_path, config=LintConfig(), write=False) - - assert tuple(sorted((fault.file_path.name, fault.code) for fault in result.faults)) == ( - test_case.expected_description_faults + report: dict[str, object] = format_project_json( + project_dir=tmp_path, capsys=capsys, arguments=("--check",) ) + assert tuple( + sorted( + (Path(str(violation["file"])).name, violation["code"], violation["severity"]) + for violation in violation_reports(report) + ) + ) == tuple((*fault, "fault") for fault in test_case.expected_description_faults) + @pytest.mark.parametrize( "test_case", @@ -230,20 +242,20 @@ def test_given_inherited_descriptions_when_formatting_then_faults_match_compile_ ids=lambda case: case.description, ) def test_given_authored_newline_style_when_formatting_then_style_is_preserved( - test_case: FormatNewlineTestCase, tmp_path: Path + test_case: FormatNewlineTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: target: Path = tmp_path / "models" / "crlf.sql" _ = target.parent.mkdir(parents=True) target.write_bytes(test_case.contents) _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") - _ = run_format(project_dir=tmp_path, config=LintConfig()) - second: LintRunResult = run_format(project_dir=tmp_path, config=LintConfig()) + _ = format_project_json(project_dir=tmp_path, capsys=capsys) + second: dict[str, object] = format_project_json(project_dir=tmp_path, capsys=capsys) written: bytes = target.read_bytes() assert test_case.expected_newline in written assert written.count(test_case.expected_newline) == written.count(b"\n") - assert second.formatted_files == () + assert second["formatted_files"] == [] @pytest.mark.parametrize( @@ -283,6 +295,7 @@ def test_given_non_ascii_sql_when_native_linting_then_span_maps_to_authored_colu def test_given_comment_free_body_when_formatting_twice_then_output_is_canonical_and_idempotent( test_case: LintBehaviorTestCase, tmp_path: Path, + capsys: pytest.CaptureFixture[str], ) -> None: _ = test_case _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") @@ -293,17 +306,11 @@ def test_given_comment_free_body_when_formatting_twice_then_output_is_canonical_ encoding="utf-8", ) - first: LintRunResult = run_format( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - ) - second: LintRunResult = run_format( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - ) + first: dict[str, object] = format_project_json(project_dir=tmp_path, capsys=capsys) + second: dict[str, object] = format_project_json(project_dir=tmp_path, capsys=capsys) - assert len(first.formatted_files) == test_case.expected_value - assert second.formatted_files == () + assert len(formatted_files(first)) == test_case.expected_value + assert second["formatted_files"] == [] assert "SELECT\n a,\n b\nFROM items\nWHERE a = 1\n" in target.read_text(encoding="utf-8") @@ -320,6 +327,7 @@ def test_given_comment_free_body_when_formatting_twice_then_output_is_canonical_ def test_given_commented_body_when_formatting_then_comment_is_preserved_in_canonical_output( test_case: LintBehaviorTestCase, tmp_path: Path, + capsys: pytest.CaptureFixture[str], ) -> None: _ = test_case _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") @@ -328,12 +336,9 @@ def test_given_commented_body_when_formatting_then_comment_is_preserved_in_canon target.parent.mkdir() _ = target.write_text(contents, encoding="utf-8") - result: LintRunResult = run_format( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - ) + report: dict[str, object] = format_project_json(project_dir=tmp_path, capsys=capsys) - assert result.formatted_files == (target,) + assert report["formatted_files"] == [str(target)] written: str = target.read_text(encoding="utf-8") assert str(test_case.expected_value) in written assert written == ( @@ -380,17 +385,17 @@ def test_given_commented_body_when_formatting_then_comment_is_preserved_in_canon ids=lambda case: case.description, ) def test_given_fixture_ceremony_variants_when_formatting_and_linting_then_forms_are_kept( - test_case: FixtureCeremonyFormatTestCase, tmp_path: Path + test_case: FixtureCeremonyFormatTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") target: Path = tmp_path / test_case.relative_path target.parent.mkdir(parents=True) _ = target.write_text(test_case.contents, encoding="utf-8") - formatted: LintRunResult = run_format(project_dir=tmp_path, config=LintConfig(dialect="duckdb")) + formatted: dict[str, object] = format_project_json(project_dir=tmp_path, capsys=capsys) linted: LintRunResult = run_lint(project_dir=tmp_path, config=LintConfig(dialect="duckdb")) - assert formatted.violations == () + assert formatted["violations"] == [] assert target.read_text(encoding="utf-8") == test_case.expected_contents assert linted.violations == () @@ -525,8 +530,6 @@ def test_given_header_fault_suppression_when_linting_then_mandatory_fault_remain engine=LINT_ENGINE_SQLBUILD, ) - result: list[LintViolation] = apply_suppressions( - violations=[mandatory], contents_by_path={target: contents}, allow_suppressions=True - ) + result: list[LintViolation] = apply_lint_suppressions([mandatory], {target: contents}, True) assert {item.code for item in result} == test_case.expected_value diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py index e2b44d8e12..60700b0045 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py @@ -101,3 +101,19 @@ class FullTargetResolutionTestCase: description: str project_target: TargetConfig expected_target: TargetConfig + + +@dataclass(frozen=True) +class ScenarioNamespaceConfigTestCase: + description: str + project: str + local: str + expected_value: str + + +@dataclass(frozen=True) +class InvalidScenarioNamespaceConfigTestCase: + description: str + filename: str + value_toml: str + expected_error: str = "scenario.run_namespace must be 1–128" diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py new file mode 100644 index 0000000000..d54a6514fe --- /dev/null +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_resolve_effective_scenario_config.py @@ -0,0 +1,71 @@ +"""Scenario run namespace configuration loading and validation.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import discover_project_inputs +from sqlbuild.compiler.discovery.exceptions import ProjectConfigError +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( + resolve_effective_scenario_config, +) +from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( + InvalidScenarioNamespaceConfigTestCase, + ScenarioNamespaceConfigTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + ( + ScenarioNamespaceConfigTestCase( + "both config files", project="project", local="local", expected_value="local" + ), + ), + ids=lambda case: case.description, +) +def test_given_project_and_local_config_when_discovering_then_loads_namespace( + test_case: ScenarioNamespaceConfigTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n[scenario]\nrun_namespace = "{test_case.project}"\n' + ) + (tmp_path / "sqlbuild_local.toml").write_text( + f'[scenario]\nrun_namespace = "{test_case.local}"\n' + ) + inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + assert ( + resolve_effective_scenario_config( + project_config=inputs.project_config, local_config=inputs.local_config + ).run_namespace + == test_case.expected_value + ) + + +@pytest.mark.parametrize( + "test_case", + ( + InvalidScenarioNamespaceConfigTestCase("project empty", "sqlbuild_project.toml", '""'), + InvalidScenarioNamespaceConfigTestCase( + "local invalid characters", "sqlbuild_local.toml", '"job/1"' + ), + InvalidScenarioNamespaceConfigTestCase("project numeric", "sqlbuild_project.toml", "42"), + InvalidScenarioNamespaceConfigTestCase("local boolean", "sqlbuild_local.toml", "true"), + ), + ids=lambda case: case.description, +) +def test_given_invalid_config_namespace_when_discovering_then_schema_validation_rejects( + test_case: InvalidScenarioNamespaceConfigTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + with (tmp_path / test_case.filename).open("a") as config: + config.write(f"\n[scenario]\nrun_namespace = {test_case.value_toml}\n") + with pytest.raises(ProjectConfigError, match=test_case.expected_error): + discover_project_inputs(project_dir=tmp_path) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])